Removing Unused Techpack dataipa code

Change-Id: I69665b12382d7fe1ce0a53f165a6bb03487a681c
Signed-off-by: Arnav Sharma <[email protected]>

Makefile

@@ -1,23 +1,3 @@
 # SPDX-License-Identifier: GPL-2.0-only
-#
-# Makefile for the MSM specific device drivers.
-#
-
-ifeq ($(CONFIG_ARCH_KONA), y)
-include $(srctree)/techpack/dataipa/config/konadataipa.conf
-endif
-
-ifeq ($(CONFIG_ARCH_KONA), y)
-LINUXINCLUDE    += -include $(srctree)/techpack/dataipa/config/konadataipaconf.h
-endif
-
-ifeq ($(CONFIG_ARCH_LITO), y)
-include $(srctree)/techpack/dataipa/config/litodataipa.conf
-endif
-
-ifeq ($(CONFIG_ARCH_LITO), y)
-LINUXINCLUDE    += -include $(srctree)/techpack/dataipa/config/litodataipaconf.h
-endif
-
-obj-$(CONFIG_GSI) += gsi/
-obj-$(CONFIG_IPA3) += ipa/
+ccflags-y := -Wno-unused-function
+obj-y := dataipa.o

config/konadataipa.conf

@@ -1,11 +0,0 @@
-export CONFIG_GSI=y
-export CONFIG_GSI_REGISTER_VERSION_2=y
-export CONFIG_IPA3=y
-export CONFIG_IPA_WDI_UNIFIED_API=y
-export CONFIG_RMNET_IPA3=y
-export CONFIG_RNDIS_IPA=y
-export CONFIG_IPA3_MHI_PRIME_MANAGER=y
-export CONFIG_IPA_UT=y
-export CONFIG_IPA3_REGDUMP=y
-export CONFIG_IPA3_REGDUMP_IPA_4_5=y
-export CONFIG_IPA3_REGDUMP_NUM_EXTRA_ENDP_REGS=y

config/konadataipaconf.h

@@ -1,16 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2019, The Linux Foundation. All rights reserved.
- */
-
-#define CONFIG_GSI 1
-#define CONFIG_GSI_REGISTER_VERSION_2 1
-#define CONFIG_IPA3 1
-#define CONFIG_IPA_WDI_UNIFIED_API 1
-#define CONFIG_RMNET_IPA3 1
-#define CONFIG_RNDIS_IPA 1
-#define CONFIG_IPA3_MHI_PRIME_MANAGER 1
-#define CONFIG_IPA_UT 1
-#define CONFIG_IPA3_REGDUMP 1
-#define CONFIG_IPA3_REGDUMP_IPA_4_5 1
-#define CONFIG_IPA3_REGDUMP_NUM_EXTRA_ENDP_REGS 0

config/litodataipa.conf

@@ -1,10 +0,0 @@
-export CONFIG_GSI=y
-export CONFIG_GSI_REGISTER_VERSION_2=y
-export CONFIG_IPA3=y
-export CONFIG_IPA_WDI_UNIFIED_API=y
-export CONFIG_RMNET_IPA3=y
-export CONFIG_RNDIS_IPA=y
-export CONFIG_IPA_UT=y
-export CONFIG_IPA3_REGDUMP=y
-export CONFIG_IPA3_REGDUMP_IPA_4_5=y
-export CONFIG_IPA3_REGDUMP_NUM_EXTRA_ENDP_REGS=y

config/litodataipaconf.h

@@ -1,15 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2019, The Linux Foundation. All rights reserved.
- */
-
-#define CONFIG_GSI 1
-#define CONFIG_GSI_REGISTER_VERSION_2 1
-#define CONFIG_IPA3 1
-#define CONFIG_IPA_WDI_UNIFIED_API 1
-#define CONFIG_RMNET_IPA3 1
-#define CONFIG_RNDIS_IPA 1
-#define CONFIG_IPA_UT 1
-#define CONFIG_IPA3_REGDUMP 1
-#define CONFIG_IPA3_REGDUMP_IPA_4_5 1
-#define CONFIG_IPA3_REGDUMP_NUM_EXTRA_ENDP_REGS 0

gsi/Makefile

@@ -1,6 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-
-gsidbg-$(CONFIG_DEBUG_FS) += gsi_dbg.o
-obj-$(CONFIG_GSI) += gsi.o gsidbg.o
-
-obj-$(CONFIG_IPA_EMULATION) += gsi_emulation.o

gsi/gsi.c

@@ -1,4389 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2015-2019, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/of.h>
-#include <linux/interrupt.h>
-#include <linux/io.h>
-#include <linux/log2.h>
-#include <linux/module.h>
-#include <linux/msm_gsi.h>
-#include <linux/platform_device.h>
-#include <linux/delay.h>
-#include "gsi.h"
-#include "gsi_reg.h"
-#include "gsi_emulation.h"
-
-#define GSI_CMD_TIMEOUT (5*HZ)
-#define GSI_START_CMD_TIMEOUT_MS 1000
-#define GSI_CMD_POLL_CNT 5
-#define GSI_STOP_CMD_TIMEOUT_MS 200
-#define GSI_MAX_CH_LOW_WEIGHT 15
-
-#define GSI_STOP_CMD_POLL_CNT 4
-#define GSI_STOP_IN_PROC_CMD_POLL_CNT 2
-
-#define GSI_RESET_WA_MIN_SLEEP 1000
-#define GSI_RESET_WA_MAX_SLEEP 2000
-#define GSI_CHNL_STATE_MAX_RETRYCNT 10
-
-#define GSI_STTS_REG_BITS 32
-
-#ifndef CONFIG_DEBUG_FS
-void gsi_debugfs_init(void)
-{
-}
-#endif
-
-static const struct of_device_id msm_gsi_match[] = {
-	{ .compatible = "qcom,msm_gsi", },
-	{ },
-};
-
-
-#if defined(CONFIG_IPA_EMULATION)
-static bool running_emulation = true;
-#else
-static bool running_emulation;
-#endif
-
-struct gsi_ctx *gsi_ctx;
-
-static void __gsi_config_type_irq(int ee, uint32_t mask, uint32_t val)
-{
-	uint32_t curr;
-
-	curr = gsi_readl(gsi_ctx->base +
-			GSI_EE_n_CNTXT_TYPE_IRQ_MSK_OFFS(ee));
-	gsi_writel((curr & ~mask) | (val & mask), gsi_ctx->base +
-			GSI_EE_n_CNTXT_TYPE_IRQ_MSK_OFFS(ee));
-}
-
-static void __gsi_config_ch_irq(int ee, uint32_t mask, uint32_t val)
-{
-	uint32_t curr;
-
-	curr = gsi_readl(gsi_ctx->base +
-			GSI_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_OFFS(ee));
-	gsi_writel((curr & ~mask) | (val & mask), gsi_ctx->base +
-			GSI_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_OFFS(ee));
-}
-
-static void __gsi_config_evt_irq(int ee, uint32_t mask, uint32_t val)
-{
-	uint32_t curr;
-
-	curr = gsi_readl(gsi_ctx->base +
-			GSI_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_OFFS(ee));
-	gsi_writel((curr & ~mask) | (val & mask), gsi_ctx->base +
-			GSI_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_OFFS(ee));
-}
-
-static void __gsi_config_ieob_irq(int ee, uint32_t mask, uint32_t val)
-{
-	uint32_t curr;
-
-	curr = gsi_readl(gsi_ctx->base +
-			GSI_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_OFFS(ee));
-	gsi_writel((curr & ~mask) | (val & mask), gsi_ctx->base +
-			GSI_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_OFFS(ee));
-	GSIDBG("current IEO_IRQ_MSK: 0x%x, change to: 0x%x\n",
-		curr, ((curr & ~mask) | (val & mask)));
-}
-
-static void __gsi_config_glob_irq(int ee, uint32_t mask, uint32_t val)
-{
-	uint32_t curr;
-
-	curr = gsi_readl(gsi_ctx->base +
-			GSI_EE_n_CNTXT_GLOB_IRQ_EN_OFFS(ee));
-	gsi_writel((curr & ~mask) | (val & mask), gsi_ctx->base +
-			GSI_EE_n_CNTXT_GLOB_IRQ_EN_OFFS(ee));
-}
-
-static void __gsi_config_gen_irq(int ee, uint32_t mask, uint32_t val)
-{
-	uint32_t curr;
-
-	curr = gsi_readl(gsi_ctx->base +
-			GSI_EE_n_CNTXT_GSI_IRQ_EN_OFFS(ee));
-	gsi_writel((curr & ~mask) | (val & mask), gsi_ctx->base +
-			GSI_EE_n_CNTXT_GSI_IRQ_EN_OFFS(ee));
-}
-
-static void gsi_channel_state_change_wait(unsigned long chan_hdl,
-	struct gsi_chan_ctx *ctx,
-	uint32_t tm, enum gsi_ch_cmd_opcode op)
-{
-	int poll_cnt;
-	int gsi_pending_intr;
-	int res;
-	uint32_t type;
-	uint32_t val;
-	int ee = gsi_ctx->per.ee;
-	enum gsi_chan_state curr_state = GSI_CHAN_STATE_NOT_ALLOCATED;
-	int stop_in_proc_retry = 0;
-	int stop_retry = 0;
-
-	/*
-	 * Start polling the GSI channel for
-	 * duration = tm * GSI_CMD_POLL_CNT.
-	 * We need to do polling of gsi state for improving debugability
-	 * of gsi hw state.
-	 */
-
-	for (poll_cnt = 0;
-		poll_cnt < GSI_CMD_POLL_CNT;
-		poll_cnt++) {
-		res = wait_for_completion_timeout(&ctx->compl,
-			msecs_to_jiffies(tm));
-
-		/* Interrupt received, return */
-		if (res != 0)
-			return;
-
-		type = gsi_readl(gsi_ctx->base +
-			GSI_EE_n_CNTXT_TYPE_IRQ_OFFS(gsi_ctx->per.ee));
-
-		gsi_pending_intr = gsi_readl(gsi_ctx->base +
-			GSI_EE_n_CNTXT_SRC_GSI_CH_IRQ_OFFS(ee));
-
-		/* Update the channel state only if interrupt was raised
-		 * on praticular channel and also checking global interrupt
-		 * is raised for channel control.
-		 */
-		if ((type & GSI_EE_n_CNTXT_TYPE_IRQ_CH_CTRL_BMSK) &&
-				((gsi_pending_intr >> chan_hdl) & 1)) {
-			/*
-			 * Check channel state here in case the channel is
-			 * already started but interrupt is not yet received.
-			 */
-			val = gsi_readl(gsi_ctx->base +
-				GSI_EE_n_GSI_CH_k_CNTXT_0_OFFS(chan_hdl,
-					gsi_ctx->per.ee));
-			curr_state = (val &
-				GSI_EE_n_GSI_CH_k_CNTXT_0_CHSTATE_BMSK) >>
-				GSI_EE_n_GSI_CH_k_CNTXT_0_CHSTATE_SHFT;
-		}
-
-		if (op == GSI_CH_START) {
-			if (curr_state == GSI_CHAN_STATE_STARTED) {
-				ctx->state = curr_state;
-				return;
-			}
-		}
-
-		if (op == GSI_CH_STOP) {
-			if (curr_state == GSI_CHAN_STATE_STOPPED)
-				stop_retry++;
-			else if (curr_state == GSI_CHAN_STATE_STOP_IN_PROC)
-				stop_in_proc_retry++;
-		}
-
-		/* if interrupt marked reg after poll count reaching to max
-		 * keep loop to continue reach max stop proc and max stop count.
-		 */
-		if (stop_retry == 1 || stop_in_proc_retry == 1)
-			poll_cnt = 0;
-
-		/* If stop channel retry reached to max count
-		 * clear the pending interrupt, if channel already stopped.
-		 */
-		if (stop_retry == GSI_STOP_CMD_POLL_CNT) {
-			gsi_writel(gsi_pending_intr, gsi_ctx->base +
-				GSI_EE_n_CNTXT_SRC_GSI_CH_IRQ_CLR_OFFS(ee));
-			ctx->state = curr_state;
-			return;
-		}
-
-		/* If channel state stop in progress case no need
-		 * to wait for long time.
-		 */
-		if (stop_in_proc_retry == GSI_STOP_IN_PROC_CMD_POLL_CNT) {
-			ctx->state = curr_state;
-			return;
-		}
-
-		GSIDBG("GSI wait on chan_hld=%lu irqtyp=%u state=%u intr=%u\n",
-			chan_hdl,
-			type,
-			ctx->state,
-			gsi_pending_intr);
-	}
-
-	GSIDBG("invalidating the channel state when timeout happens\n");
-	ctx->state = curr_state;
-}
-
-static void gsi_handle_ch_ctrl(int ee)
-{
-	uint32_t ch;
-	int i;
-	uint32_t val;
-	struct gsi_chan_ctx *ctx;
-
-	ch = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_CNTXT_SRC_GSI_CH_IRQ_OFFS(ee));
-	gsi_writel(ch, gsi_ctx->base +
-		GSI_EE_n_CNTXT_SRC_GSI_CH_IRQ_CLR_OFFS(ee));
-	GSIDBG("ch %x\n", ch);
-	for (i = 0; i < GSI_STTS_REG_BITS; i++) {
-		if ((1 << i) & ch) {
-			if (i >= gsi_ctx->max_ch || i >= GSI_CHAN_MAX) {
-				GSIERR("invalid channel %d\n", i);
-				break;
-			}
-
-			ctx = &gsi_ctx->chan[i];
-			val = gsi_readl(gsi_ctx->base +
-				GSI_EE_n_GSI_CH_k_CNTXT_0_OFFS(i, ee));
-			ctx->state = (val &
-				GSI_EE_n_GSI_CH_k_CNTXT_0_CHSTATE_BMSK) >>
-				GSI_EE_n_GSI_CH_k_CNTXT_0_CHSTATE_SHFT;
-			GSIDBG("ch %u state updated to %u\n", i, ctx->state);
-			complete(&ctx->compl);
-			gsi_ctx->ch_dbg[i].cmd_completed++;
-		}
-	}
-}
-
-static void gsi_handle_ev_ctrl(int ee)
-{
-	uint32_t ch;
-	int i;
-	uint32_t val;
-	struct gsi_evt_ctx *ctx;
-
-	ch = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_CNTXT_SRC_EV_CH_IRQ_OFFS(ee));
-	gsi_writel(ch, gsi_ctx->base +
-		GSI_EE_n_CNTXT_SRC_EV_CH_IRQ_CLR_OFFS(ee));
-	GSIDBG("ev %x\n", ch);
-	for (i = 0; i < GSI_STTS_REG_BITS; i++) {
-		if ((1 << i) & ch) {
-			if (i >= gsi_ctx->max_ev || i >= GSI_EVT_RING_MAX) {
-				GSIERR("invalid event %d\n", i);
-				break;
-			}
-
-			ctx = &gsi_ctx->evtr[i];
-			val = gsi_readl(gsi_ctx->base +
-				GSI_EE_n_EV_CH_k_CNTXT_0_OFFS(i, ee));
-			ctx->state = (val &
-				GSI_EE_n_EV_CH_k_CNTXT_0_CHSTATE_BMSK) >>
-				GSI_EE_n_EV_CH_k_CNTXT_0_CHSTATE_SHFT;
-			GSIDBG("evt %u state updated to %u\n", i, ctx->state);
-			complete(&ctx->compl);
-		}
-	}
-}
-
-static void gsi_handle_glob_err(uint32_t err)
-{
-	struct gsi_log_err *log;
-	struct gsi_chan_ctx *ch;
-	struct gsi_evt_ctx *ev;
-	struct gsi_chan_err_notify chan_notify;
-	struct gsi_evt_err_notify evt_notify;
-	struct gsi_per_notify per_notify;
-	uint32_t val;
-	enum gsi_err_type err_type;
-
-	log = (struct gsi_log_err *)&err;
-	GSIERR("log err_type=%u ee=%u idx=%u\n", log->err_type, log->ee,
-			log->virt_idx);
-	GSIERR("code=%u arg1=%u arg2=%u arg3=%u\n", log->code, log->arg1,
-			log->arg2, log->arg3);
-
-	err_type = log->err_type;
-	/*
-	 * These are errors thrown by hardware. We need
-	 * BUG_ON() to capture the hardware state right
-	 * when it is unexpected.
-	 */
-	switch (err_type) {
-	case GSI_ERR_TYPE_GLOB:
-		per_notify.evt_id = GSI_PER_EVT_GLOB_ERROR;
-		per_notify.user_data = gsi_ctx->per.user_data;
-		per_notify.data.err_desc = err & 0xFFFF;
-		gsi_ctx->per.notify_cb(&per_notify);
-		break;
-	case GSI_ERR_TYPE_CHAN:
-		if (WARN_ON(log->virt_idx >= gsi_ctx->max_ch)) {
-			GSIERR("Unexpected ch %d\n", log->virt_idx);
-			return;
-		}
-
-		ch = &gsi_ctx->chan[log->virt_idx];
-		chan_notify.chan_user_data = ch->props.chan_user_data;
-		chan_notify.err_desc = err & 0xFFFF;
-		if (log->code == GSI_INVALID_TRE_ERR) {
-			if (log->ee != gsi_ctx->per.ee) {
-				GSIERR("unexpected EE in event %d\n", log->ee);
-				GSI_ASSERT();
-			}
-
-			val = gsi_readl(gsi_ctx->base +
-				GSI_EE_n_GSI_CH_k_CNTXT_0_OFFS(log->virt_idx,
-					gsi_ctx->per.ee));
-			ch->state = (val &
-				GSI_EE_n_GSI_CH_k_CNTXT_0_CHSTATE_BMSK) >>
-				GSI_EE_n_GSI_CH_k_CNTXT_0_CHSTATE_SHFT;
-			GSIDBG("ch %u state updated to %u\n", log->virt_idx,
-					ch->state);
-			ch->stats.invalid_tre_error++;
-			if (ch->state == GSI_CHAN_STATE_ERROR) {
-				GSIERR("Unexpected channel state %d\n",
-					ch->state);
-				GSI_ASSERT();
-			}
-			chan_notify.evt_id = GSI_CHAN_INVALID_TRE_ERR;
-		} else if (log->code == GSI_OUT_OF_BUFFERS_ERR) {
-			if (log->ee != gsi_ctx->per.ee) {
-				GSIERR("unexpected EE in event %d\n", log->ee);
-				GSI_ASSERT();
-			}
-			chan_notify.evt_id = GSI_CHAN_OUT_OF_BUFFERS_ERR;
-		} else if (log->code == GSI_OUT_OF_RESOURCES_ERR) {
-			if (log->ee != gsi_ctx->per.ee) {
-				GSIERR("unexpected EE in event %d\n", log->ee);
-				GSI_ASSERT();
-			}
-			chan_notify.evt_id = GSI_CHAN_OUT_OF_RESOURCES_ERR;
-			complete(&ch->compl);
-		} else if (log->code == GSI_UNSUPPORTED_INTER_EE_OP_ERR) {
-			chan_notify.evt_id =
-				GSI_CHAN_UNSUPPORTED_INTER_EE_OP_ERR;
-		} else if (log->code == GSI_NON_ALLOCATED_EVT_ACCESS_ERR) {
-			if (log->ee != gsi_ctx->per.ee) {
-				GSIERR("unexpected EE in event %d\n", log->ee);
-				GSI_ASSERT();
-			}
-			chan_notify.evt_id =
-				GSI_CHAN_NON_ALLOCATED_EVT_ACCESS_ERR;
-		} else if (log->code == GSI_HWO_1_ERR) {
-			if (log->ee != gsi_ctx->per.ee) {
-				GSIERR("unexpected EE in event %d\n", log->ee);
-				GSI_ASSERT();
-			}
-			chan_notify.evt_id = GSI_CHAN_HWO_1_ERR;
-		} else {
-			GSIERR("unexpected event log code %d\n", log->code);
-			GSI_ASSERT();
-		}
-		ch->props.err_cb(&chan_notify);
-		break;
-	case GSI_ERR_TYPE_EVT:
-		if (WARN_ON(log->virt_idx >= gsi_ctx->max_ev)) {
-			GSIERR("Unexpected ev %d\n", log->virt_idx);
-			return;
-		}
-
-		ev = &gsi_ctx->evtr[log->virt_idx];
-		evt_notify.user_data = ev->props.user_data;
-		evt_notify.err_desc = err & 0xFFFF;
-		if (log->code == GSI_OUT_OF_BUFFERS_ERR) {
-			if (log->ee != gsi_ctx->per.ee) {
-				GSIERR("unexpected EE in event %d\n", log->ee);
-				GSI_ASSERT();
-			}
-			evt_notify.evt_id = GSI_EVT_OUT_OF_BUFFERS_ERR;
-		} else if (log->code == GSI_OUT_OF_RESOURCES_ERR) {
-			if (log->ee != gsi_ctx->per.ee) {
-				GSIERR("unexpected EE in event %d\n", log->ee);
-				GSI_ASSERT();
-			}
-			evt_notify.evt_id = GSI_EVT_OUT_OF_RESOURCES_ERR;
-			complete(&ev->compl);
-		} else if (log->code == GSI_UNSUPPORTED_INTER_EE_OP_ERR) {
-			evt_notify.evt_id = GSI_EVT_UNSUPPORTED_INTER_EE_OP_ERR;
-		} else if (log->code == GSI_EVT_RING_EMPTY_ERR) {
-			if (log->ee != gsi_ctx->per.ee) {
-				GSIERR("unexpected EE in event %d\n", log->ee);
-				GSI_ASSERT();
-			}
-			evt_notify.evt_id = GSI_EVT_EVT_RING_EMPTY_ERR;
-		} else {
-			GSIERR("unexpected event log code %d\n", log->code);
-			GSI_ASSERT();
-		}
-		ev->props.err_cb(&evt_notify);
-		break;
-	}
-}
-
-static void gsi_handle_gp_int1(void)
-{
-	complete(&gsi_ctx->gen_ee_cmd_compl);
-}
-
-static void gsi_handle_glob_ee(int ee)
-{
-	uint32_t val;
-	uint32_t err;
-	struct gsi_per_notify notify;
-	uint32_t clr = ~0;
-
-	val = gsi_readl(gsi_ctx->base +
-			GSI_EE_n_CNTXT_GLOB_IRQ_STTS_OFFS(ee));
-
-	notify.user_data = gsi_ctx->per.user_data;
-
-	if (val & GSI_EE_n_CNTXT_GLOB_IRQ_STTS_ERROR_INT_BMSK) {
-		err = gsi_readl(gsi_ctx->base +
-			GSI_EE_n_ERROR_LOG_OFFS(ee));
-		if (gsi_ctx->per.ver >= GSI_VER_1_2)
-			gsi_writel(0, gsi_ctx->base +
-				GSI_EE_n_ERROR_LOG_OFFS(ee));
-		gsi_writel(clr, gsi_ctx->base +
-			GSI_EE_n_ERROR_LOG_CLR_OFFS(ee));
-		gsi_handle_glob_err(err);
-	}
-
-	if (val & GSI_EE_n_CNTXT_GLOB_IRQ_EN_GP_INT1_BMSK)
-		gsi_handle_gp_int1();
-
-	if (val & GSI_EE_n_CNTXT_GLOB_IRQ_EN_GP_INT2_BMSK) {
-		notify.evt_id = GSI_PER_EVT_GLOB_GP2;
-		gsi_ctx->per.notify_cb(&notify);
-	}
-
-	if (val & GSI_EE_n_CNTXT_GLOB_IRQ_EN_GP_INT3_BMSK) {
-		notify.evt_id = GSI_PER_EVT_GLOB_GP3;
-		gsi_ctx->per.notify_cb(&notify);
-	}
-
-	gsi_writel(val, gsi_ctx->base +
-			GSI_EE_n_CNTXT_GLOB_IRQ_CLR_OFFS(ee));
-}
-
-static void gsi_incr_ring_wp(struct gsi_ring_ctx *ctx)
-{
-	ctx->wp_local += ctx->elem_sz;
-	if (ctx->wp_local == ctx->end)
-		ctx->wp_local = ctx->base;
-}
-
-static void gsi_incr_ring_rp(struct gsi_ring_ctx *ctx)
-{
-	ctx->rp_local += ctx->elem_sz;
-	if (ctx->rp_local == ctx->end)
-		ctx->rp_local = ctx->base;
-}
-
-uint16_t gsi_find_idx_from_addr(struct gsi_ring_ctx *ctx, uint64_t addr)
-{
-	WARN_ON(addr < ctx->base || addr >= ctx->end);
-	return (uint32_t)(addr - ctx->base) / ctx->elem_sz;
-}
-
-static uint16_t gsi_get_complete_num(struct gsi_ring_ctx *ctx, uint64_t addr1,
-		uint64_t addr2)
-{
-	uint32_t addr_diff;
-
-	GSIDBG_LOW("gsi base addr 0x%llx end addr 0x%llx\n",
-		ctx->base, ctx->end);
-
-	if (addr1 < ctx->base || addr1 >= ctx->end) {
-		GSIERR("address = 0x%llx not in range\n", addr1);
-		GSI_ASSERT();
-	}
-
-	if (addr2 < ctx->base || addr2 >= ctx->end) {
-		GSIERR("address = 0x%llx not in range\n", addr2);
-		GSI_ASSERT();
-	}
-
-	addr_diff = (uint32_t)(addr2 - addr1);
-	if (addr1 < addr2)
-		return addr_diff / ctx->elem_sz;
-	else
-		return (addr_diff + ctx->len) / ctx->elem_sz;
-}
-
-static void gsi_process_chan(struct gsi_xfer_compl_evt *evt,
-		struct gsi_chan_xfer_notify *notify, bool callback)
-{
-	uint32_t ch_id;
-	struct gsi_chan_ctx *ch_ctx;
-	uint16_t rp_idx;
-	uint64_t rp;
-
-	ch_id = evt->chid;
-	if (WARN_ON(ch_id >= gsi_ctx->max_ch)) {
-		GSIERR("Unexpected ch %d\n", ch_id);
-		return;
-	}
-
-	ch_ctx = &gsi_ctx->chan[ch_id];
-	if (WARN_ON(ch_ctx->props.prot != GSI_CHAN_PROT_GPI &&
-		ch_ctx->props.prot != GSI_CHAN_PROT_GCI))
-		return;
-
-	if (evt->type != GSI_XFER_COMPL_TYPE_GCI) {
-		rp = evt->xfer_ptr;
-
-		if (ch_ctx->ring.rp_local != rp) {
-			ch_ctx->stats.completed +=
-				gsi_get_complete_num(&ch_ctx->ring,
-				ch_ctx->ring.rp_local, rp);
-			ch_ctx->ring.rp_local = rp;
-		}
-
-
-		/* the element at RP is also processed */
-		gsi_incr_ring_rp(&ch_ctx->ring);
-
-		ch_ctx->ring.rp = ch_ctx->ring.rp_local;
-		rp_idx = gsi_find_idx_from_addr(&ch_ctx->ring, rp);
-		notify->veid = GSI_VEID_DEFAULT;
-	} else {
-		rp_idx = evt->cookie;
-		notify->veid = evt->veid;
-	}
-
-	ch_ctx->stats.completed++;
-
-	WARN_ON(!ch_ctx->user_data[rp_idx].valid);
-	notify->xfer_user_data = ch_ctx->user_data[rp_idx].p;
-	ch_ctx->user_data[rp_idx].valid = false;
-
-	notify->chan_user_data = ch_ctx->props.chan_user_data;
-	notify->evt_id = evt->code;
-	notify->bytes_xfered = evt->len;
-
-	if (callback) {
-		if (atomic_read(&ch_ctx->poll_mode)) {
-			GSIERR("Calling client callback in polling mode\n");
-			WARN_ON(1);
-		}
-		ch_ctx->props.xfer_cb(notify);
-	}
-}
-
-static void gsi_process_evt_re(struct gsi_evt_ctx *ctx,
-		struct gsi_chan_xfer_notify *notify, bool callback)
-{
-	struct gsi_xfer_compl_evt *evt;
-
-	evt = (struct gsi_xfer_compl_evt *)(ctx->ring.base_va +
-			ctx->ring.rp_local - ctx->ring.base);
-	gsi_process_chan(evt, notify, callback);
-	gsi_incr_ring_rp(&ctx->ring);
-	/* recycle this element */
-	gsi_incr_ring_wp(&ctx->ring);
-	ctx->stats.completed++;
-}
-
-static void gsi_ring_evt_doorbell(struct gsi_evt_ctx *ctx)
-{
-	uint32_t val;
-
-	ctx->ring.wp = ctx->ring.wp_local;
-	val = (ctx->ring.wp_local &
-			GSI_EE_n_EV_CH_k_DOORBELL_0_WRITE_PTR_LSB_BMSK) <<
-			GSI_EE_n_EV_CH_k_DOORBELL_0_WRITE_PTR_LSB_SHFT;
-	gsi_writel(val, gsi_ctx->base +
-			GSI_EE_n_EV_CH_k_DOORBELL_0_OFFS(ctx->id,
-				gsi_ctx->per.ee));
-}
-
-static void gsi_ring_chan_doorbell(struct gsi_chan_ctx *ctx)
-{
-	uint32_t val;
-
-	/*
-	 * allocate new events for this channel first
-	 * before submitting the new TREs.
-	 * for TO_GSI channels the event ring doorbell is rang as part of
-	 * interrupt handling.
-	 */
-	if (ctx->evtr && ctx->props.dir == GSI_CHAN_DIR_FROM_GSI)
-		gsi_ring_evt_doorbell(ctx->evtr);
-	ctx->ring.wp = ctx->ring.wp_local;
-
-	val = (ctx->ring.wp_local &
-			GSI_EE_n_GSI_CH_k_DOORBELL_0_WRITE_PTR_LSB_BMSK) <<
-			GSI_EE_n_GSI_CH_k_DOORBELL_0_WRITE_PTR_LSB_SHFT;
-	gsi_writel(val, gsi_ctx->base +
-			GSI_EE_n_GSI_CH_k_DOORBELL_0_OFFS(ctx->props.ch_id,
-				gsi_ctx->per.ee));
-}
-
-static void gsi_handle_ieob(int ee)
-{
-	uint32_t ch;
-	int i;
-	uint64_t rp;
-	struct gsi_evt_ctx *ctx;
-	struct gsi_chan_xfer_notify notify;
-	unsigned long flags;
-	unsigned long cntr;
-	uint32_t msk;
-	bool empty;
-
-	ch = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_CNTXT_SRC_IEOB_IRQ_OFFS(ee));
-	msk = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_OFFS(ee));
-	gsi_writel(ch & msk, gsi_ctx->base +
-		GSI_EE_n_CNTXT_SRC_IEOB_IRQ_CLR_OFFS(ee));
-
-	for (i = 0; i < GSI_STTS_REG_BITS; i++) {
-		if ((1 << i) & ch & msk) {
-			if (i >= gsi_ctx->max_ev || i >= GSI_EVT_RING_MAX) {
-				GSIERR("invalid event %d\n", i);
-				break;
-			}
-			ctx = &gsi_ctx->evtr[i];
-
-			/*
-			 * Don't handle MSI interrupts, only handle IEOB
-			 * IRQs
-			 */
-			if (ctx->props.intr == GSI_INTR_MSI)
-				continue;
-
-			if (ctx->props.intf != GSI_EVT_CHTYPE_GPI_EV) {
-				GSIERR("Unexpected irq intf %d\n",
-					ctx->props.intf);
-				GSI_ASSERT();
-			}
-			spin_lock_irqsave(&ctx->ring.slock, flags);
-check_again:
-			cntr = 0;
-			empty = true;
-			rp = gsi_readl(gsi_ctx->base +
-				GSI_EE_n_EV_CH_k_CNTXT_4_OFFS(i, ee));
-			rp |= ctx->ring.rp & 0xFFFFFFFF00000000;
-
-			ctx->ring.rp = rp;
-			while (ctx->ring.rp_local != rp) {
-				++cntr;
-				if (ctx->props.exclusive &&
-					atomic_read(&ctx->chan->poll_mode)) {
-					cntr = 0;
-					break;
-				}
-				gsi_process_evt_re(ctx, &notify, true);
-				empty = false;
-			}
-			if (!empty)
-				gsi_ring_evt_doorbell(ctx);
-			if (cntr != 0)
-				goto check_again;
-			spin_unlock_irqrestore(&ctx->ring.slock, flags);
-		}
-	}
-}
-
-static void gsi_handle_inter_ee_ch_ctrl(int ee)
-{
-	uint32_t ch;
-	int i;
-
-	ch = gsi_readl(gsi_ctx->base +
-		GSI_INTER_EE_n_SRC_GSI_CH_IRQ_OFFS(ee));
-	gsi_writel(ch, gsi_ctx->base +
-		GSI_INTER_EE_n_SRC_GSI_CH_IRQ_CLR_OFFS(ee));
-	for (i = 0; i < GSI_STTS_REG_BITS; i++) {
-		if ((1 << i) & ch) {
-			/* not currently expected */
-			GSIERR("ch %u was inter-EE changed\n", i);
-		}
-	}
-}
-
-static void gsi_handle_inter_ee_ev_ctrl(int ee)
-{
-	uint32_t ch;
-	int i;
-
-	ch = gsi_readl(gsi_ctx->base +
-		GSI_INTER_EE_n_SRC_EV_CH_IRQ_OFFS(ee));
-	gsi_writel(ch, gsi_ctx->base +
-		GSI_INTER_EE_n_SRC_EV_CH_IRQ_CLR_OFFS(ee));
-	for (i = 0; i < GSI_STTS_REG_BITS; i++) {
-		if ((1 << i) & ch) {
-			/* not currently expected */
-			GSIERR("evt %u was inter-EE changed\n", i);
-		}
-	}
-}
-
-static void gsi_handle_general(int ee)
-{
-	uint32_t val;
-	struct gsi_per_notify notify;
-
-	val = gsi_readl(gsi_ctx->base +
-			GSI_EE_n_CNTXT_GSI_IRQ_STTS_OFFS(ee));
-
-	notify.user_data = gsi_ctx->per.user_data;
-
-	if (val & GSI_EE_n_CNTXT_GSI_IRQ_CLR_GSI_MCS_STACK_OVRFLOW_BMSK)
-		notify.evt_id = GSI_PER_EVT_GENERAL_MCS_STACK_OVERFLOW;
-
-	if (val & GSI_EE_n_CNTXT_GSI_IRQ_CLR_GSI_CMD_FIFO_OVRFLOW_BMSK)
-		notify.evt_id = GSI_PER_EVT_GENERAL_CMD_FIFO_OVERFLOW;
-
-	if (val & GSI_EE_n_CNTXT_GSI_IRQ_CLR_GSI_BUS_ERROR_BMSK)
-		notify.evt_id = GSI_PER_EVT_GENERAL_BUS_ERROR;
-
-	if (val & GSI_EE_n_CNTXT_GSI_IRQ_CLR_GSI_BREAK_POINT_BMSK)
-		notify.evt_id = GSI_PER_EVT_GENERAL_BREAK_POINT;
-
-	if (gsi_ctx->per.notify_cb)
-		gsi_ctx->per.notify_cb(&notify);
-
-	gsi_writel(val, gsi_ctx->base +
-			GSI_EE_n_CNTXT_GSI_IRQ_CLR_OFFS(ee));
-}
-
-#define GSI_ISR_MAX_ITER 50
-
-static void gsi_handle_irq(void)
-{
-	uint32_t type;
-	int ee = gsi_ctx->per.ee;
-	unsigned long cnt = 0;
-
-	while (1) {
-		type = gsi_readl(gsi_ctx->base +
-			GSI_EE_n_CNTXT_TYPE_IRQ_OFFS(ee));
-
-		if (!type)
-			break;
-
-		GSIDBG_LOW("type 0x%x\n", type);
-
-		if (type & GSI_EE_n_CNTXT_TYPE_IRQ_CH_CTRL_BMSK)
-			gsi_handle_ch_ctrl(ee);
-
-		if (type & GSI_EE_n_CNTXT_TYPE_IRQ_EV_CTRL_BMSK)
-			gsi_handle_ev_ctrl(ee);
-
-		if (type & GSI_EE_n_CNTXT_TYPE_IRQ_GLOB_EE_BMSK)
-			gsi_handle_glob_ee(ee);
-
-		if (type & GSI_EE_n_CNTXT_TYPE_IRQ_IEOB_BMSK)
-			gsi_handle_ieob(ee);
-
-		if (type & GSI_EE_n_CNTXT_TYPE_IRQ_INTER_EE_CH_CTRL_BMSK)
-			gsi_handle_inter_ee_ch_ctrl(ee);
-
-		if (type & GSI_EE_n_CNTXT_TYPE_IRQ_INTER_EE_EV_CTRL_BMSK)
-			gsi_handle_inter_ee_ev_ctrl(ee);
-
-		if (type & GSI_EE_n_CNTXT_TYPE_IRQ_GENERAL_BMSK)
-			gsi_handle_general(ee);
-
-		if (++cnt > GSI_ISR_MAX_ITER) {
-			/*
-			 * Max number of spurious interrupts from hardware.
-			 * Unexpected hardware state.
-			 */
-			GSIERR("Too many spurious interrupt from GSI HW\n");
-			GSI_ASSERT();
-		}
-
-	}
-}
-
-static irqreturn_t gsi_isr(int irq, void *ctxt)
-{
-	if (gsi_ctx->per.req_clk_cb) {
-		bool granted = false;
-
-		gsi_ctx->per.req_clk_cb(gsi_ctx->per.user_data, &granted);
-		if (granted) {
-			gsi_handle_irq();
-			gsi_ctx->per.rel_clk_cb(gsi_ctx->per.user_data);
-		}
-	} else if (!gsi_ctx->per.clk_status_cb()) {
-		return IRQ_HANDLED;
-	} else {
-		gsi_handle_irq();
-	}
-	return IRQ_HANDLED;
-
-}
-
-static uint32_t gsi_get_max_channels(enum gsi_ver ver)
-{
-	uint32_t reg = 0;
-
-	switch (ver) {
-	case GSI_VER_ERR:
-	case GSI_VER_MAX:
-		GSIERR("GSI version is not supported %d\n", ver);
-		WARN_ON(1);
-		break;
-	case GSI_VER_1_0:
-		reg = gsi_readl(gsi_ctx->base +
-			GSI_V1_0_EE_n_GSI_HW_PARAM_OFFS(gsi_ctx->per.ee));
-		reg = (reg & GSI_V1_0_EE_n_GSI_HW_PARAM_GSI_CH_NUM_BMSK) >>
-			GSI_V1_0_EE_n_GSI_HW_PARAM_GSI_CH_NUM_SHFT;
-		break;
-	case GSI_VER_1_2:
-		reg = gsi_readl(gsi_ctx->base +
-			GSI_V1_2_EE_n_GSI_HW_PARAM_0_OFFS(gsi_ctx->per.ee));
-		reg = (reg & GSI_V1_2_EE_n_GSI_HW_PARAM_0_GSI_CH_NUM_BMSK) >>
-			GSI_V1_2_EE_n_GSI_HW_PARAM_0_GSI_CH_NUM_SHFT;
-		break;
-	case GSI_VER_1_3:
-		reg = gsi_readl(gsi_ctx->base +
-			GSI_V1_3_EE_n_GSI_HW_PARAM_2_OFFS(gsi_ctx->per.ee));
-		reg = (reg &
-			GSI_V1_3_EE_n_GSI_HW_PARAM_2_GSI_NUM_CH_PER_EE_BMSK) >>
-			GSI_V1_3_EE_n_GSI_HW_PARAM_2_GSI_NUM_CH_PER_EE_SHFT;
-		break;
-	case GSI_VER_2_0:
-		reg = gsi_readl(gsi_ctx->base +
-			GSI_V2_0_EE_n_GSI_HW_PARAM_2_OFFS(gsi_ctx->per.ee));
-		reg = (reg &
-			GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_NUM_CH_PER_EE_BMSK) >>
-			GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_NUM_CH_PER_EE_SHFT;
-		break;
-	case GSI_VER_2_2:
-		reg = gsi_readl(gsi_ctx->base +
-			GSI_V2_2_EE_n_GSI_HW_PARAM_2_OFFS(gsi_ctx->per.ee));
-		reg = (reg &
-			GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_NUM_CH_PER_EE_BMSK) >>
-			GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_NUM_CH_PER_EE_SHFT;
-		break;
-	case GSI_VER_2_5:
-		reg = gsi_readl(gsi_ctx->base +
-			GSI_V2_5_EE_n_GSI_HW_PARAM_2_OFFS(gsi_ctx->per.ee));
-		reg = (reg &
-			GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_NUM_CH_PER_EE_BMSK) >>
-			GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_NUM_CH_PER_EE_SHFT;
-		break;
-	case GSI_VER_2_7:
-		reg = gsi_readl(gsi_ctx->base +
-			GSI_V2_7_EE_n_GSI_HW_PARAM_2_OFFS(gsi_ctx->per.ee));
-		reg = (reg &
-			GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_NUM_CH_PER_EE_BMSK) >>
-			GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_NUM_CH_PER_EE_SHFT;
-		break;
-	}
-
-	GSIDBG("max channels %d\n", reg);
-
-	return reg;
-}
-
-static uint32_t gsi_get_max_event_rings(enum gsi_ver ver)
-{
-	uint32_t reg = 0;
-
-	switch (ver) {
-	case GSI_VER_ERR:
-	case GSI_VER_MAX:
-		GSIERR("GSI version is not supported %d\n", ver);
-		WARN_ON(1);
-		break;
-	case GSI_VER_1_0:
-		reg = gsi_readl(gsi_ctx->base +
-			GSI_V1_0_EE_n_GSI_HW_PARAM_OFFS(gsi_ctx->per.ee));
-		reg = (reg & GSI_V1_0_EE_n_GSI_HW_PARAM_GSI_EV_CH_NUM_BMSK) >>
-			GSI_V1_0_EE_n_GSI_HW_PARAM_GSI_EV_CH_NUM_SHFT;
-		break;
-	case GSI_VER_1_2:
-		reg = gsi_readl(gsi_ctx->base +
-			GSI_V1_2_EE_n_GSI_HW_PARAM_0_OFFS(gsi_ctx->per.ee));
-		reg = (reg & GSI_V1_2_EE_n_GSI_HW_PARAM_0_GSI_EV_CH_NUM_BMSK) >>
-			GSI_V1_2_EE_n_GSI_HW_PARAM_0_GSI_EV_CH_NUM_SHFT;
-		break;
-	case GSI_VER_1_3:
-		reg = gsi_readl(gsi_ctx->base +
-			GSI_V1_3_EE_n_GSI_HW_PARAM_2_OFFS(gsi_ctx->per.ee));
-		reg = (reg &
-			GSI_V1_3_EE_n_GSI_HW_PARAM_2_GSI_NUM_EV_PER_EE_BMSK) >>
-			GSI_V1_3_EE_n_GSI_HW_PARAM_2_GSI_NUM_EV_PER_EE_SHFT;
-		break;
-	case GSI_VER_2_0:
-		reg = gsi_readl(gsi_ctx->base +
-			GSI_V2_0_EE_n_GSI_HW_PARAM_2_OFFS(gsi_ctx->per.ee));
-		reg = (reg &
-			GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_NUM_EV_PER_EE_BMSK) >>
-			GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_NUM_EV_PER_EE_SHFT;
-		break;
-	case GSI_VER_2_2:
-		reg = gsi_readl(gsi_ctx->base +
-			GSI_V2_2_EE_n_GSI_HW_PARAM_2_OFFS(gsi_ctx->per.ee));
-		reg = (reg &
-			GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_NUM_EV_PER_EE_BMSK) >>
-			GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_NUM_EV_PER_EE_SHFT;
-		break;
-	case GSI_VER_2_5:
-		reg = gsi_readl(gsi_ctx->base +
-			GSI_V2_5_EE_n_GSI_HW_PARAM_2_OFFS(gsi_ctx->per.ee));
-		reg = (reg &
-			GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_NUM_EV_PER_EE_BMSK) >>
-			GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_NUM_EV_PER_EE_SHFT;
-		break;
-	case GSI_VER_2_7:
-		reg = gsi_readl(gsi_ctx->base +
-			GSI_V2_7_EE_n_GSI_HW_PARAM_2_OFFS(gsi_ctx->per.ee));
-		reg = (reg &
-			GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_NUM_EV_PER_EE_BMSK) >>
-			GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_NUM_EV_PER_EE_SHFT;
-		break;
-	}
-
-	GSIDBG("max event rings %d\n", reg);
-
-	return reg;
-}
-int gsi_complete_clk_grant(unsigned long dev_hdl)
-{
-	unsigned long flags;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (!gsi_ctx->per_registered) {
-		GSIERR("no client registered\n");
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (dev_hdl != (uintptr_t)gsi_ctx) {
-		GSIERR("bad params dev_hdl=0x%lx gsi_ctx=0x%pK\n", dev_hdl,
-				gsi_ctx);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	spin_lock_irqsave(&gsi_ctx->slock, flags);
-	gsi_handle_irq();
-	gsi_ctx->per.rel_clk_cb(gsi_ctx->per.user_data);
-	spin_unlock_irqrestore(&gsi_ctx->slock, flags);
-
-	return GSI_STATUS_SUCCESS;
-}
-EXPORT_SYMBOL(gsi_complete_clk_grant);
-
-int gsi_map_base(phys_addr_t gsi_base_addr, u32 gsi_size)
-{
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	gsi_ctx->base = devm_ioremap_nocache(
-		gsi_ctx->dev, gsi_base_addr, gsi_size);
-
-	if (!gsi_ctx->base) {
-		GSIERR("failed to map access to GSI HW\n");
-		return -GSI_STATUS_RES_ALLOC_FAILURE;
-	}
-
-	GSIDBG("GSI base(%pa) mapped to (%pK) with len (0x%x)\n",
-		&gsi_base_addr,
-		gsi_ctx->base,
-		gsi_size);
-
-	return 0;
-}
-EXPORT_SYMBOL(gsi_map_base);
-
-int gsi_unmap_base(void)
-{
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (!gsi_ctx->base) {
-		GSIERR("access to GSI HW has not been mapped\n");
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	devm_iounmap(gsi_ctx->dev, gsi_ctx->base);
-
-	gsi_ctx->base = NULL;
-
-	return 0;
-}
-EXPORT_SYMBOL(gsi_unmap_base);
-
-int gsi_register_device(struct gsi_per_props *props, unsigned long *dev_hdl)
-{
-	int res;
-	uint32_t val;
-	int needed_reg_ver;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (!props || !dev_hdl) {
-		GSIERR("bad params props=%pK dev_hdl=%pK\n", props, dev_hdl);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (props->ver <= GSI_VER_ERR || props->ver >= GSI_VER_MAX) {
-		GSIERR("bad params gsi_ver=%d\n", props->ver);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (!props->notify_cb) {
-		GSIERR("notify callback must be provided\n");
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (props->req_clk_cb && !props->rel_clk_cb) {
-		GSIERR("rel callback  must be provided\n");
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (gsi_ctx->per_registered) {
-		GSIERR("per already registered\n");
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	switch (props->ver) {
-	case GSI_VER_1_0:
-	case GSI_VER_1_2:
-	case GSI_VER_1_3:
-	case GSI_VER_2_0:
-	case GSI_VER_2_2:
-		needed_reg_ver = GSI_REGISTER_VER_1;
-		break;
-	case GSI_VER_2_5:
-	case GSI_VER_2_7:
-		needed_reg_ver = GSI_REGISTER_VER_2;
-		break;
-	case GSI_VER_ERR:
-	case GSI_VER_MAX:
-	default:
-		GSIERR("GSI version is not supported %d\n", props->ver);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (needed_reg_ver != GSI_REGISTER_VER_CURRENT) {
-		GSIERR("Invalid register version. current=%d, needed=%d\n",
-			GSI_REGISTER_VER_CURRENT, needed_reg_ver);
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-	GSIDBG("gsi ver %d register ver %d needed register ver %d\n",
-		props->ver, GSI_REGISTER_VER_CURRENT, needed_reg_ver);
-
-	spin_lock_init(&gsi_ctx->slock);
-	if (props->intr == GSI_INTR_IRQ) {
-		if (!props->irq) {
-			GSIERR("bad irq specified %u\n", props->irq);
-			return -GSI_STATUS_INVALID_PARAMS;
-		}
-		/*
-		 * On a real UE, there are two separate interrupt
-		 * vectors that get directed toward the GSI/IPA
-		 * drivers.  They are handled by gsi_isr() and
-		 * (ipa_isr() or ipa3_isr()) respectively.  In the
-		 * emulation environment, this is not the case;
-		 * instead, interrupt vectors are routed to the
-		 * emualation hardware's interrupt controller, which
-		 * in turn, forwards a single interrupt to the GSI/IPA
-		 * driver.  When the new interrupt vector is received,
-		 * the driver needs to probe the interrupt
-		 * controller's registers so see if one, the other, or
-		 * both interrupts have occurred.  Given the above, we
-		 * now need to handle both situations, namely: the
-		 * emulator's and the real UE.
-		 */
-		if (running_emulation) {
-			/*
-			 * New scheme involving the emulator's
-			 * interrupt controller.
-			 */
-			res = devm_request_threaded_irq(
-				gsi_ctx->dev,
-				props->irq,
-				/* top half handler to follow */
-				emulator_hard_irq_isr,
-				/* threaded bottom half handler to follow */
-				emulator_soft_irq_isr,
-				IRQF_SHARED,
-				"emulator_intcntrlr",
-				gsi_ctx);
-		} else {
-			/*
-			 * Traditional scheme used on the real UE.
-			 */
-			res = devm_request_irq(gsi_ctx->dev, props->irq,
-				gsi_isr,
-				props->req_clk_cb ? IRQF_TRIGGER_RISING :
-					IRQF_TRIGGER_HIGH,
-				"gsi",
-				gsi_ctx);
-		}
-		if (res) {
-			GSIERR(
-			 "failed to register isr for %u\n",
-			 props->irq);
-			return -GSI_STATUS_ERROR;
-		}
-		GSIDBG(
-			"succeeded to register isr for %u\n",
-			props->irq);
-
-		res = enable_irq_wake(props->irq);
-		if (res)
-			GSIERR("failed to enable wake irq %u\n", props->irq);
-		else
-			GSIERR("GSI irq is wake enabled %u\n", props->irq);
-
-	} else {
-		GSIERR("do not support interrupt type %u\n", props->intr);
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	/*
-	 * If base not previously mapped via gsi_map_base(), map it
-	 * now...
-	 */
-	if (!gsi_ctx->base) {
-		res = gsi_map_base(props->phys_addr, props->size);
-		if (res)
-			return res;
-	}
-
-	if (running_emulation) {
-		GSIDBG("GSI SW ver register value 0x%x\n",
-		       gsi_readl(gsi_ctx->base +
-		       GSI_EE_n_GSI_SW_VERSION_OFFS(0)));
-		gsi_ctx->intcntrlr_mem_size =
-		    props->emulator_intcntrlr_size;
-		gsi_ctx->intcntrlr_base =
-		    devm_ioremap_nocache(
-			gsi_ctx->dev,
-			props->emulator_intcntrlr_addr,
-			props->emulator_intcntrlr_size);
-		if (!gsi_ctx->intcntrlr_base) {
-			GSIERR(
-			  "failed to remap emulator's interrupt controller HW\n");
-			gsi_unmap_base();
-			devm_free_irq(gsi_ctx->dev, props->irq, gsi_ctx);
-			return -GSI_STATUS_RES_ALLOC_FAILURE;
-		}
-
-		GSIDBG(
-		    "Emulator's interrupt controller base(%pa) mapped to (%pK) with len (0x%lx)\n",
-		    &(props->emulator_intcntrlr_addr),
-		    gsi_ctx->intcntrlr_base,
-		    props->emulator_intcntrlr_size);
-
-		gsi_ctx->intcntrlr_gsi_isr = gsi_isr;
-		gsi_ctx->intcntrlr_client_isr =
-		    props->emulator_intcntrlr_client_isr;
-	}
-
-	gsi_ctx->per = *props;
-	gsi_ctx->per_registered = true;
-	mutex_init(&gsi_ctx->mlock);
-	atomic_set(&gsi_ctx->num_chan, 0);
-	atomic_set(&gsi_ctx->num_evt_ring, 0);
-	gsi_ctx->max_ch = gsi_get_max_channels(gsi_ctx->per.ver);
-	if (gsi_ctx->max_ch == 0) {
-		gsi_unmap_base();
-		if (running_emulation)
-			devm_iounmap(gsi_ctx->dev, gsi_ctx->intcntrlr_base);
-		gsi_ctx->base = gsi_ctx->intcntrlr_base = NULL;
-		devm_free_irq(gsi_ctx->dev, props->irq, gsi_ctx);
-		GSIERR("failed to get max channels\n");
-		return -GSI_STATUS_ERROR;
-	}
-	gsi_ctx->max_ev = gsi_get_max_event_rings(gsi_ctx->per.ver);
-	if (gsi_ctx->max_ev == 0) {
-		gsi_unmap_base();
-		if (running_emulation)
-			devm_iounmap(gsi_ctx->dev, gsi_ctx->intcntrlr_base);
-		gsi_ctx->base = gsi_ctx->intcntrlr_base = NULL;
-		devm_free_irq(gsi_ctx->dev, props->irq, gsi_ctx);
-		GSIERR("failed to get max event rings\n");
-		return -GSI_STATUS_ERROR;
-	}
-
-	if (gsi_ctx->max_ev > GSI_EVT_RING_MAX) {
-		GSIERR("max event rings are beyond absolute maximum\n");
-		return -GSI_STATUS_ERROR;
-	}
-
-	if (props->mhi_er_id_limits_valid &&
-	    props->mhi_er_id_limits[0] > (gsi_ctx->max_ev - 1)) {
-		gsi_unmap_base();
-		if (running_emulation)
-			devm_iounmap(gsi_ctx->dev, gsi_ctx->intcntrlr_base);
-		gsi_ctx->base = gsi_ctx->intcntrlr_base = NULL;
-		devm_free_irq(gsi_ctx->dev, props->irq, gsi_ctx);
-		GSIERR("MHI event ring start id %u is beyond max %u\n",
-			props->mhi_er_id_limits[0], gsi_ctx->max_ev);
-		return -GSI_STATUS_ERROR;
-	}
-
-	gsi_ctx->evt_bmap = ~((1 << gsi_ctx->max_ev) - 1);
-
-	/* exclude reserved mhi events */
-	if (props->mhi_er_id_limits_valid)
-		gsi_ctx->evt_bmap |=
-			((1 << (props->mhi_er_id_limits[1] + 1)) - 1) ^
-			((1 << (props->mhi_er_id_limits[0])) - 1);
-
-	/*
-	 * enable all interrupts but GSI_BREAK_POINT.
-	 * Inter EE commands / interrupt are no supported.
-	 */
-	__gsi_config_type_irq(props->ee, ~0, ~0);
-	__gsi_config_ch_irq(props->ee, ~0, ~0);
-	__gsi_config_evt_irq(props->ee, ~0, ~0);
-	__gsi_config_ieob_irq(props->ee, ~0, ~0);
-	__gsi_config_glob_irq(props->ee, ~0, ~0);
-	__gsi_config_gen_irq(props->ee, ~0,
-		~GSI_EE_n_CNTXT_GSI_IRQ_CLR_GSI_BREAK_POINT_BMSK);
-
-	gsi_writel(props->intr, gsi_ctx->base +
-			GSI_EE_n_CNTXT_INTSET_OFFS(gsi_ctx->per.ee));
-	/* set GSI_TOP_EE_n_CNTXT_MSI_BASE_LSB/MSB to 0 */
-	if ((gsi_ctx->per.ver >= GSI_VER_2_0) &&
-		(props->intr != GSI_INTR_MSI)) {
-		gsi_writel(0, gsi_ctx->base +
-			GSI_EE_n_CNTXT_MSI_BASE_LSB(gsi_ctx->per.ee));
-		gsi_writel(0, gsi_ctx->base +
-			GSI_EE_n_CNTXT_MSI_BASE_MSB(gsi_ctx->per.ee));
-	}
-
-	val = gsi_readl(gsi_ctx->base +
-			GSI_EE_n_GSI_STATUS_OFFS(gsi_ctx->per.ee));
-	if (val & GSI_EE_n_GSI_STATUS_ENABLED_BMSK)
-		gsi_ctx->enabled = true;
-	else
-		GSIERR("Manager EE has not enabled GSI, GSI un-usable\n");
-
-	if (gsi_ctx->per.ver >= GSI_VER_1_2)
-		gsi_writel(0, gsi_ctx->base +
-			GSI_EE_n_ERROR_LOG_OFFS(gsi_ctx->per.ee));
-
-	if (running_emulation) {
-		/*
-		 * Set up the emulator's interrupt controller...
-		 */
-		res = setup_emulator_cntrlr(
-		    gsi_ctx->intcntrlr_base, gsi_ctx->intcntrlr_mem_size);
-		if (res != 0) {
-			gsi_unmap_base();
-			devm_iounmap(gsi_ctx->dev, gsi_ctx->intcntrlr_base);
-			gsi_ctx->base = gsi_ctx->intcntrlr_base = NULL;
-			devm_free_irq(gsi_ctx->dev, props->irq, gsi_ctx);
-			GSIERR("setup_emulator_cntrlr() failed\n");
-			return res;
-		}
-	}
-
-	*dev_hdl = (uintptr_t)gsi_ctx;
-
-	return GSI_STATUS_SUCCESS;
-}
-EXPORT_SYMBOL(gsi_register_device);
-
-int gsi_write_device_scratch(unsigned long dev_hdl,
-		struct gsi_device_scratch *val)
-{
-	unsigned int max_usb_pkt_size = 0;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (!gsi_ctx->per_registered) {
-		GSIERR("no client registered\n");
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (dev_hdl != (uintptr_t)gsi_ctx) {
-		GSIERR("bad params dev_hdl=0x%lx gsi_ctx=0x%pK\n", dev_hdl,
-				gsi_ctx);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (val->max_usb_pkt_size_valid &&
-			val->max_usb_pkt_size != 1024 &&
-			val->max_usb_pkt_size != 512 &&
-			val->max_usb_pkt_size != 64) {
-		GSIERR("bad USB max pkt size dev_hdl=0x%lx sz=%u\n", dev_hdl,
-				val->max_usb_pkt_size);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	mutex_lock(&gsi_ctx->mlock);
-	if (val->mhi_base_chan_idx_valid)
-		gsi_ctx->scratch.word0.s.mhi_base_chan_idx =
-			val->mhi_base_chan_idx;
-
-	if (val->max_usb_pkt_size_valid) {
-		max_usb_pkt_size = 2;
-		if (val->max_usb_pkt_size > 64)
-			max_usb_pkt_size =
-				(val->max_usb_pkt_size == 1024) ? 1 : 0;
-		gsi_ctx->scratch.word0.s.max_usb_pkt_size = max_usb_pkt_size;
-	}
-
-	gsi_writel(gsi_ctx->scratch.word0.val,
-			gsi_ctx->base +
-			GSI_EE_n_CNTXT_SCRATCH_0_OFFS(gsi_ctx->per.ee));
-	mutex_unlock(&gsi_ctx->mlock);
-
-	return GSI_STATUS_SUCCESS;
-}
-EXPORT_SYMBOL(gsi_write_device_scratch);
-
-int gsi_deregister_device(unsigned long dev_hdl, bool force)
-{
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (!gsi_ctx->per_registered) {
-		GSIERR("no client registered\n");
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (dev_hdl != (uintptr_t)gsi_ctx) {
-		GSIERR("bad params dev_hdl=0x%lx gsi_ctx=0x%pK\n", dev_hdl,
-				gsi_ctx);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (!force && atomic_read(&gsi_ctx->num_chan)) {
-		GSIERR("cannot deregister %u channels are still connected\n",
-				atomic_read(&gsi_ctx->num_chan));
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	if (!force && atomic_read(&gsi_ctx->num_evt_ring)) {
-		GSIERR("cannot deregister %u events are still connected\n",
-				atomic_read(&gsi_ctx->num_evt_ring));
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	/* disable all interrupts */
-	__gsi_config_type_irq(gsi_ctx->per.ee, ~0, 0);
-	__gsi_config_ch_irq(gsi_ctx->per.ee, ~0, 0);
-	__gsi_config_evt_irq(gsi_ctx->per.ee, ~0, 0);
-	__gsi_config_ieob_irq(gsi_ctx->per.ee, ~0, 0);
-	__gsi_config_glob_irq(gsi_ctx->per.ee, ~0, 0);
-	__gsi_config_gen_irq(gsi_ctx->per.ee, ~0, 0);
-
-	devm_free_irq(gsi_ctx->dev, gsi_ctx->per.irq, gsi_ctx);
-	gsi_unmap_base();
-	memset(gsi_ctx, 0, sizeof(*gsi_ctx));
-
-	return GSI_STATUS_SUCCESS;
-}
-EXPORT_SYMBOL(gsi_deregister_device);
-
-static void gsi_program_evt_ring_ctx(struct gsi_evt_ring_props *props,
-		uint8_t evt_id, unsigned int ee)
-{
-	uint32_t val;
-
-	GSIDBG("intf=%u intr=%u re=%u\n", props->intf, props->intr,
-			props->re_size);
-
-	val = (((props->intf << GSI_EE_n_EV_CH_k_CNTXT_0_CHTYPE_SHFT) &
-			GSI_EE_n_EV_CH_k_CNTXT_0_CHTYPE_BMSK) |
-		((props->intr << GSI_EE_n_EV_CH_k_CNTXT_0_INTYPE_SHFT) &
-			GSI_EE_n_EV_CH_k_CNTXT_0_INTYPE_BMSK) |
-		((props->re_size << GSI_EE_n_EV_CH_k_CNTXT_0_ELEMENT_SIZE_SHFT)
-			& GSI_EE_n_EV_CH_k_CNTXT_0_ELEMENT_SIZE_BMSK));
-
-	gsi_writel(val, gsi_ctx->base +
-			GSI_EE_n_EV_CH_k_CNTXT_0_OFFS(evt_id, ee));
-
-	val = (props->ring_len & GSI_EE_n_EV_CH_k_CNTXT_1_R_LENGTH_BMSK) <<
-		GSI_EE_n_EV_CH_k_CNTXT_1_R_LENGTH_SHFT;
-	gsi_writel(val, gsi_ctx->base +
-			GSI_EE_n_EV_CH_k_CNTXT_1_OFFS(evt_id, ee));
-
-	val = (props->ring_base_addr &
-			GSI_EE_n_EV_CH_k_CNTXT_2_R_BASE_ADDR_LSBS_BMSK) <<
-		GSI_EE_n_EV_CH_k_CNTXT_2_R_BASE_ADDR_LSBS_SHFT;
-	gsi_writel(val, gsi_ctx->base +
-			GSI_EE_n_EV_CH_k_CNTXT_2_OFFS(evt_id, ee));
-
-	val = ((props->ring_base_addr >> 32) &
-		GSI_EE_n_EV_CH_k_CNTXT_3_R_BASE_ADDR_MSBS_BMSK) <<
-		GSI_EE_n_EV_CH_k_CNTXT_3_R_BASE_ADDR_MSBS_SHFT;
-	gsi_writel(val, gsi_ctx->base +
-			GSI_EE_n_EV_CH_k_CNTXT_3_OFFS(evt_id, ee));
-
-	val = (((props->int_modt << GSI_EE_n_EV_CH_k_CNTXT_8_INT_MODT_SHFT) &
-		GSI_EE_n_EV_CH_k_CNTXT_8_INT_MODT_BMSK) |
-		((props->int_modc << GSI_EE_n_EV_CH_k_CNTXT_8_INT_MODC_SHFT) &
-		 GSI_EE_n_EV_CH_k_CNTXT_8_INT_MODC_BMSK));
-	gsi_writel(val, gsi_ctx->base +
-			GSI_EE_n_EV_CH_k_CNTXT_8_OFFS(evt_id, ee));
-
-	val = (props->intvec & GSI_EE_n_EV_CH_k_CNTXT_9_INTVEC_BMSK) <<
-		GSI_EE_n_EV_CH_k_CNTXT_9_INTVEC_SHFT;
-	gsi_writel(val, gsi_ctx->base +
-			GSI_EE_n_EV_CH_k_CNTXT_9_OFFS(evt_id, ee));
-
-	val = (props->msi_addr & GSI_EE_n_EV_CH_k_CNTXT_10_MSI_ADDR_LSB_BMSK) <<
-		GSI_EE_n_EV_CH_k_CNTXT_10_MSI_ADDR_LSB_SHFT;
-	gsi_writel(val, gsi_ctx->base +
-			GSI_EE_n_EV_CH_k_CNTXT_10_OFFS(evt_id, ee));
-
-	val = ((props->msi_addr >> 32) &
-		GSI_EE_n_EV_CH_k_CNTXT_11_MSI_ADDR_MSB_BMSK) <<
-		GSI_EE_n_EV_CH_k_CNTXT_11_MSI_ADDR_MSB_SHFT;
-	gsi_writel(val, gsi_ctx->base +
-			GSI_EE_n_EV_CH_k_CNTXT_11_OFFS(evt_id, ee));
-
-	val = (props->rp_update_addr &
-			GSI_EE_n_EV_CH_k_CNTXT_12_RP_UPDATE_ADDR_LSB_BMSK) <<
-		GSI_EE_n_EV_CH_k_CNTXT_12_RP_UPDATE_ADDR_LSB_SHFT;
-	gsi_writel(val, gsi_ctx->base +
-			GSI_EE_n_EV_CH_k_CNTXT_12_OFFS(evt_id, ee));
-
-	val = ((props->rp_update_addr >> 32) &
-		GSI_EE_n_EV_CH_k_CNTXT_13_RP_UPDATE_ADDR_MSB_BMSK) <<
-		GSI_EE_n_EV_CH_k_CNTXT_13_RP_UPDATE_ADDR_MSB_SHFT;
-	gsi_writel(val, gsi_ctx->base +
-			GSI_EE_n_EV_CH_k_CNTXT_13_OFFS(evt_id, ee));
-}
-
-static void gsi_init_evt_ring(struct gsi_evt_ring_props *props,
-		struct gsi_ring_ctx *ctx)
-{
-	ctx->base_va = (uintptr_t)props->ring_base_vaddr;
-	ctx->base = props->ring_base_addr;
-	ctx->wp = ctx->base;
-	ctx->rp = ctx->base;
-	ctx->wp_local = ctx->base;
-	ctx->rp_local = ctx->base;
-	ctx->len = props->ring_len;
-	ctx->elem_sz = props->re_size;
-	ctx->max_num_elem = ctx->len / ctx->elem_sz - 1;
-	ctx->end = ctx->base + (ctx->max_num_elem + 1) * ctx->elem_sz;
-}
-
-static void gsi_prime_evt_ring(struct gsi_evt_ctx *ctx)
-{
-	unsigned long flags;
-	uint32_t val;
-
-	spin_lock_irqsave(&ctx->ring.slock, flags);
-	memset((void *)ctx->ring.base_va, 0, ctx->ring.len);
-	ctx->ring.wp_local = ctx->ring.base +
-		ctx->ring.max_num_elem * ctx->ring.elem_sz;
-
-	/* write order MUST be MSB followed by LSB */
-	val = ((ctx->ring.wp_local >> 32) &
-		GSI_EE_n_EV_CH_k_DOORBELL_1_WRITE_PTR_MSB_BMSK) <<
-		GSI_EE_n_EV_CH_k_DOORBELL_1_WRITE_PTR_MSB_SHFT;
-	gsi_writel(val, gsi_ctx->base +
-		GSI_EE_n_EV_CH_k_DOORBELL_1_OFFS(ctx->id,
-		gsi_ctx->per.ee));
-
-	gsi_ring_evt_doorbell(ctx);
-	spin_unlock_irqrestore(&ctx->ring.slock, flags);
-}
-
-static void gsi_prime_evt_ring_wdi(struct gsi_evt_ctx *ctx)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&ctx->ring.slock, flags);
-	if (ctx->ring.base_va)
-		memset((void *)ctx->ring.base_va, 0, ctx->ring.len);
-	ctx->ring.wp_local = ctx->ring.base +
-		((ctx->ring.max_num_elem + 2) * ctx->ring.elem_sz);
-	gsi_ring_evt_doorbell(ctx);
-	spin_unlock_irqrestore(&ctx->ring.slock, flags);
-}
-
-static int gsi_validate_evt_ring_props(struct gsi_evt_ring_props *props)
-{
-	uint64_t ra;
-
-	if ((props->re_size == GSI_EVT_RING_RE_SIZE_4B &&
-				props->ring_len % 4) ||
-			(props->re_size == GSI_EVT_RING_RE_SIZE_8B &&
-				 props->ring_len % 8) ||
-			(props->re_size == GSI_EVT_RING_RE_SIZE_16B &&
-				 props->ring_len % 16)) {
-		GSIERR("bad params ring_len %u not a multiple of RE size %u\n",
-				props->ring_len, props->re_size);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	ra = props->ring_base_addr;
-	do_div(ra, roundup_pow_of_two(props->ring_len));
-
-	if (props->ring_base_addr != ra * roundup_pow_of_two(props->ring_len)) {
-		GSIERR("bad params ring base not aligned 0x%llx align 0x%lx\n",
-				props->ring_base_addr,
-				roundup_pow_of_two(props->ring_len));
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (props->intf == GSI_EVT_CHTYPE_GPI_EV &&
-			!props->ring_base_vaddr) {
-		GSIERR("protocol %u requires ring base VA\n", props->intf);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (props->intf == GSI_EVT_CHTYPE_MHI_EV &&
-			(!props->evchid_valid ||
-			props->evchid > gsi_ctx->per.mhi_er_id_limits[1] ||
-			props->evchid < gsi_ctx->per.mhi_er_id_limits[0])) {
-		GSIERR("MHI requires evchid valid=%d val=%u\n",
-				props->evchid_valid, props->evchid);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (props->intf != GSI_EVT_CHTYPE_MHI_EV &&
-			props->evchid_valid) {
-		GSIERR("protocol %u cannot specify evchid\n", props->intf);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (!props->err_cb) {
-		GSIERR("err callback must be provided\n");
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	return GSI_STATUS_SUCCESS;
-}
-
-/**
- * gsi_cleanup_xfer_user_data: cleanup the user data array using callback passed
- *	by IPA driver. Need to do this in GSI since only GSI knows which TRE
- *	are being used or not. However, IPA is the one that does cleaning,
- *	therefore we pass a callback from IPA and call it using params from GSI
- *
- * @chan_hdl: hdl of the gsi channel user data array to be cleaned
- * @cleanup_cb: callback used to clean the user data array. takes 2 inputs
- *	@chan_user_data: ipa_sys_context of the gsi_channel
- *	@xfer_uder_data: user data array element (rx_pkt wrapper)
- *
- * Returns: 0 on success, negative on failure
- */
-static int gsi_cleanup_xfer_user_data(unsigned long chan_hdl,
-	void (*cleanup_cb)(void *chan_user_data, void *xfer_user_data))
-{
-	struct gsi_chan_ctx *ctx;
-	uint64_t i;
-	uint16_t rp_idx;
-
-	ctx = &gsi_ctx->chan[chan_hdl];
-	if (ctx->state != GSI_CHAN_STATE_ALLOCATED) {
-		GSIERR("bad state %d\n", ctx->state);
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	/* for coalescing, traverse the whole array */
-	if (ctx->props.prot == GSI_CHAN_PROT_GCI) {
-		size_t user_data_size =
-			ctx->ring.max_num_elem + 1 + GSI_VEID_MAX;
-		for (i = 0; i < user_data_size; i++) {
-			if (ctx->user_data[i].valid)
-				cleanup_cb(ctx->props.chan_user_data,
-					ctx->user_data[i].p);
-		}
-	} else {
-		/* for non-coalescing, clean between RP and WP */
-		while (ctx->ring.rp_local != ctx->ring.wp_local) {
-			rp_idx = gsi_find_idx_from_addr(&ctx->ring,
-				ctx->ring.rp_local);
-			WARN_ON(!ctx->user_data[rp_idx].valid);
-			cleanup_cb(ctx->props.chan_user_data,
-				ctx->user_data[rp_idx].p);
-			gsi_incr_ring_rp(&ctx->ring);
-		}
-	}
-	return 0;
-}
-
-int gsi_alloc_evt_ring(struct gsi_evt_ring_props *props, unsigned long dev_hdl,
-		unsigned long *evt_ring_hdl)
-{
-	unsigned long evt_id;
-	enum gsi_evt_ch_cmd_opcode op = GSI_EVT_ALLOCATE;
-	uint32_t val;
-	struct gsi_evt_ctx *ctx;
-	int res;
-	int ee;
-	unsigned long flags;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (!props || !evt_ring_hdl || dev_hdl != (uintptr_t)gsi_ctx) {
-		GSIERR("bad params props=%pK dev_hdl=0x%lx evt_ring_hdl=%pK\n",
-				props, dev_hdl, evt_ring_hdl);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (gsi_validate_evt_ring_props(props)) {
-		GSIERR("invalid params\n");
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (!props->evchid_valid) {
-		mutex_lock(&gsi_ctx->mlock);
-		evt_id = find_first_zero_bit(&gsi_ctx->evt_bmap,
-				sizeof(unsigned long) * BITS_PER_BYTE);
-		if (evt_id == sizeof(unsigned long) * BITS_PER_BYTE) {
-			GSIERR("failed to alloc event ID\n");
-			mutex_unlock(&gsi_ctx->mlock);
-			return -GSI_STATUS_RES_ALLOC_FAILURE;
-		}
-		set_bit(evt_id, &gsi_ctx->evt_bmap);
-		mutex_unlock(&gsi_ctx->mlock);
-	} else {
-		evt_id = props->evchid;
-	}
-	GSIDBG("Using %lu as virt evt id\n", evt_id);
-
-	ctx = &gsi_ctx->evtr[evt_id];
-	memset(ctx, 0, sizeof(*ctx));
-	mutex_init(&ctx->mlock);
-	init_completion(&ctx->compl);
-	atomic_set(&ctx->chan_ref_cnt, 0);
-	ctx->props = *props;
-
-	mutex_lock(&gsi_ctx->mlock);
-	val = (((evt_id << GSI_EE_n_EV_CH_CMD_CHID_SHFT) &
-			GSI_EE_n_EV_CH_CMD_CHID_BMSK) |
-		((op << GSI_EE_n_EV_CH_CMD_OPCODE_SHFT) &
-			GSI_EE_n_EV_CH_CMD_OPCODE_BMSK));
-	ee = gsi_ctx->per.ee;
-	gsi_writel(val, gsi_ctx->base +
-			GSI_EE_n_EV_CH_CMD_OFFS(ee));
-	res = wait_for_completion_timeout(&ctx->compl, GSI_CMD_TIMEOUT);
-	if (res == 0) {
-		GSIERR("evt_id=%lu timed out\n", evt_id);
-		if (!props->evchid_valid)
-			clear_bit(evt_id, &gsi_ctx->evt_bmap);
-		mutex_unlock(&gsi_ctx->mlock);
-		return -GSI_STATUS_TIMED_OUT;
-	}
-
-	if (ctx->state != GSI_EVT_RING_STATE_ALLOCATED) {
-		GSIERR("evt_id=%lu allocation failed state=%u\n",
-				evt_id, ctx->state);
-		if (!props->evchid_valid)
-			clear_bit(evt_id, &gsi_ctx->evt_bmap);
-		mutex_unlock(&gsi_ctx->mlock);
-		return -GSI_STATUS_RES_ALLOC_FAILURE;
-	}
-
-	gsi_program_evt_ring_ctx(props, evt_id, gsi_ctx->per.ee);
-
-	spin_lock_init(&ctx->ring.slock);
-	gsi_init_evt_ring(props, &ctx->ring);
-
-	ctx->id = evt_id;
-	*evt_ring_hdl = evt_id;
-	atomic_inc(&gsi_ctx->num_evt_ring);
-	if (props->intf == GSI_EVT_CHTYPE_GPI_EV)
-		gsi_prime_evt_ring(ctx);
-	else if (props->intf == GSI_EVT_CHTYPE_WDI2_EV)
-		gsi_prime_evt_ring_wdi(ctx);
-	mutex_unlock(&gsi_ctx->mlock);
-
-	spin_lock_irqsave(&gsi_ctx->slock, flags);
-	gsi_writel(1 << evt_id, gsi_ctx->base +
-			GSI_EE_n_CNTXT_SRC_IEOB_IRQ_CLR_OFFS(ee));
-
-	/* enable ieob interrupts for GPI, enable MSI interrupts */
-	if ((props->intf != GSI_EVT_CHTYPE_GPI_EV) &&
-		(props->intr != GSI_INTR_MSI))
-		__gsi_config_ieob_irq(gsi_ctx->per.ee, 1 << evt_id, 0);
-	else
-		__gsi_config_ieob_irq(gsi_ctx->per.ee, 1 << ctx->id, ~0);
-	spin_unlock_irqrestore(&gsi_ctx->slock, flags);
-
-	return GSI_STATUS_SUCCESS;
-}
-EXPORT_SYMBOL(gsi_alloc_evt_ring);
-
-static void __gsi_write_evt_ring_scratch(unsigned long evt_ring_hdl,
-		union __packed gsi_evt_scratch val)
-{
-	gsi_writel(val.data.word1, gsi_ctx->base +
-		GSI_EE_n_EV_CH_k_SCRATCH_0_OFFS(evt_ring_hdl,
-			gsi_ctx->per.ee));
-	gsi_writel(val.data.word2, gsi_ctx->base +
-		GSI_EE_n_EV_CH_k_SCRATCH_1_OFFS(evt_ring_hdl,
-			gsi_ctx->per.ee));
-}
-
-int gsi_write_evt_ring_scratch(unsigned long evt_ring_hdl,
-		union __packed gsi_evt_scratch val)
-{
-	struct gsi_evt_ctx *ctx;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (evt_ring_hdl >= gsi_ctx->max_ev) {
-		GSIERR("bad params evt_ring_hdl=%lu\n", evt_ring_hdl);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	ctx = &gsi_ctx->evtr[evt_ring_hdl];
-
-	if (ctx->state != GSI_EVT_RING_STATE_ALLOCATED) {
-		GSIERR("bad state %d\n",
-				gsi_ctx->evtr[evt_ring_hdl].state);
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	mutex_lock(&ctx->mlock);
-	ctx->scratch = val;
-	__gsi_write_evt_ring_scratch(evt_ring_hdl, val);
-	mutex_unlock(&ctx->mlock);
-
-	return GSI_STATUS_SUCCESS;
-}
-EXPORT_SYMBOL(gsi_write_evt_ring_scratch);
-
-int gsi_dealloc_evt_ring(unsigned long evt_ring_hdl)
-{
-	uint32_t val;
-	enum gsi_evt_ch_cmd_opcode op = GSI_EVT_DE_ALLOC;
-	struct gsi_evt_ctx *ctx;
-	int res;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (evt_ring_hdl >= gsi_ctx->max_ev ||
-			evt_ring_hdl >= GSI_EVT_RING_MAX) {
-		GSIERR("bad params evt_ring_hdl=%lu\n", evt_ring_hdl);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	ctx = &gsi_ctx->evtr[evt_ring_hdl];
-
-	if (atomic_read(&ctx->chan_ref_cnt)) {
-		GSIERR("%d channels still using this event ring\n",
-			atomic_read(&ctx->chan_ref_cnt));
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	if (ctx->state != GSI_EVT_RING_STATE_ALLOCATED) {
-		GSIERR("bad state %d\n", ctx->state);
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	mutex_lock(&gsi_ctx->mlock);
-	reinit_completion(&ctx->compl);
-	val = (((evt_ring_hdl << GSI_EE_n_EV_CH_CMD_CHID_SHFT) &
-			GSI_EE_n_EV_CH_CMD_CHID_BMSK) |
-		((op << GSI_EE_n_EV_CH_CMD_OPCODE_SHFT) &
-			 GSI_EE_n_EV_CH_CMD_OPCODE_BMSK));
-	gsi_writel(val, gsi_ctx->base +
-			GSI_EE_n_EV_CH_CMD_OFFS(gsi_ctx->per.ee));
-	res = wait_for_completion_timeout(&ctx->compl, GSI_CMD_TIMEOUT);
-	if (res == 0) {
-		GSIERR("evt_id=%lu timed out\n", evt_ring_hdl);
-		mutex_unlock(&gsi_ctx->mlock);
-		return -GSI_STATUS_TIMED_OUT;
-	}
-
-	if (ctx->state != GSI_EVT_RING_STATE_NOT_ALLOCATED) {
-		GSIERR("evt_id=%lu unexpected state=%u\n", evt_ring_hdl,
-				ctx->state);
-		/*
-		 * IPA Hardware returned GSI RING not allocated, which is
-		 * unexpected hardware state.
-		 */
-		GSI_ASSERT();
-	}
-	mutex_unlock(&gsi_ctx->mlock);
-
-	if (!ctx->props.evchid_valid) {
-		mutex_lock(&gsi_ctx->mlock);
-		clear_bit(evt_ring_hdl, &gsi_ctx->evt_bmap);
-		mutex_unlock(&gsi_ctx->mlock);
-	}
-	atomic_dec(&gsi_ctx->num_evt_ring);
-
-	return GSI_STATUS_SUCCESS;
-}
-EXPORT_SYMBOL(gsi_dealloc_evt_ring);
-
-int gsi_query_evt_ring_db_addr(unsigned long evt_ring_hdl,
-		uint32_t *db_addr_wp_lsb, uint32_t *db_addr_wp_msb)
-{
-	struct gsi_evt_ctx *ctx;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (!db_addr_wp_msb || !db_addr_wp_lsb) {
-		GSIERR("bad params msb=%pK lsb=%pK\n", db_addr_wp_msb,
-				db_addr_wp_lsb);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (evt_ring_hdl >= gsi_ctx->max_ev) {
-		GSIERR("bad params evt_ring_hdl=%lu\n", evt_ring_hdl);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	ctx = &gsi_ctx->evtr[evt_ring_hdl];
-
-	if (ctx->state != GSI_EVT_RING_STATE_ALLOCATED) {
-		GSIERR("bad state %d\n",
-				gsi_ctx->evtr[evt_ring_hdl].state);
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	*db_addr_wp_lsb = gsi_ctx->per.phys_addr +
-		GSI_EE_n_EV_CH_k_DOORBELL_0_OFFS(evt_ring_hdl, gsi_ctx->per.ee);
-	*db_addr_wp_msb = gsi_ctx->per.phys_addr +
-		GSI_EE_n_EV_CH_k_DOORBELL_1_OFFS(evt_ring_hdl, gsi_ctx->per.ee);
-
-	return GSI_STATUS_SUCCESS;
-}
-EXPORT_SYMBOL(gsi_query_evt_ring_db_addr);
-
-int gsi_ring_evt_ring_db(unsigned long evt_ring_hdl, uint64_t value)
-{
-	struct gsi_evt_ctx *ctx;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (evt_ring_hdl >= gsi_ctx->max_ev) {
-		GSIERR("bad params evt_ring_hdl=%lu\n", evt_ring_hdl);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	ctx = &gsi_ctx->evtr[evt_ring_hdl];
-
-	if (ctx->state != GSI_EVT_RING_STATE_ALLOCATED) {
-		GSIERR("bad state %d\n",
-				gsi_ctx->evtr[evt_ring_hdl].state);
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	ctx->ring.wp_local = value;
-	gsi_ring_evt_doorbell(ctx);
-
-	return GSI_STATUS_SUCCESS;
-}
-EXPORT_SYMBOL(gsi_ring_evt_ring_db);
-
-int gsi_ring_ch_ring_db(unsigned long chan_hdl, uint64_t value)
-{
-	struct gsi_chan_ctx *ctx;
-	uint32_t val;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (chan_hdl >= gsi_ctx->max_ch) {
-		GSIERR("bad chan_hdl=%lu\n", chan_hdl);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	ctx = &gsi_ctx->chan[chan_hdl];
-
-	if (ctx->state != GSI_CHAN_STATE_STARTED) {
-		GSIERR("bad state %d\n", ctx->state);
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	ctx->ring.wp_local = value;
-
-	/* write MSB first */
-	val = ((ctx->ring.wp_local >> 32) &
-		GSI_EE_n_GSI_CH_k_DOORBELL_1_WRITE_PTR_MSB_BMSK) <<
-		GSI_EE_n_GSI_CH_k_DOORBELL_1_WRITE_PTR_MSB_SHFT;
-	gsi_writel(val, gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_DOORBELL_1_OFFS(ctx->props.ch_id,
-			gsi_ctx->per.ee));
-
-	gsi_ring_chan_doorbell(ctx);
-
-	return GSI_STATUS_SUCCESS;
-}
-EXPORT_SYMBOL(gsi_ring_ch_ring_db);
-
-int gsi_reset_evt_ring(unsigned long evt_ring_hdl)
-{
-	uint32_t val;
-	enum gsi_evt_ch_cmd_opcode op = GSI_EVT_RESET;
-	struct gsi_evt_ctx *ctx;
-	int res;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (evt_ring_hdl >= gsi_ctx->max_ev) {
-		GSIERR("bad params evt_ring_hdl=%lu\n", evt_ring_hdl);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	ctx = &gsi_ctx->evtr[evt_ring_hdl];
-
-	if (ctx->state != GSI_EVT_RING_STATE_ALLOCATED) {
-		GSIERR("bad state %d\n", ctx->state);
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	mutex_lock(&gsi_ctx->mlock);
-	reinit_completion(&ctx->compl);
-	val = (((evt_ring_hdl << GSI_EE_n_EV_CH_CMD_CHID_SHFT) &
-			GSI_EE_n_EV_CH_CMD_CHID_BMSK) |
-		((op << GSI_EE_n_EV_CH_CMD_OPCODE_SHFT) &
-			 GSI_EE_n_EV_CH_CMD_OPCODE_BMSK));
-	gsi_writel(val, gsi_ctx->base +
-			GSI_EE_n_EV_CH_CMD_OFFS(gsi_ctx->per.ee));
-	res = wait_for_completion_timeout(&ctx->compl, GSI_CMD_TIMEOUT);
-	if (res == 0) {
-		GSIERR("evt_id=%lu timed out\n", evt_ring_hdl);
-		mutex_unlock(&gsi_ctx->mlock);
-		return -GSI_STATUS_TIMED_OUT;
-	}
-
-	if (ctx->state != GSI_EVT_RING_STATE_ALLOCATED) {
-		GSIERR("evt_id=%lu unexpected state=%u\n", evt_ring_hdl,
-				ctx->state);
-		/*
-		 * IPA Hardware returned GSI RING not allocated, which is
-		 * unexpected. Indicates hardware instability.
-		 */
-		GSI_ASSERT();
-	}
-
-	gsi_program_evt_ring_ctx(&ctx->props, evt_ring_hdl, gsi_ctx->per.ee);
-	gsi_init_evt_ring(&ctx->props, &ctx->ring);
-
-	/* restore scratch */
-	__gsi_write_evt_ring_scratch(evt_ring_hdl, ctx->scratch);
-
-	if (ctx->props.intf == GSI_EVT_CHTYPE_GPI_EV)
-		gsi_prime_evt_ring(ctx);
-	if (ctx->props.intf == GSI_EVT_CHTYPE_WDI2_EV)
-		gsi_prime_evt_ring_wdi(ctx);
-	mutex_unlock(&gsi_ctx->mlock);
-
-	return GSI_STATUS_SUCCESS;
-}
-EXPORT_SYMBOL(gsi_reset_evt_ring);
-
-int gsi_get_evt_ring_cfg(unsigned long evt_ring_hdl,
-		struct gsi_evt_ring_props *props, union gsi_evt_scratch *scr)
-{
-	struct gsi_evt_ctx *ctx;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (!props || !scr) {
-		GSIERR("bad params props=%pK scr=%pK\n", props, scr);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (evt_ring_hdl >= gsi_ctx->max_ev) {
-		GSIERR("bad params evt_ring_hdl=%lu\n", evt_ring_hdl);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	ctx = &gsi_ctx->evtr[evt_ring_hdl];
-
-	if (ctx->state == GSI_EVT_RING_STATE_NOT_ALLOCATED) {
-		GSIERR("bad state %d\n", ctx->state);
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	mutex_lock(&ctx->mlock);
-	*props = ctx->props;
-	*scr = ctx->scratch;
-	mutex_unlock(&ctx->mlock);
-
-	return GSI_STATUS_SUCCESS;
-}
-EXPORT_SYMBOL(gsi_get_evt_ring_cfg);
-
-int gsi_set_evt_ring_cfg(unsigned long evt_ring_hdl,
-		struct gsi_evt_ring_props *props, union gsi_evt_scratch *scr)
-{
-	struct gsi_evt_ctx *ctx;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (!props || gsi_validate_evt_ring_props(props)) {
-		GSIERR("bad params props=%pK\n", props);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (evt_ring_hdl >= gsi_ctx->max_ev) {
-		GSIERR("bad params evt_ring_hdl=%lu\n", evt_ring_hdl);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	ctx = &gsi_ctx->evtr[evt_ring_hdl];
-
-	if (ctx->state != GSI_EVT_RING_STATE_ALLOCATED) {
-		GSIERR("bad state %d\n", ctx->state);
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	if (ctx->props.exclusive != props->exclusive) {
-		GSIERR("changing immutable fields not supported\n");
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	mutex_lock(&ctx->mlock);
-	ctx->props = *props;
-	if (scr)
-		ctx->scratch = *scr;
-	mutex_unlock(&ctx->mlock);
-
-	return gsi_reset_evt_ring(evt_ring_hdl);
-}
-EXPORT_SYMBOL(gsi_set_evt_ring_cfg);
-
-static void gsi_program_chan_ctx_qos(struct gsi_chan_props *props,
-	unsigned int ee)
-{
-	uint32_t val;
-
-	val =
-	(((props->low_weight <<
-		GSI_EE_n_GSI_CH_k_QOS_WRR_WEIGHT_SHFT) &
-		GSI_EE_n_GSI_CH_k_QOS_WRR_WEIGHT_BMSK) |
-	((props->max_prefetch <<
-		 GSI_EE_n_GSI_CH_k_QOS_MAX_PREFETCH_SHFT) &
-		 GSI_EE_n_GSI_CH_k_QOS_MAX_PREFETCH_BMSK) |
-	((props->use_db_eng <<
-		GSI_EE_n_GSI_CH_k_QOS_USE_DB_ENG_SHFT) &
-		 GSI_EE_n_GSI_CH_k_QOS_USE_DB_ENG_BMSK));
-	if (gsi_ctx->per.ver >= GSI_VER_2_0)
-		val |= ((props->prefetch_mode <<
-			GSI_EE_n_GSI_CH_k_QOS_USE_ESCAPE_BUF_ONLY_SHFT)
-			& GSI_EE_n_GSI_CH_k_QOS_USE_ESCAPE_BUF_ONLY_BMSK);
-
-	gsi_writel(val, gsi_ctx->base +
-			GSI_EE_n_GSI_CH_k_QOS_OFFS(props->ch_id, ee));
-}
-
-static void gsi_program_chan_ctx_qos_v2_5(struct gsi_chan_props *props,
-	unsigned int ee)
-{
-	uint32_t val;
-
-	val =
-	(((props->low_weight <<
-		GSI_V2_5_EE_n_GSI_CH_k_QOS_WRR_WEIGHT_SHFT) &
-		GSI_V2_5_EE_n_GSI_CH_k_QOS_WRR_WEIGHT_BMSK) |
-	((props->max_prefetch <<
-		 GSI_V2_5_EE_n_GSI_CH_k_QOS_MAX_PREFETCH_SHFT) &
-		 GSI_V2_5_EE_n_GSI_CH_k_QOS_MAX_PREFETCH_BMSK) |
-	((props->use_db_eng <<
-		GSI_V2_5_EE_n_GSI_CH_k_QOS_USE_DB_ENG_SHFT) &
-		GSI_V2_5_EE_n_GSI_CH_k_QOS_USE_DB_ENG_BMSK) |
-	((props->prefetch_mode <<
-		GSI_V2_5_EE_n_GSI_CH_k_QOS_PREFETCH_MODE_SHFT) &
-		GSI_V2_5_EE_n_GSI_CH_k_QOS_PREFETCH_MODE_BMSK) |
-	((props->empty_lvl_threshold <<
-		GSI_V2_5_EE_n_GSI_CH_k_QOS_EMPTY_LVL_THRSHOLD_SHFT) &
-		GSI_V2_5_EE_n_GSI_CH_k_QOS_EMPTY_LVL_THRSHOLD_BMSK));
-
-	gsi_writel(val, gsi_ctx->base +
-			GSI_V2_5_EE_n_GSI_CH_k_QOS_OFFS(props->ch_id, ee));
-}
-
-static void gsi_program_chan_ctx(struct gsi_chan_props *props, unsigned int ee,
-		uint8_t erindex)
-{
-	uint32_t val;
-	uint32_t prot;
-	uint32_t prot_msb;
-
-	switch (props->prot) {
-	case GSI_CHAN_PROT_MHI:
-	case GSI_CHAN_PROT_XHCI:
-	case GSI_CHAN_PROT_GPI:
-	case GSI_CHAN_PROT_XDCI:
-	case GSI_CHAN_PROT_WDI2:
-	case GSI_CHAN_PROT_WDI3:
-	case GSI_CHAN_PROT_GCI:
-	case GSI_CHAN_PROT_MHIP:
-		prot_msb = 0;
-		break;
-	case GSI_CHAN_PROT_AQC:
-	case GSI_CHAN_PROT_11AD:
-		prot_msb = 1;
-		break;
-	default:
-		GSIERR("Unsupported protocol %d\n", props->prot);
-		WARN_ON(1);
-		return;
-	}
-	prot = props->prot;
-
-	val = ((prot <<
-		GSI_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_PROTOCOL_SHFT) &
-		GSI_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_PROTOCOL_BMSK);
-	if (gsi_ctx->per.ver >= GSI_VER_2_5) {
-		val |= ((prot_msb <<
-		GSI_V2_5_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_PROTOCOL_MSB_SHFT) &
-		GSI_V2_5_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_PROTOCOL_MSB_BMSK);
-	}
-
-	val |= (((props->dir << GSI_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_DIR_SHFT) &
-			 GSI_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_DIR_BMSK) |
-		((erindex << GSI_EE_n_GSI_CH_k_CNTXT_0_ERINDEX_SHFT) &
-			 GSI_EE_n_GSI_CH_k_CNTXT_0_ERINDEX_BMSK) |
-		((props->re_size << GSI_EE_n_GSI_CH_k_CNTXT_0_ELEMENT_SIZE_SHFT)
-			 & GSI_EE_n_GSI_CH_k_CNTXT_0_ELEMENT_SIZE_BMSK));
-	gsi_writel(val, gsi_ctx->base +
-			GSI_EE_n_GSI_CH_k_CNTXT_0_OFFS(props->ch_id, ee));
-
-	val = (props->ring_len & GSI_EE_n_GSI_CH_k_CNTXT_1_R_LENGTH_BMSK) <<
-		GSI_EE_n_GSI_CH_k_CNTXT_1_R_LENGTH_SHFT;
-	gsi_writel(val, gsi_ctx->base +
-			GSI_EE_n_GSI_CH_k_CNTXT_1_OFFS(props->ch_id, ee));
-
-	val = (props->ring_base_addr &
-			GSI_EE_n_GSI_CH_k_CNTXT_2_R_BASE_ADDR_LSBS_BMSK) <<
-		GSI_EE_n_GSI_CH_k_CNTXT_2_R_BASE_ADDR_LSBS_SHFT;
-	gsi_writel(val, gsi_ctx->base +
-			GSI_EE_n_GSI_CH_k_CNTXT_2_OFFS(props->ch_id, ee));
-
-	val = ((props->ring_base_addr >> 32) &
-		GSI_EE_n_GSI_CH_k_CNTXT_3_R_BASE_ADDR_MSBS_BMSK) <<
-		GSI_EE_n_GSI_CH_k_CNTXT_3_R_BASE_ADDR_MSBS_SHFT;
-	gsi_writel(val, gsi_ctx->base +
-			GSI_EE_n_GSI_CH_k_CNTXT_3_OFFS(props->ch_id, ee));
-
-	if (gsi_ctx->per.ver >= GSI_VER_2_5)
-		gsi_program_chan_ctx_qos_v2_5(props, ee);
-	else
-		gsi_program_chan_ctx_qos(props, ee);
-}
-
-static void gsi_init_chan_ring(struct gsi_chan_props *props,
-		struct gsi_ring_ctx *ctx)
-{
-	ctx->base_va = (uintptr_t)props->ring_base_vaddr;
-	ctx->base = props->ring_base_addr;
-	ctx->wp = ctx->base;
-	ctx->rp = ctx->base;
-	ctx->wp_local = ctx->base;
-	ctx->rp_local = ctx->base;
-	ctx->len = props->ring_len;
-	ctx->elem_sz = props->re_size;
-	ctx->max_num_elem = ctx->len / ctx->elem_sz - 1;
-	ctx->end = ctx->base + (ctx->max_num_elem + 1) *
-		ctx->elem_sz;
-}
-
-static int gsi_validate_channel_props(struct gsi_chan_props *props)
-{
-	uint64_t ra;
-	uint64_t last;
-
-	if (props->ch_id >= gsi_ctx->max_ch) {
-		GSIERR("ch_id %u invalid\n", props->ch_id);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if ((props->re_size == GSI_CHAN_RE_SIZE_4B &&
-				props->ring_len % 4) ||
-			(props->re_size == GSI_CHAN_RE_SIZE_8B &&
-				 props->ring_len % 8) ||
-			(props->re_size == GSI_CHAN_RE_SIZE_16B &&
-				 props->ring_len % 16) ||
-			(props->re_size == GSI_CHAN_RE_SIZE_32B &&
-				 props->ring_len % 32)) {
-		GSIERR("bad params ring_len %u not a multiple of re size %u\n",
-				props->ring_len, props->re_size);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	ra = props->ring_base_addr;
-	do_div(ra, roundup_pow_of_two(props->ring_len));
-
-	if (props->ring_base_addr != ra * roundup_pow_of_two(props->ring_len)) {
-		GSIERR("bad params ring base not aligned 0x%llx align 0x%lx\n",
-				props->ring_base_addr,
-				roundup_pow_of_two(props->ring_len));
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	last = props->ring_base_addr + props->ring_len - props->re_size;
-
-	/* MSB should stay same within the ring */
-	if ((props->ring_base_addr & 0xFFFFFFFF00000000ULL) !=
-	    (last & 0xFFFFFFFF00000000ULL)) {
-		GSIERR("MSB is not fixed on ring base 0x%llx size 0x%x\n",
-			props->ring_base_addr,
-			props->ring_len);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (props->prot == GSI_CHAN_PROT_GPI &&
-			!props->ring_base_vaddr) {
-		GSIERR("protocol %u requires ring base VA\n", props->prot);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (props->low_weight > GSI_MAX_CH_LOW_WEIGHT) {
-		GSIERR("invalid channel low weight %u\n", props->low_weight);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (props->prot == GSI_CHAN_PROT_GPI && !props->xfer_cb) {
-		GSIERR("xfer callback must be provided\n");
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (!props->err_cb) {
-		GSIERR("err callback must be provided\n");
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	return GSI_STATUS_SUCCESS;
-}
-
-int gsi_alloc_channel(struct gsi_chan_props *props, unsigned long dev_hdl,
-		unsigned long *chan_hdl)
-{
-	struct gsi_chan_ctx *ctx;
-	uint32_t val;
-	int res;
-	int ee;
-	enum gsi_ch_cmd_opcode op = GSI_CH_ALLOCATE;
-	uint8_t erindex;
-	struct gsi_user_data *user_data;
-	size_t user_data_size;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (!props || !chan_hdl || dev_hdl != (uintptr_t)gsi_ctx) {
-		GSIERR("bad params props=%pK dev_hdl=0x%lx chan_hdl=%pK\n",
-				props, dev_hdl, chan_hdl);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (gsi_validate_channel_props(props)) {
-		GSIERR("bad params\n");
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (props->evt_ring_hdl != ~0) {
-		if (props->evt_ring_hdl >= gsi_ctx->max_ev) {
-			GSIERR("invalid evt ring=%lu\n", props->evt_ring_hdl);
-			return -GSI_STATUS_INVALID_PARAMS;
-		}
-
-		if (atomic_read(
-			&gsi_ctx->evtr[props->evt_ring_hdl].chan_ref_cnt) &&
-			gsi_ctx->evtr[props->evt_ring_hdl].props.exclusive &&
-			gsi_ctx->evtr[props->evt_ring_hdl].chan->props.prot !=
-			GSI_CHAN_PROT_GCI) {
-			GSIERR("evt ring=%lu exclusively used by ch_hdl=%pK\n",
-				props->evt_ring_hdl, chan_hdl);
-			return -GSI_STATUS_UNSUPPORTED_OP;
-		}
-	}
-
-	ctx = &gsi_ctx->chan[props->ch_id];
-	if (ctx->allocated) {
-		GSIERR("chan %d already allocated\n", props->ch_id);
-		return -GSI_STATUS_NODEV;
-	}
-	memset(ctx, 0, sizeof(*ctx));
-
-	/* For IPA offloaded WDI channels not required user_data pointer */
-	if (props->prot != GSI_CHAN_PROT_WDI2 &&
-		props->prot != GSI_CHAN_PROT_WDI3)
-		user_data_size = props->ring_len / props->re_size;
-	else
-		user_data_size = props->re_size;
-	/*
-	 * GCI channels might have OOO event completions up to GSI_VEID_MAX.
-	 * user_data needs to be large enough to accommodate those.
-	 * TODO: increase user data size if GSI_VEID_MAX is not enough
-	 */
-	if (props->prot == GSI_CHAN_PROT_GCI)
-		user_data_size += GSI_VEID_MAX;
-
-	user_data = devm_kzalloc(gsi_ctx->dev,
-		user_data_size * sizeof(*user_data),
-		GFP_KERNEL);
-	if (user_data == NULL) {
-		GSIERR("context not allocated\n");
-		return -GSI_STATUS_RES_ALLOC_FAILURE;
-	}
-
-	mutex_init(&ctx->mlock);
-	init_completion(&ctx->compl);
-	atomic_set(&ctx->poll_mode, GSI_CHAN_MODE_CALLBACK);
-	ctx->props = *props;
-
-	if (gsi_ctx->per.ver != GSI_VER_2_2) {
-		mutex_lock(&gsi_ctx->mlock);
-		ee = gsi_ctx->per.ee;
-		gsi_ctx->ch_dbg[props->ch_id].ch_allocate++;
-		val = (((props->ch_id << GSI_EE_n_GSI_CH_CMD_CHID_SHFT) &
-					GSI_EE_n_GSI_CH_CMD_CHID_BMSK) |
-				((op << GSI_EE_n_GSI_CH_CMD_OPCODE_SHFT) &
-				 GSI_EE_n_GSI_CH_CMD_OPCODE_BMSK));
-		gsi_writel(val, gsi_ctx->base +
-				GSI_EE_n_GSI_CH_CMD_OFFS(ee));
-		res = wait_for_completion_timeout(&ctx->compl, GSI_CMD_TIMEOUT);
-		if (res == 0) {
-			GSIERR("chan_hdl=%u timed out\n", props->ch_id);
-			mutex_unlock(&gsi_ctx->mlock);
-			devm_kfree(gsi_ctx->dev, user_data);
-			return -GSI_STATUS_TIMED_OUT;
-		}
-		if (ctx->state != GSI_CHAN_STATE_ALLOCATED) {
-			GSIERR("chan_hdl=%u allocation failed state=%d\n",
-					props->ch_id, ctx->state);
-			mutex_unlock(&gsi_ctx->mlock);
-			devm_kfree(gsi_ctx->dev, user_data);
-			return -GSI_STATUS_RES_ALLOC_FAILURE;
-		}
-		mutex_unlock(&gsi_ctx->mlock);
-	} else {
-		mutex_lock(&gsi_ctx->mlock);
-		ctx->state = GSI_CHAN_STATE_ALLOCATED;
-		mutex_unlock(&gsi_ctx->mlock);
-	}
-	erindex = props->evt_ring_hdl != ~0 ? props->evt_ring_hdl :
-		GSI_NO_EVT_ERINDEX;
-	if (erindex != GSI_NO_EVT_ERINDEX && erindex >= GSI_EVT_RING_MAX) {
-		GSIERR("invalid erindex %u\n", erindex);
-		devm_kfree(gsi_ctx->dev, user_data);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (erindex < GSI_EVT_RING_MAX) {
-		ctx->evtr = &gsi_ctx->evtr[erindex];
-		if (props->prot != GSI_CHAN_PROT_GCI)
-			atomic_inc(&ctx->evtr->chan_ref_cnt);
-		if (props->prot != GSI_CHAN_PROT_GCI &&
-			ctx->evtr->props.exclusive &&
-			atomic_read(&ctx->evtr->chan_ref_cnt) == 1)
-			ctx->evtr->chan = ctx;
-	}
-
-	gsi_program_chan_ctx(props, gsi_ctx->per.ee, erindex);
-
-	spin_lock_init(&ctx->ring.slock);
-	gsi_init_chan_ring(props, &ctx->ring);
-	if (!props->max_re_expected)
-		ctx->props.max_re_expected = ctx->ring.max_num_elem;
-	ctx->user_data = user_data;
-	*chan_hdl = props->ch_id;
-	ctx->allocated = true;
-	ctx->stats.dp.last_timestamp = jiffies_to_msecs(jiffies);
-	atomic_inc(&gsi_ctx->num_chan);
-
-	return GSI_STATUS_SUCCESS;
-}
-EXPORT_SYMBOL(gsi_alloc_channel);
-
-static int gsi_alloc_ap_channel(unsigned int chan_hdl)
-{
-	struct gsi_chan_ctx *ctx;
-	uint32_t val;
-	int res;
-	int ee;
-	enum gsi_ch_cmd_opcode op = GSI_CH_ALLOCATE;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	ctx = &gsi_ctx->chan[chan_hdl];
-	if (ctx->allocated) {
-		GSIERR("chan %d already allocated\n", chan_hdl);
-		return -GSI_STATUS_NODEV;
-	}
-
-	memset(ctx, 0, sizeof(*ctx));
-
-	mutex_init(&ctx->mlock);
-	init_completion(&ctx->compl);
-	atomic_set(&ctx->poll_mode, GSI_CHAN_MODE_CALLBACK);
-
-	mutex_lock(&gsi_ctx->mlock);
-	ee = gsi_ctx->per.ee;
-	gsi_ctx->ch_dbg[chan_hdl].ch_allocate++;
-	val = (((chan_hdl << GSI_EE_n_GSI_CH_CMD_CHID_SHFT) &
-				GSI_EE_n_GSI_CH_CMD_CHID_BMSK) |
-			((op << GSI_EE_n_GSI_CH_CMD_OPCODE_SHFT) &
-			 GSI_EE_n_GSI_CH_CMD_OPCODE_BMSK));
-	gsi_writel(val, gsi_ctx->base +
-			GSI_EE_n_GSI_CH_CMD_OFFS(ee));
-	res = wait_for_completion_timeout(&ctx->compl, GSI_CMD_TIMEOUT);
-	if (res == 0) {
-		GSIERR("chan_hdl=%u timed out\n", chan_hdl);
-		mutex_unlock(&gsi_ctx->mlock);
-		return -GSI_STATUS_TIMED_OUT;
-	}
-	if (ctx->state != GSI_CHAN_STATE_ALLOCATED) {
-		GSIERR("chan_hdl=%u allocation failed state=%d\n",
-				chan_hdl, ctx->state);
-		mutex_unlock(&gsi_ctx->mlock);
-		return -GSI_STATUS_RES_ALLOC_FAILURE;
-	}
-	mutex_unlock(&gsi_ctx->mlock);
-
-	return GSI_STATUS_SUCCESS;
-}
-
-static void __gsi_write_channel_scratch(unsigned long chan_hdl,
-		union __packed gsi_channel_scratch val)
-{
-	gsi_writel(val.data.word1, gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_SCRATCH_0_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-	gsi_writel(val.data.word2, gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_SCRATCH_1_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-	gsi_writel(val.data.word3, gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_SCRATCH_2_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-
-	gsi_writel(val.data.word4, gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_SCRATCH_3_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-}
-
-int gsi_write_channel_scratch3_reg(unsigned long chan_hdl,
-		union __packed gsi_wdi_channel_scratch3_reg val)
-{
-	struct gsi_chan_ctx *ctx;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (chan_hdl >= gsi_ctx->max_ch) {
-		GSIERR("bad params chan_hdl=%lu\n", chan_hdl);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	ctx = &gsi_ctx->chan[chan_hdl];
-
-	mutex_lock(&ctx->mlock);
-
-	ctx->scratch.wdi.endp_metadatareg_offset =
-				val.wdi.endp_metadatareg_offset;
-	ctx->scratch.wdi.qmap_id = val.wdi.qmap_id;
-
-	gsi_writel(val.data.word1, gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_SCRATCH_3_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-	mutex_unlock(&ctx->mlock);
-	return GSI_STATUS_SUCCESS;
-}
-EXPORT_SYMBOL(gsi_write_channel_scratch3_reg);
-
-int gsi_write_channel_scratch2_reg(unsigned long chan_hdl,
-		union __packed gsi_wdi2_channel_scratch2_reg val)
-{
-	struct gsi_chan_ctx *ctx;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (chan_hdl >= gsi_ctx->max_ch) {
-		GSIERR("bad params chan_hdl=%lu\n", chan_hdl);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	ctx = &gsi_ctx->chan[chan_hdl];
-
-	mutex_lock(&ctx->mlock);
-
-	ctx->scratch.wdi2_new.endp_metadatareg_offset =
-				val.wdi.endp_metadatareg_offset;
-	ctx->scratch.wdi2_new.qmap_id = val.wdi.qmap_id;
-	val.wdi.update_ri_moderation_threshold =
-		ctx->scratch.wdi2_new.update_ri_moderation_threshold;
-	gsi_writel(val.data.word1, gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_SCRATCH_2_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-	mutex_unlock(&ctx->mlock);
-	return GSI_STATUS_SUCCESS;
-}
-EXPORT_SYMBOL(gsi_write_channel_scratch2_reg);
-
-static void __gsi_read_channel_scratch(unsigned long chan_hdl,
-		union __packed gsi_channel_scratch * val)
-{
-	val->data.word1 = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_SCRATCH_0_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-
-	val->data.word2 = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_SCRATCH_1_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-
-	val->data.word3 = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_SCRATCH_2_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-
-	val->data.word4 = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_SCRATCH_3_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-}
-
-static union __packed gsi_channel_scratch __gsi_update_mhi_channel_scratch(
-	unsigned long chan_hdl, struct __packed gsi_mhi_channel_scratch mscr)
-{
-	union __packed gsi_channel_scratch scr;
-
-	/* below sequence is not atomic. assumption is sequencer specific fields
-	 * will remain unchanged across this sequence
-	 */
-
-	/* READ */
-	scr.data.word1 = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_SCRATCH_0_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-
-	scr.data.word2 = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_SCRATCH_1_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-
-	scr.data.word3 = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_SCRATCH_2_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-
-	scr.data.word4 = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_SCRATCH_3_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-
-	/* UPDATE */
-	scr.mhi.mhi_host_wp_addr = mscr.mhi_host_wp_addr;
-	scr.mhi.assert_bit40 = mscr.assert_bit40;
-	scr.mhi.polling_configuration = mscr.polling_configuration;
-	scr.mhi.burst_mode_enabled = mscr.burst_mode_enabled;
-	scr.mhi.polling_mode = mscr.polling_mode;
-	scr.mhi.oob_mod_threshold = mscr.oob_mod_threshold;
-
-	if (gsi_ctx->per.ver < GSI_VER_2_5) {
-		scr.mhi.max_outstanding_tre = mscr.max_outstanding_tre;
-		scr.mhi.outstanding_threshold = mscr.outstanding_threshold;
-	}
-
-	/* WRITE */
-	gsi_writel(scr.data.word1, gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_SCRATCH_0_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-
-	gsi_writel(scr.data.word2, gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_SCRATCH_1_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-
-	gsi_writel(scr.data.word3, gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_SCRATCH_2_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-
-	gsi_writel(scr.data.word4, gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_SCRATCH_3_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-
-	return scr;
-}
-
-int gsi_write_channel_scratch(unsigned long chan_hdl,
-		union __packed gsi_channel_scratch val)
-{
-	struct gsi_chan_ctx *ctx;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (chan_hdl >= gsi_ctx->max_ch) {
-		GSIERR("bad params chan_hdl=%lu\n", chan_hdl);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (gsi_ctx->chan[chan_hdl].state != GSI_CHAN_STATE_ALLOCATED &&
-		gsi_ctx->chan[chan_hdl].state != GSI_CHAN_STATE_STOPPED) {
-		GSIERR("bad state %d\n",
-				gsi_ctx->chan[chan_hdl].state);
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	ctx = &gsi_ctx->chan[chan_hdl];
-
-	mutex_lock(&ctx->mlock);
-	ctx->scratch = val;
-	__gsi_write_channel_scratch(chan_hdl, val);
-	mutex_unlock(&ctx->mlock);
-
-	return GSI_STATUS_SUCCESS;
-}
-EXPORT_SYMBOL(gsi_write_channel_scratch);
-
-int gsi_read_channel_scratch(unsigned long chan_hdl,
-		union __packed gsi_channel_scratch *val)
-{
-	struct gsi_chan_ctx *ctx;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (chan_hdl >= gsi_ctx->max_ch) {
-		GSIERR("bad params chan_hdl=%lu\n", chan_hdl);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (gsi_ctx->chan[chan_hdl].state != GSI_CHAN_STATE_ALLOCATED &&
-		gsi_ctx->chan[chan_hdl].state != GSI_CHAN_STATE_STARTED &&
-		gsi_ctx->chan[chan_hdl].state != GSI_CHAN_STATE_STOPPED) {
-		GSIERR("bad state %d\n",
-				gsi_ctx->chan[chan_hdl].state);
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	ctx = &gsi_ctx->chan[chan_hdl];
-
-	mutex_lock(&ctx->mlock);
-	__gsi_read_channel_scratch(chan_hdl, val);
-	mutex_unlock(&ctx->mlock);
-
-	return GSI_STATUS_SUCCESS;
-}
-EXPORT_SYMBOL(gsi_read_channel_scratch);
-
-int gsi_update_mhi_channel_scratch(unsigned long chan_hdl,
-		struct __packed gsi_mhi_channel_scratch mscr)
-{
-	struct gsi_chan_ctx *ctx;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (chan_hdl >= gsi_ctx->max_ch) {
-		GSIERR("bad params chan_hdl=%lu\n", chan_hdl);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (gsi_ctx->chan[chan_hdl].state != GSI_CHAN_STATE_ALLOCATED &&
-		gsi_ctx->chan[chan_hdl].state != GSI_CHAN_STATE_STOPPED) {
-		GSIERR("bad state %d\n",
-				gsi_ctx->chan[chan_hdl].state);
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	ctx = &gsi_ctx->chan[chan_hdl];
-
-	mutex_lock(&ctx->mlock);
-	ctx->scratch = __gsi_update_mhi_channel_scratch(chan_hdl, mscr);
-	mutex_unlock(&ctx->mlock);
-
-	return GSI_STATUS_SUCCESS;
-}
-EXPORT_SYMBOL(gsi_update_mhi_channel_scratch);
-
-int gsi_query_channel_db_addr(unsigned long chan_hdl,
-		uint32_t *db_addr_wp_lsb, uint32_t *db_addr_wp_msb)
-{
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (!db_addr_wp_msb || !db_addr_wp_lsb) {
-		GSIERR("bad params msb=%pK lsb=%pK\n", db_addr_wp_msb,
-				db_addr_wp_lsb);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (chan_hdl >= gsi_ctx->max_ch) {
-		GSIERR("bad params chan_hdl=%lu\n", chan_hdl);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (gsi_ctx->chan[chan_hdl].state == GSI_CHAN_STATE_NOT_ALLOCATED) {
-		GSIERR("bad state %d\n",
-				gsi_ctx->chan[chan_hdl].state);
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	*db_addr_wp_lsb = gsi_ctx->per.phys_addr +
-		GSI_EE_n_GSI_CH_k_DOORBELL_0_OFFS(chan_hdl, gsi_ctx->per.ee);
-	*db_addr_wp_msb = gsi_ctx->per.phys_addr +
-		GSI_EE_n_GSI_CH_k_DOORBELL_1_OFFS(chan_hdl, gsi_ctx->per.ee);
-
-	return GSI_STATUS_SUCCESS;
-}
-EXPORT_SYMBOL(gsi_query_channel_db_addr);
-
-int gsi_start_channel(unsigned long chan_hdl)
-{
-	enum gsi_ch_cmd_opcode op = GSI_CH_START;
-	uint32_t val;
-	struct gsi_chan_ctx *ctx;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (chan_hdl >= gsi_ctx->max_ch) {
-		GSIERR("bad params chan_hdl=%lu\n", chan_hdl);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	ctx = &gsi_ctx->chan[chan_hdl];
-
-	if (ctx->state != GSI_CHAN_STATE_ALLOCATED &&
-		ctx->state != GSI_CHAN_STATE_STOP_IN_PROC &&
-		ctx->state != GSI_CHAN_STATE_STOPPED) {
-		GSIERR("bad state %d\n", ctx->state);
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	mutex_lock(&gsi_ctx->mlock);
-	reinit_completion(&ctx->compl);
-
-	/* check if INTSET is in IRQ mode for GPI channel */
-	val = gsi_readl(gsi_ctx->base +
-			GSI_EE_n_CNTXT_INTSET_OFFS(gsi_ctx->per.ee));
-	if (ctx->evtr->props.intf == GSI_EVT_CHTYPE_GPI_EV &&
-		val != GSI_INTR_IRQ) {
-		GSIERR("GSI_EE_n_CNTXT_INTSET_OFFS %d\n", val);
-		BUG();
-	}
-
-	gsi_ctx->ch_dbg[chan_hdl].ch_start++;
-	val = (((chan_hdl << GSI_EE_n_GSI_CH_CMD_CHID_SHFT) &
-			GSI_EE_n_GSI_CH_CMD_CHID_BMSK) |
-		((op << GSI_EE_n_GSI_CH_CMD_OPCODE_SHFT) &
-		 GSI_EE_n_GSI_CH_CMD_OPCODE_BMSK));
-	gsi_writel(val, gsi_ctx->base +
-			GSI_EE_n_GSI_CH_CMD_OFFS(gsi_ctx->per.ee));
-
-	GSIDBG("GSI Channel Start, waiting for completion\n");
-	gsi_channel_state_change_wait(chan_hdl,
-		ctx,
-		GSI_START_CMD_TIMEOUT_MS, op);
-
-	if (ctx->state != GSI_CHAN_STATE_STARTED) {
-		/*
-		 * Hardware returned unexpected status, unexpected
-		 * hardware state.
-		 */
-		GSIERR("chan=%lu timed out, unexpected state=%u\n",
-			chan_hdl, ctx->state);
-		GSI_ASSERT();
-	}
-
-	GSIDBG("GSI Channel=%lu Start success\n", chan_hdl);
-
-	/* write order MUST be MSB followed by LSB */
-	val = ((ctx->ring.wp_local >> 32) &
-		GSI_EE_n_GSI_CH_k_DOORBELL_1_WRITE_PTR_MSB_BMSK) <<
-		GSI_EE_n_GSI_CH_k_DOORBELL_1_WRITE_PTR_MSB_SHFT;
-	gsi_writel(val, gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_DOORBELL_1_OFFS(ctx->props.ch_id,
-		gsi_ctx->per.ee));
-
-	mutex_unlock(&gsi_ctx->mlock);
-
-	return GSI_STATUS_SUCCESS;
-}
-EXPORT_SYMBOL(gsi_start_channel);
-
-int gsi_stop_channel(unsigned long chan_hdl)
-{
-	enum gsi_ch_cmd_opcode op = GSI_CH_STOP;
-	int res;
-	uint32_t val;
-	struct gsi_chan_ctx *ctx;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (chan_hdl >= gsi_ctx->max_ch) {
-		GSIERR("bad params chan_hdl=%lu\n", chan_hdl);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	ctx = &gsi_ctx->chan[chan_hdl];
-
-	if (ctx->state == GSI_CHAN_STATE_STOPPED) {
-		GSIDBG("chan_hdl=%lu already stopped\n", chan_hdl);
-		return GSI_STATUS_SUCCESS;
-	}
-
-	if (ctx->state != GSI_CHAN_STATE_STARTED &&
-		ctx->state != GSI_CHAN_STATE_STOP_IN_PROC &&
-		ctx->state != GSI_CHAN_STATE_ERROR) {
-		GSIERR("bad state %d\n", ctx->state);
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	mutex_lock(&gsi_ctx->mlock);
-	reinit_completion(&ctx->compl);
-
-	/* check if INTSET is in IRQ mode for GPI channel */
-	val = gsi_readl(gsi_ctx->base +
-			GSI_EE_n_CNTXT_INTSET_OFFS(gsi_ctx->per.ee));
-	if (ctx->evtr->props.intf == GSI_EVT_CHTYPE_GPI_EV &&
-		val != GSI_INTR_IRQ) {
-		GSIERR("GSI_EE_n_CNTXT_INTSET_OFFS %d\n", val);
-		BUG();
-	}
-
-	gsi_ctx->ch_dbg[chan_hdl].ch_stop++;
-	val = (((chan_hdl << GSI_EE_n_GSI_CH_CMD_CHID_SHFT) &
-			GSI_EE_n_GSI_CH_CMD_CHID_BMSK) |
-		((op << GSI_EE_n_GSI_CH_CMD_OPCODE_SHFT) &
-		 GSI_EE_n_GSI_CH_CMD_OPCODE_BMSK));
-	gsi_writel(val, gsi_ctx->base +
-			GSI_EE_n_GSI_CH_CMD_OFFS(gsi_ctx->per.ee));
-
-	GSIDBG("GSI Channel Stop, waiting for completion\n");
-	gsi_channel_state_change_wait(chan_hdl,
-		ctx,
-		GSI_STOP_CMD_TIMEOUT_MS, op);
-
-	if (ctx->state != GSI_CHAN_STATE_STOPPED &&
-		ctx->state != GSI_CHAN_STATE_STOP_IN_PROC) {
-		GSIERR("chan=%lu unexpected state=%u\n", chan_hdl, ctx->state);
-		res = -GSI_STATUS_BAD_STATE;
-		BUG();
-		goto free_lock;
-	}
-
-	if (ctx->state == GSI_CHAN_STATE_STOP_IN_PROC) {
-		GSIERR("chan=%lu busy try again\n", chan_hdl);
-		res = -GSI_STATUS_AGAIN;
-		goto free_lock;
-	}
-
-	res = GSI_STATUS_SUCCESS;
-
-free_lock:
-	mutex_unlock(&gsi_ctx->mlock);
-	return res;
-}
-EXPORT_SYMBOL(gsi_stop_channel);
-
-int gsi_stop_db_channel(unsigned long chan_hdl)
-{
-	enum gsi_ch_cmd_opcode op = GSI_CH_DB_STOP;
-	int res;
-	uint32_t val;
-	struct gsi_chan_ctx *ctx;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (chan_hdl >= gsi_ctx->max_ch) {
-		GSIERR("bad params chan_hdl=%lu\n", chan_hdl);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	ctx = &gsi_ctx->chan[chan_hdl];
-
-	if (ctx->state == GSI_CHAN_STATE_STOPPED) {
-		GSIDBG("chan_hdl=%lu already stopped\n", chan_hdl);
-		return GSI_STATUS_SUCCESS;
-	}
-
-	if (ctx->state != GSI_CHAN_STATE_STARTED &&
-		ctx->state != GSI_CHAN_STATE_STOP_IN_PROC) {
-		GSIERR("bad state %d\n", ctx->state);
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	mutex_lock(&gsi_ctx->mlock);
-	reinit_completion(&ctx->compl);
-
-	gsi_ctx->ch_dbg[chan_hdl].ch_db_stop++;
-	val = (((chan_hdl << GSI_EE_n_GSI_CH_CMD_CHID_SHFT) &
-			GSI_EE_n_GSI_CH_CMD_CHID_BMSK) |
-		((op << GSI_EE_n_GSI_CH_CMD_OPCODE_SHFT) &
-		 GSI_EE_n_GSI_CH_CMD_OPCODE_BMSK));
-	gsi_writel(val, gsi_ctx->base +
-			GSI_EE_n_GSI_CH_CMD_OFFS(gsi_ctx->per.ee));
-	res = wait_for_completion_timeout(&ctx->compl,
-			msecs_to_jiffies(GSI_STOP_CMD_TIMEOUT_MS));
-	if (res == 0) {
-		GSIERR("chan_hdl=%lu timed out\n", chan_hdl);
-		res = -GSI_STATUS_TIMED_OUT;
-		goto free_lock;
-	}
-
-	if (ctx->state != GSI_CHAN_STATE_STOPPED &&
-		ctx->state != GSI_CHAN_STATE_STOP_IN_PROC) {
-		GSIERR("chan=%lu unexpected state=%u\n", chan_hdl, ctx->state);
-		res = -GSI_STATUS_BAD_STATE;
-		goto free_lock;
-	}
-
-	if (ctx->state == GSI_CHAN_STATE_STOP_IN_PROC) {
-		GSIERR("chan=%lu busy try again\n", chan_hdl);
-		res = -GSI_STATUS_AGAIN;
-		goto free_lock;
-	}
-
-	res = GSI_STATUS_SUCCESS;
-
-free_lock:
-	mutex_unlock(&gsi_ctx->mlock);
-	return res;
-}
-EXPORT_SYMBOL(gsi_stop_db_channel);
-
-int gsi_reset_channel(unsigned long chan_hdl)
-{
-	enum gsi_ch_cmd_opcode op = GSI_CH_RESET;
-	int res;
-	uint32_t val;
-	struct gsi_chan_ctx *ctx;
-	bool reset_done = false;
-	uint32_t retry_cnt = 0;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (chan_hdl >= gsi_ctx->max_ch) {
-		GSIERR("bad params chan_hdl=%lu\n", chan_hdl);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	ctx = &gsi_ctx->chan[chan_hdl];
-
-	/*
-	 * In WDI3 case, if SAP enabled but no client connected,
-	 * GSI will be in allocated state. When SAP disabled,
-	 * gsi_reset_channel will be called and reset is needed.
-	 */
-	if (ctx->state != GSI_CHAN_STATE_STOPPED &&
-		ctx->state != GSI_CHAN_STATE_ALLOCATED) {
-		GSIERR("bad state %d\n", ctx->state);
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	mutex_lock(&gsi_ctx->mlock);
-
-reset:
-	reinit_completion(&ctx->compl);
-	gsi_ctx->ch_dbg[chan_hdl].ch_reset++;
-	val = (((chan_hdl << GSI_EE_n_GSI_CH_CMD_CHID_SHFT) &
-			GSI_EE_n_GSI_CH_CMD_CHID_BMSK) |
-		((op << GSI_EE_n_GSI_CH_CMD_OPCODE_SHFT) &
-		 GSI_EE_n_GSI_CH_CMD_OPCODE_BMSK));
-	gsi_writel(val, gsi_ctx->base +
-			GSI_EE_n_GSI_CH_CMD_OFFS(gsi_ctx->per.ee));
-	res = wait_for_completion_timeout(&ctx->compl, GSI_CMD_TIMEOUT);
-	if (res == 0) {
-		GSIERR("chan_hdl=%lu timed out\n", chan_hdl);
-		mutex_unlock(&gsi_ctx->mlock);
-		return -GSI_STATUS_TIMED_OUT;
-	}
-
-revrfy_chnlstate:
-	if (ctx->state != GSI_CHAN_STATE_ALLOCATED) {
-		GSIERR("chan_hdl=%lu unexpected state=%u\n", chan_hdl,
-				ctx->state);
-		/* GSI register update state not sync with gsi channel
-		 * context state not sync, need to wait for 1ms to sync.
-		 */
-		retry_cnt++;
-		if (retry_cnt <= GSI_CHNL_STATE_MAX_RETRYCNT) {
-			usleep_range(GSI_RESET_WA_MIN_SLEEP,
-				GSI_RESET_WA_MAX_SLEEP);
-			goto revrfy_chnlstate;
-		}
-		/*
-		 * Hardware returned incorrect state, unexpected
-		 * hardware state.
-		 */
-		GSI_ASSERT();
-	}
-
-	/* Hardware issue fixed from GSI 2.0 and no need for the WA */
-	if (gsi_ctx->per.ver >= GSI_VER_2_0)
-		reset_done = true;
-
-	/* workaround: reset GSI producers again */
-	if (ctx->props.dir == GSI_CHAN_DIR_FROM_GSI && !reset_done) {
-		usleep_range(GSI_RESET_WA_MIN_SLEEP, GSI_RESET_WA_MAX_SLEEP);
-		reset_done = true;
-		goto reset;
-	}
-
-	if (ctx->props.cleanup_cb)
-		gsi_cleanup_xfer_user_data(chan_hdl, ctx->props.cleanup_cb);
-
-	gsi_program_chan_ctx(&ctx->props, gsi_ctx->per.ee,
-			ctx->evtr ? ctx->evtr->id : GSI_NO_EVT_ERINDEX);
-	gsi_init_chan_ring(&ctx->props, &ctx->ring);
-
-	/* restore scratch */
-	__gsi_write_channel_scratch(chan_hdl, ctx->scratch);
-
-	mutex_unlock(&gsi_ctx->mlock);
-
-	return GSI_STATUS_SUCCESS;
-}
-EXPORT_SYMBOL(gsi_reset_channel);
-
-int gsi_dealloc_channel(unsigned long chan_hdl)
-{
-	enum gsi_ch_cmd_opcode op = GSI_CH_DE_ALLOC;
-	int res;
-	uint32_t val;
-	struct gsi_chan_ctx *ctx;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (chan_hdl >= gsi_ctx->max_ch) {
-		GSIERR("bad params chan_hdl=%lu\n", chan_hdl);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	ctx = &gsi_ctx->chan[chan_hdl];
-
-	if (ctx->state != GSI_CHAN_STATE_ALLOCATED) {
-		GSIERR("bad state %d\n", ctx->state);
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	/*In GSI_VER_2_2 version deallocation channel not supported*/
-	if (gsi_ctx->per.ver != GSI_VER_2_2) {
-		mutex_lock(&gsi_ctx->mlock);
-		reinit_completion(&ctx->compl);
-
-		gsi_ctx->ch_dbg[chan_hdl].ch_de_alloc++;
-		val = (((chan_hdl << GSI_EE_n_GSI_CH_CMD_CHID_SHFT) &
-					GSI_EE_n_GSI_CH_CMD_CHID_BMSK) |
-				((op << GSI_EE_n_GSI_CH_CMD_OPCODE_SHFT) &
-				 GSI_EE_n_GSI_CH_CMD_OPCODE_BMSK));
-		gsi_writel(val, gsi_ctx->base +
-				GSI_EE_n_GSI_CH_CMD_OFFS(gsi_ctx->per.ee));
-		res = wait_for_completion_timeout(&ctx->compl, GSI_CMD_TIMEOUT);
-		if (res == 0) {
-			GSIERR("chan_hdl=%lu timed out\n", chan_hdl);
-			mutex_unlock(&gsi_ctx->mlock);
-			return -GSI_STATUS_TIMED_OUT;
-		}
-		if (ctx->state != GSI_CHAN_STATE_NOT_ALLOCATED) {
-			GSIERR("chan_hdl=%lu unexpected state=%u\n", chan_hdl,
-					ctx->state);
-			/* Hardware returned incorrect value */
-			GSI_ASSERT();
-		}
-
-		mutex_unlock(&gsi_ctx->mlock);
-	} else {
-		mutex_lock(&gsi_ctx->mlock);
-		GSIDBG("In GSI_VER_2_2 channel deallocation not supported\n");
-		ctx->state = GSI_CHAN_STATE_NOT_ALLOCATED;
-		GSIDBG("chan_hdl=%lu Channel state = %u\n", chan_hdl,
-								ctx->state);
-		mutex_unlock(&gsi_ctx->mlock);
-	}
-	devm_kfree(gsi_ctx->dev, ctx->user_data);
-	ctx->allocated = false;
-	if (ctx->evtr && (ctx->props.prot != GSI_CHAN_PROT_GCI))
-		atomic_dec(&ctx->evtr->chan_ref_cnt);
-	atomic_dec(&gsi_ctx->num_chan);
-
-	return GSI_STATUS_SUCCESS;
-}
-EXPORT_SYMBOL(gsi_dealloc_channel);
-
-void gsi_update_ch_dp_stats(struct gsi_chan_ctx *ctx, uint16_t used)
-{
-	unsigned long now = jiffies_to_msecs(jiffies);
-	unsigned long elapsed;
-
-	if (used == 0) {
-		elapsed = now - ctx->stats.dp.last_timestamp;
-		if (ctx->stats.dp.empty_time < elapsed)
-			ctx->stats.dp.empty_time = elapsed;
-	}
-
-	if (used <= ctx->props.max_re_expected / 3)
-		++ctx->stats.dp.ch_below_lo;
-	else if (used <= 2 * ctx->props.max_re_expected / 3)
-		++ctx->stats.dp.ch_below_hi;
-	else
-		++ctx->stats.dp.ch_above_hi;
-	ctx->stats.dp.last_timestamp = now;
-}
-
-static void __gsi_query_channel_free_re(struct gsi_chan_ctx *ctx,
-		uint16_t *num_free_re)
-{
-	uint16_t start;
-	uint16_t end;
-	uint64_t rp;
-	int ee = gsi_ctx->per.ee;
-	uint16_t used;
-
-	WARN_ON(ctx->props.prot != GSI_CHAN_PROT_GPI);
-
-	if (!ctx->evtr) {
-		rp = gsi_readl(gsi_ctx->base +
-			GSI_EE_n_GSI_CH_k_CNTXT_4_OFFS(ctx->props.ch_id, ee));
-		rp |= ctx->ring.rp & 0xFFFFFFFF00000000;
-
-		ctx->ring.rp = rp;
-	} else {
-		rp = ctx->ring.rp_local;
-	}
-
-	start = gsi_find_idx_from_addr(&ctx->ring, rp);
-	end = gsi_find_idx_from_addr(&ctx->ring, ctx->ring.wp_local);
-
-	if (end >= start)
-		used = end - start;
-	else
-		used = ctx->ring.max_num_elem + 1 - (start - end);
-
-	*num_free_re = ctx->ring.max_num_elem - used;
-}
-
-int gsi_query_channel_info(unsigned long chan_hdl,
-		struct gsi_chan_info *info)
-{
-	struct gsi_chan_ctx *ctx;
-	spinlock_t *slock;
-	unsigned long flags;
-	uint64_t rp;
-	uint64_t wp;
-	int ee;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (chan_hdl >= gsi_ctx->max_ch || !info) {
-		GSIERR("bad params chan_hdl=%lu info=%pK\n", chan_hdl, info);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	ctx = &gsi_ctx->chan[chan_hdl];
-	if (ctx->evtr) {
-		slock = &ctx->evtr->ring.slock;
-		info->evt_valid = true;
-	} else {
-		slock = &ctx->ring.slock;
-		info->evt_valid = false;
-	}
-
-	spin_lock_irqsave(slock, flags);
-
-	ee = gsi_ctx->per.ee;
-	rp = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_CNTXT_4_OFFS(ctx->props.ch_id, ee));
-	rp |= ((uint64_t)gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_CNTXT_5_OFFS(ctx->props.ch_id, ee))) << 32;
-	ctx->ring.rp = rp;
-	info->rp = rp;
-
-	wp = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_CNTXT_6_OFFS(ctx->props.ch_id, ee));
-	wp |= ((uint64_t)gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_CNTXT_7_OFFS(ctx->props.ch_id, ee))) << 32;
-	ctx->ring.wp = wp;
-	info->wp = wp;
-
-	if (info->evt_valid) {
-		rp = gsi_readl(gsi_ctx->base +
-			GSI_EE_n_EV_CH_k_CNTXT_4_OFFS(ctx->evtr->id, ee));
-		rp |= ((uint64_t)gsi_readl(gsi_ctx->base +
-			GSI_EE_n_EV_CH_k_CNTXT_5_OFFS(ctx->evtr->id, ee)))
-			<< 32;
-		info->evt_rp = rp;
-
-		wp = gsi_readl(gsi_ctx->base +
-			GSI_EE_n_EV_CH_k_CNTXT_6_OFFS(ctx->evtr->id, ee));
-		wp |= ((uint64_t)gsi_readl(gsi_ctx->base +
-			GSI_EE_n_EV_CH_k_CNTXT_7_OFFS(ctx->evtr->id, ee)))
-			<< 32;
-		info->evt_wp = wp;
-	}
-
-	spin_unlock_irqrestore(slock, flags);
-
-	GSIDBG("ch=%lu RP=0x%llx WP=0x%llx ev_valid=%d ERP=0x%llx EWP=0x%llx\n",
-			chan_hdl, info->rp, info->wp,
-			info->evt_valid, info->evt_rp, info->evt_wp);
-
-	return GSI_STATUS_SUCCESS;
-}
-EXPORT_SYMBOL(gsi_query_channel_info);
-
-int gsi_is_channel_empty(unsigned long chan_hdl, bool *is_empty)
-{
-	struct gsi_chan_ctx *ctx;
-	spinlock_t *slock;
-	unsigned long flags;
-	uint64_t rp;
-	uint64_t wp;
-	uint64_t rp_local;
-	int ee;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (chan_hdl >= gsi_ctx->max_ch || !is_empty) {
-		GSIERR("bad params chan_hdl=%lu is_empty=%pK\n",
-				chan_hdl, is_empty);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	ctx = &gsi_ctx->chan[chan_hdl];
-	ee = gsi_ctx->per.ee;
-
-	if (ctx->props.prot != GSI_CHAN_PROT_GPI &&
-		ctx->props.prot != GSI_CHAN_PROT_GCI) {
-		GSIERR("op not supported for protocol %u\n", ctx->props.prot);
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	if (ctx->evtr)
-		slock = &ctx->evtr->ring.slock;
-	else
-		slock = &ctx->ring.slock;
-
-	spin_lock_irqsave(slock, flags);
-
-	if (ctx->props.dir == GSI_CHAN_DIR_FROM_GSI && ctx->evtr) {
-		rp = gsi_readl(gsi_ctx->base +
-			GSI_EE_n_EV_CH_k_CNTXT_4_OFFS(ctx->evtr->id, ee));
-		rp |= ctx->evtr->ring.rp & 0xFFFFFFFF00000000;
-		ctx->evtr->ring.rp = rp;
-
-		wp = gsi_readl(gsi_ctx->base +
-			GSI_EE_n_EV_CH_k_CNTXT_6_OFFS(ctx->evtr->id, ee));
-		wp |= ctx->evtr->ring.wp & 0xFFFFFFFF00000000;
-		ctx->evtr->ring.wp = wp;
-
-		rp_local = ctx->evtr->ring.rp_local;
-	} else {
-		rp = gsi_readl(gsi_ctx->base +
-			GSI_EE_n_GSI_CH_k_CNTXT_4_OFFS(ctx->props.ch_id, ee));
-		rp |= ctx->ring.rp & 0xFFFFFFFF00000000;
-		ctx->ring.rp = rp;
-
-		wp = gsi_readl(gsi_ctx->base +
-			GSI_EE_n_GSI_CH_k_CNTXT_6_OFFS(ctx->props.ch_id, ee));
-		wp |= ctx->ring.wp & 0xFFFFFFFF00000000;
-		ctx->ring.wp = wp;
-
-		rp_local = ctx->ring.rp_local;
-	}
-
-	if (ctx->props.dir == GSI_CHAN_DIR_FROM_GSI)
-		*is_empty = (rp_local == rp) ? true : false;
-	else
-		*is_empty = (wp == rp) ? true : false;
-
-	spin_unlock_irqrestore(slock, flags);
-
-	if (ctx->props.dir == GSI_CHAN_DIR_FROM_GSI && ctx->evtr)
-		GSIDBG("ch=%ld ev=%d RP=0x%llx WP=0x%llx RP_LOCAL=0x%llx\n",
-			chan_hdl, ctx->evtr->id, rp, wp, rp_local);
-	else
-		GSIDBG("ch=%lu RP=0x%llx WP=0x%llx RP_LOCAL=0x%llx\n",
-			chan_hdl, rp, wp, rp_local);
-
-	return GSI_STATUS_SUCCESS;
-}
-EXPORT_SYMBOL(gsi_is_channel_empty);
-
-int __gsi_get_gci_cookie(struct gsi_chan_ctx *ctx, uint16_t idx)
-{
-	int i;
-	int end;
-
-	if (!ctx->user_data[idx].valid) {
-		ctx->user_data[idx].valid = true;
-		return idx;
-	}
-
-	/*
-	 * at this point we need to find an "escape buffer" for the cookie
-	 * as the userdata in this spot is in use. This happens if the TRE at
-	 * idx is not completed yet and it is getting reused by a new TRE.
-	 */
-	ctx->stats.userdata_in_use++;
-	end = ctx->ring.max_num_elem + 1;
-	for (i = 0; i < GSI_VEID_MAX; i++) {
-		if (!ctx->user_data[end + i].valid) {
-			ctx->user_data[end + i].valid = true;
-			return end + i;
-		}
-	}
-
-	/* TODO: Increase escape buffer size if we hit this */
-	GSIERR("user_data is full\n");
-	return 0xFFFF;
-}
-
-int __gsi_populate_gci_tre(struct gsi_chan_ctx *ctx,
-	struct gsi_xfer_elem *xfer)
-{
-	struct gsi_gci_tre gci_tre;
-	struct gsi_gci_tre *tre_gci_ptr;
-	uint16_t idx;
-
-	memset(&gci_tre, 0, sizeof(gci_tre));
-	if (xfer->addr & 0xFFFFFF0000000000) {
-		GSIERR("chan_hdl=%u add too large=%llx\n",
-			ctx->props.ch_id, xfer->addr);
-		return -EINVAL;
-	}
-
-	if (xfer->type != GSI_XFER_ELEM_DATA) {
-		GSIERR("chan_hdl=%u bad RE type=%u\n", ctx->props.ch_id,
-			xfer->type);
-		return -EINVAL;
-	}
-
-	idx = gsi_find_idx_from_addr(&ctx->ring, ctx->ring.wp_local);
-	tre_gci_ptr = (struct gsi_gci_tre *)(ctx->ring.base_va +
-		idx * ctx->ring.elem_sz);
-
-	gci_tre.buffer_ptr = xfer->addr;
-	gci_tre.buf_len = xfer->len;
-	gci_tre.re_type = GSI_RE_COAL;
-	gci_tre.cookie = __gsi_get_gci_cookie(ctx, idx);
-	if (gci_tre.cookie > (ctx->ring.max_num_elem + GSI_VEID_MAX))
-		return -EPERM;
-
-	/* write the TRE to ring */
-	*tre_gci_ptr = gci_tre;
-	ctx->user_data[idx].p = xfer->xfer_user_data;
-
-	return 0;
-}
-
-int __gsi_populate_tre(struct gsi_chan_ctx *ctx,
-	struct gsi_xfer_elem *xfer)
-{
-	struct gsi_tre tre;
-	struct gsi_tre *tre_ptr;
-	uint16_t idx;
-
-	memset(&tre, 0, sizeof(tre));
-	tre.buffer_ptr = xfer->addr;
-	tre.buf_len = xfer->len;
-	if (xfer->type == GSI_XFER_ELEM_DATA) {
-		tre.re_type = GSI_RE_XFER;
-	} else if (xfer->type == GSI_XFER_ELEM_IMME_CMD) {
-		tre.re_type = GSI_RE_IMMD_CMD;
-	} else if (xfer->type == GSI_XFER_ELEM_NOP) {
-		tre.re_type = GSI_RE_NOP;
-	} else {
-		GSIERR("chan_hdl=%u bad RE type=%u\n", ctx->props.ch_id,
-			xfer->type);
-		return -EINVAL;
-	}
-
-	tre.bei = (xfer->flags & GSI_XFER_FLAG_BEI) ? 1 : 0;
-	tre.ieot = (xfer->flags & GSI_XFER_FLAG_EOT) ? 1 : 0;
-	tre.ieob = (xfer->flags & GSI_XFER_FLAG_EOB) ? 1 : 0;
-	tre.chain = (xfer->flags & GSI_XFER_FLAG_CHAIN) ? 1 : 0;
-
-	idx = gsi_find_idx_from_addr(&ctx->ring, ctx->ring.wp_local);
-	tre_ptr = (struct gsi_tre *)(ctx->ring.base_va +
-		idx * ctx->ring.elem_sz);
-
-	/* write the TRE to ring */
-	*tre_ptr = tre;
-	ctx->user_data[idx].valid = true;
-	ctx->user_data[idx].p = xfer->xfer_user_data;
-
-	return 0;
-}
-
-int gsi_queue_xfer(unsigned long chan_hdl, uint16_t num_xfers,
-		struct gsi_xfer_elem *xfer, bool ring_db)
-{
-	struct gsi_chan_ctx *ctx;
-	uint16_t free;
-	uint64_t wp_rollback;
-	int i;
-	spinlock_t *slock;
-	unsigned long flags;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (chan_hdl >= gsi_ctx->max_ch || (num_xfers && !xfer)) {
-		GSIERR("bad params chan_hdl=%lu num_xfers=%u xfer=%pK\n",
-				chan_hdl, num_xfers, xfer);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	ctx = &gsi_ctx->chan[chan_hdl];
-
-	if (ctx->props.prot != GSI_CHAN_PROT_GPI &&
-			ctx->props.prot != GSI_CHAN_PROT_GCI) {
-		GSIERR("op not supported for protocol %u\n", ctx->props.prot);
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	if (ctx->evtr)
-		slock = &ctx->evtr->ring.slock;
-	else
-		slock = &ctx->ring.slock;
-
-	spin_lock_irqsave(slock, flags);
-
-	/* allow only ring doorbell */
-	if (!num_xfers)
-		goto ring_doorbell;
-
-	/*
-	 * for GCI channels the responsibility is on the caller to make sure
-	 * there is enough room in the TRE.
-	 */
-	if (ctx->props.prot != GSI_CHAN_PROT_GCI) {
-		__gsi_query_channel_free_re(ctx, &free);
-		if (num_xfers > free) {
-			GSIERR("chan_hdl=%lu num_xfers=%u free=%u\n",
-				chan_hdl, num_xfers, free);
-			spin_unlock_irqrestore(slock, flags);
-			return -GSI_STATUS_RING_INSUFFICIENT_SPACE;
-		}
-	}
-
-	wp_rollback = ctx->ring.wp_local;
-	for (i = 0; i < num_xfers; i++) {
-		if (ctx->props.prot == GSI_CHAN_PROT_GCI) {
-			if (__gsi_populate_gci_tre(ctx, &xfer[i]))
-				break;
-		} else {
-			if (__gsi_populate_tre(ctx, &xfer[i]))
-				break;
-		}
-		gsi_incr_ring_wp(&ctx->ring);
-	}
-
-	if (i != num_xfers) {
-		/* reject all the xfers */
-		ctx->ring.wp_local = wp_rollback;
-		spin_unlock_irqrestore(slock, flags);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	ctx->stats.queued += num_xfers;
-
-ring_doorbell:
-	if (ring_db) {
-		/* ensure TRE is set before ringing doorbell */
-		wmb();
-		gsi_ring_chan_doorbell(ctx);
-	}
-
-	spin_unlock_irqrestore(slock, flags);
-
-	return GSI_STATUS_SUCCESS;
-}
-EXPORT_SYMBOL(gsi_queue_xfer);
-
-int gsi_start_xfer(unsigned long chan_hdl)
-{
-	struct gsi_chan_ctx *ctx;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (chan_hdl >= gsi_ctx->max_ch) {
-		GSIERR("bad params chan_hdl=%lu\n", chan_hdl);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	ctx = &gsi_ctx->chan[chan_hdl];
-
-	if (ctx->props.prot != GSI_CHAN_PROT_GPI &&
-		ctx->props.prot != GSI_CHAN_PROT_GCI) {
-		GSIERR("op not supported for protocol %u\n", ctx->props.prot);
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	if (ctx->state == GSI_CHAN_STATE_NOT_ALLOCATED) {
-		GSIERR("bad state %d\n", ctx->state);
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	if (ctx->ring.wp == ctx->ring.wp_local)
-		return GSI_STATUS_SUCCESS;
-
-	gsi_ring_chan_doorbell(ctx);
-
-	return GSI_STATUS_SUCCESS;
-};
-EXPORT_SYMBOL(gsi_start_xfer);
-
-int gsi_poll_channel(unsigned long chan_hdl,
-		struct gsi_chan_xfer_notify *notify)
-{
-	int unused_var;
-
-	return gsi_poll_n_channel(chan_hdl, notify, 1, &unused_var);
-}
-EXPORT_SYMBOL(gsi_poll_channel);
-
-int gsi_poll_n_channel(unsigned long chan_hdl,
-		struct gsi_chan_xfer_notify *notify,
-		int expected_num, int *actual_num)
-{
-	struct gsi_chan_ctx *ctx;
-	uint64_t rp;
-	int ee;
-	int i;
-	unsigned long flags;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (chan_hdl >= gsi_ctx->max_ch || !notify ||
-	    !actual_num || expected_num <= 0) {
-		GSIERR("bad params chan_hdl=%lu notify=%pK\n",
-			chan_hdl, notify);
-		GSIERR("actual_num=%pK expected_num=%d\n",
-			actual_num, expected_num);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	ctx = &gsi_ctx->chan[chan_hdl];
-	ee = gsi_ctx->per.ee;
-
-	if (ctx->props.prot != GSI_CHAN_PROT_GPI &&
-		ctx->props.prot != GSI_CHAN_PROT_GCI) {
-		GSIERR("op not supported for protocol %u\n", ctx->props.prot);
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	if (!ctx->evtr) {
-		GSIERR("no event ring associated chan_hdl=%lu\n", chan_hdl);
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	spin_lock_irqsave(&ctx->evtr->ring.slock, flags);
-	if (ctx->evtr->ring.rp == ctx->evtr->ring.rp_local) {
-		/* update rp to see of we have anything new to process */
-		rp = gsi_readl(gsi_ctx->base +
-			GSI_EE_n_EV_CH_k_CNTXT_4_OFFS(ctx->evtr->id, ee));
-		rp |= ctx->ring.rp & 0xFFFFFFFF00000000ULL;
-
-		ctx->evtr->ring.rp = rp;
-		/* read gsi event ring rp again if last read is empty */
-		if (rp == ctx->evtr->ring.rp_local) {
-			/* event ring is empty */
-			gsi_writel(1 << ctx->evtr->id, gsi_ctx->base +
-				GSI_EE_n_CNTXT_SRC_IEOB_IRQ_CLR_OFFS(ee));
-			/* do another read to close a small window */
-			__iowmb();
-			rp = gsi_readl(gsi_ctx->base +
-				GSI_EE_n_EV_CH_k_CNTXT_4_OFFS(
-				ctx->evtr->id, ee));
-			rp |= ctx->ring.rp & 0xFFFFFFFF00000000ULL;
-			ctx->evtr->ring.rp = rp;
-			if (rp == ctx->evtr->ring.rp_local) {
-				spin_unlock_irqrestore(
-					&ctx->evtr->ring.slock,
-					flags);
-				ctx->stats.poll_empty++;
-				return GSI_STATUS_POLL_EMPTY;
-			}
-		}
-	}
-
-	*actual_num = gsi_get_complete_num(&ctx->evtr->ring,
-			ctx->evtr->ring.rp_local, ctx->evtr->ring.rp);
-
-	if (*actual_num > expected_num)
-		*actual_num = expected_num;
-
-	for (i = 0; i < *actual_num; i++)
-		gsi_process_evt_re(ctx->evtr, notify + i, false);
-
-	spin_unlock_irqrestore(&ctx->evtr->ring.slock, flags);
-	ctx->stats.poll_ok++;
-
-	return GSI_STATUS_SUCCESS;
-}
-EXPORT_SYMBOL(gsi_poll_n_channel);
-
-int gsi_config_channel_mode(unsigned long chan_hdl, enum gsi_chan_mode mode)
-{
-	struct gsi_chan_ctx *ctx;
-	enum gsi_chan_mode curr;
-	unsigned long flags;
-	enum gsi_chan_mode chan_mode;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (chan_hdl >= gsi_ctx->max_ch) {
-		GSIERR("bad params chan_hdl=%lu mode=%u\n", chan_hdl, mode);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	ctx = &gsi_ctx->chan[chan_hdl];
-
-	if (ctx->props.prot != GSI_CHAN_PROT_GPI &&
-		ctx->props.prot != GSI_CHAN_PROT_GCI) {
-		GSIERR("op not supported for protocol %u\n", ctx->props.prot);
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	if (!ctx->evtr || !ctx->evtr->props.exclusive) {
-		GSIERR("cannot configure mode on chan_hdl=%lu\n",
-				chan_hdl);
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	if (atomic_read(&ctx->poll_mode))
-		curr = GSI_CHAN_MODE_POLL;
-	else
-		curr = GSI_CHAN_MODE_CALLBACK;
-
-	if (mode == curr) {
-		GSIERR("already in requested mode %u chan_hdl=%lu\n",
-				curr, chan_hdl);
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-	spin_lock_irqsave(&gsi_ctx->slock, flags);
-	if (curr == GSI_CHAN_MODE_CALLBACK &&
-			mode == GSI_CHAN_MODE_POLL) {
-		__gsi_config_ieob_irq(gsi_ctx->per.ee, 1 << ctx->evtr->id, 0);
-		gsi_writel(1 << ctx->evtr->id, gsi_ctx->base +
-			GSI_EE_n_CNTXT_SRC_IEOB_IRQ_CLR_OFFS(gsi_ctx->per.ee));
-		atomic_set(&ctx->poll_mode, mode);
-		if (ctx->props.prot == GSI_CHAN_PROT_GCI)
-			atomic_set(&ctx->evtr->chan->poll_mode, mode);
-		GSIDBG("set gsi_ctx evtr_id %d to %d mode\n",
-			ctx->evtr->id, mode);
-		ctx->stats.callback_to_poll++;
-	}
-
-	if (curr == GSI_CHAN_MODE_POLL &&
-			mode == GSI_CHAN_MODE_CALLBACK) {
-		atomic_set(&ctx->poll_mode, mode);
-		if (ctx->props.prot == GSI_CHAN_PROT_GCI)
-			atomic_set(&ctx->evtr->chan->poll_mode, mode);
-		__gsi_config_ieob_irq(gsi_ctx->per.ee, 1 << ctx->evtr->id, ~0);
-		GSIDBG("set gsi_ctx evtr_id %d to %d mode\n",
-			ctx->evtr->id, mode);
-
-		/*
-		 * In GSI 2.2 and 2.5 there is a limitation that can lead
-		 * to losing an interrupt. For these versions an
-		 * explicit check is needed after enabling the interrupt
-		 */
-		if (gsi_ctx->per.ver == GSI_VER_2_2 ||
-		    gsi_ctx->per.ver == GSI_VER_2_5) {
-			u32 src = gsi_readl(gsi_ctx->base +
-				GSI_EE_n_CNTXT_SRC_IEOB_IRQ_OFFS(
-					gsi_ctx->per.ee));
-			if (src & (1 << ctx->evtr->id)) {
-				__gsi_config_ieob_irq(
-					gsi_ctx->per.ee, 1 << ctx->evtr->id, 0);
-				gsi_writel(1 << ctx->evtr->id, gsi_ctx->base +
-					GSI_EE_n_CNTXT_SRC_IEOB_IRQ_CLR_OFFS(
-							gsi_ctx->per.ee));
-				spin_unlock_irqrestore(&gsi_ctx->slock, flags);
-				spin_lock_irqsave(&ctx->evtr->ring.slock,
-									flags);
-				chan_mode = atomic_xchg(&ctx->poll_mode,
-						GSI_CHAN_MODE_POLL);
-				spin_unlock_irqrestore(
-					&ctx->evtr->ring.slock, flags);
-				ctx->stats.poll_pending_irq++;
-				GSIDBG("In IEOB WA pnd cnt = %ld prvmode = %d\n",
-						ctx->stats.poll_pending_irq,
-						chan_mode);
-				if (chan_mode == GSI_CHAN_MODE_POLL)
-					return GSI_STATUS_SUCCESS;
-				else
-					return -GSI_STATUS_PENDING_IRQ;
-			}
-		}
-		ctx->stats.poll_to_callback++;
-	}
-	spin_unlock_irqrestore(&gsi_ctx->slock, flags);
-	return GSI_STATUS_SUCCESS;
-}
-EXPORT_SYMBOL(gsi_config_channel_mode);
-
-int gsi_get_channel_cfg(unsigned long chan_hdl, struct gsi_chan_props *props,
-		union gsi_channel_scratch *scr)
-{
-	struct gsi_chan_ctx *ctx;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (!props || !scr) {
-		GSIERR("bad params props=%pK scr=%pK\n", props, scr);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (chan_hdl >= gsi_ctx->max_ch) {
-		GSIERR("bad params chan_hdl=%lu\n", chan_hdl);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	ctx = &gsi_ctx->chan[chan_hdl];
-
-	if (ctx->state == GSI_CHAN_STATE_NOT_ALLOCATED) {
-		GSIERR("bad state %d\n", ctx->state);
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	mutex_lock(&ctx->mlock);
-	*props = ctx->props;
-	*scr = ctx->scratch;
-	mutex_unlock(&ctx->mlock);
-
-	return GSI_STATUS_SUCCESS;
-}
-EXPORT_SYMBOL(gsi_get_channel_cfg);
-
-int gsi_set_channel_cfg(unsigned long chan_hdl, struct gsi_chan_props *props,
-		union gsi_channel_scratch *scr)
-{
-	struct gsi_chan_ctx *ctx;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (!props || gsi_validate_channel_props(props)) {
-		GSIERR("bad params props=%pK\n", props);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (chan_hdl >= gsi_ctx->max_ch) {
-		GSIERR("bad params chan_hdl=%lu\n", chan_hdl);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	ctx = &gsi_ctx->chan[chan_hdl];
-
-	if (ctx->state != GSI_CHAN_STATE_ALLOCATED) {
-		GSIERR("bad state %d\n", ctx->state);
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	if (ctx->props.ch_id != props->ch_id ||
-		ctx->props.evt_ring_hdl != props->evt_ring_hdl) {
-		GSIERR("changing immutable fields not supported\n");
-		return -GSI_STATUS_UNSUPPORTED_OP;
-	}
-
-	mutex_lock(&ctx->mlock);
-	ctx->props = *props;
-	if (scr)
-		ctx->scratch = *scr;
-	gsi_program_chan_ctx(&ctx->props, gsi_ctx->per.ee,
-			ctx->evtr ? ctx->evtr->id : GSI_NO_EVT_ERINDEX);
-	gsi_init_chan_ring(&ctx->props, &ctx->ring);
-
-	/* restore scratch */
-	__gsi_write_channel_scratch(chan_hdl, ctx->scratch);
-	mutex_unlock(&ctx->mlock);
-
-	return GSI_STATUS_SUCCESS;
-}
-EXPORT_SYMBOL(gsi_set_channel_cfg);
-
-static void gsi_configure_ieps(void *base, enum gsi_ver ver)
-{
-	void __iomem *gsi_base = base;
-
-	gsi_writel(1, gsi_base + GSI_GSI_IRAM_PTR_CH_CMD_OFFS);
-	gsi_writel(2, gsi_base + GSI_GSI_IRAM_PTR_CH_DB_OFFS);
-	gsi_writel(3, gsi_base + GSI_GSI_IRAM_PTR_CH_DIS_COMP_OFFS);
-	gsi_writel(4, gsi_base + GSI_GSI_IRAM_PTR_CH_EMPTY_OFFS);
-	gsi_writel(5, gsi_base + GSI_GSI_IRAM_PTR_EE_GENERIC_CMD_OFFS);
-	gsi_writel(6, gsi_base + GSI_GSI_IRAM_PTR_EVENT_GEN_COMP_OFFS);
-	gsi_writel(7, gsi_base + GSI_GSI_IRAM_PTR_INT_MOD_STOPPED_OFFS);
-	gsi_writel(8, gsi_base + GSI_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_0_OFFS);
-	gsi_writel(9, gsi_base + GSI_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_2_OFFS);
-	gsi_writel(10, gsi_base + GSI_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_1_OFFS);
-	gsi_writel(11, gsi_base + GSI_GSI_IRAM_PTR_NEW_RE_OFFS);
-	gsi_writel(12, gsi_base + GSI_GSI_IRAM_PTR_READ_ENG_COMP_OFFS);
-	gsi_writel(13, gsi_base + GSI_GSI_IRAM_PTR_TIMER_EXPIRED_OFFS);
-	gsi_writel(14, gsi_base + GSI_GSI_IRAM_PTR_EV_DB_OFFS);
-	gsi_writel(15, gsi_base + GSI_GSI_IRAM_PTR_UC_GP_INT_OFFS);
-	gsi_writel(16, gsi_base + GSI_GSI_IRAM_PTR_WRITE_ENG_COMP_OFFS);
-
-	if (ver >= GSI_VER_2_5)
-		gsi_writel(17,
-			gsi_base + GSI_V2_5_GSI_IRAM_PTR_TLV_CH_NOT_FULL_OFFS);
-}
-
-static void gsi_configure_bck_prs_matrix(void *base)
-{
-	void __iomem *gsi_base = (void __iomem *) base;
-
-	/*
-	 * For now, these are default values. In the future, GSI FW image will
-	 * produce optimized back-pressure values based on the FW image.
-	 */
-	gsi_writel(0xfffffffe,
-		gsi_base + GSI_IC_DISABLE_CHNL_BCK_PRS_LSB_OFFS);
-	gsi_writel(0xffffffff,
-		gsi_base + GSI_IC_DISABLE_CHNL_BCK_PRS_MSB_OFFS);
-	gsi_writel(0xffffffbf, gsi_base + GSI_IC_GEN_EVNT_BCK_PRS_LSB_OFFS);
-	gsi_writel(0xffffffff, gsi_base + GSI_IC_GEN_EVNT_BCK_PRS_MSB_OFFS);
-	gsi_writel(0xffffefff, gsi_base + GSI_IC_GEN_INT_BCK_PRS_LSB_OFFS);
-	gsi_writel(0xffffffff, gsi_base + GSI_IC_GEN_INT_BCK_PRS_MSB_OFFS);
-	gsi_writel(0xffffefff,
-		gsi_base + GSI_IC_STOP_INT_MOD_BCK_PRS_LSB_OFFS);
-	gsi_writel(0xffffffff,
-		gsi_base + GSI_IC_STOP_INT_MOD_BCK_PRS_MSB_OFFS);
-	gsi_writel(0x00000000,
-		gsi_base + GSI_IC_PROCESS_DESC_BCK_PRS_LSB_OFFS);
-	gsi_writel(0x00000000,
-		gsi_base + GSI_IC_PROCESS_DESC_BCK_PRS_MSB_OFFS);
-	gsi_writel(0xf9ffffff, gsi_base + GSI_IC_TLV_STOP_BCK_PRS_LSB_OFFS);
-	gsi_writel(0xffffffff, gsi_base + GSI_IC_TLV_STOP_BCK_PRS_MSB_OFFS);
-	gsi_writel(0xf9ffffff, gsi_base + GSI_IC_TLV_RESET_BCK_PRS_LSB_OFFS);
-	gsi_writel(0xffffffff, gsi_base + GSI_IC_TLV_RESET_BCK_PRS_MSB_OFFS);
-	gsi_writel(0xffffffff, gsi_base + GSI_IC_RGSTR_TIMER_BCK_PRS_LSB_OFFS);
-	gsi_writel(0xfffffffe, gsi_base + GSI_IC_RGSTR_TIMER_BCK_PRS_MSB_OFFS);
-	gsi_writel(0xffffffff, gsi_base + GSI_IC_READ_BCK_PRS_LSB_OFFS);
-	gsi_writel(0xffffefff, gsi_base + GSI_IC_READ_BCK_PRS_MSB_OFFS);
-	gsi_writel(0xffffffff, gsi_base + GSI_IC_WRITE_BCK_PRS_LSB_OFFS);
-	gsi_writel(0xffffdfff, gsi_base + GSI_IC_WRITE_BCK_PRS_MSB_OFFS);
-	gsi_writel(0xffffffff,
-		gsi_base + GSI_IC_UCONTROLLER_GPR_BCK_PRS_LSB_OFFS);
-	gsi_writel(0xff03ffff,
-		gsi_base + GSI_IC_UCONTROLLER_GPR_BCK_PRS_MSB_OFFS);
-}
-
-int gsi_configure_regs(phys_addr_t per_base_addr, enum gsi_ver ver)
-{
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (!gsi_ctx->base) {
-		GSIERR("access to GSI HW has not been mapped\n");
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (ver <= GSI_VER_ERR || ver >= GSI_VER_MAX) {
-		GSIERR("Incorrect version %d\n", ver);
-		return -GSI_STATUS_ERROR;
-	}
-
-	gsi_writel(0, gsi_ctx->base + GSI_GSI_PERIPH_BASE_ADDR_MSB_OFFS);
-	gsi_writel(per_base_addr,
-			gsi_ctx->base + GSI_GSI_PERIPH_BASE_ADDR_LSB_OFFS);
-	gsi_configure_bck_prs_matrix((void *)gsi_ctx->base);
-	gsi_configure_ieps(gsi_ctx->base, ver);
-
-	return 0;
-}
-EXPORT_SYMBOL(gsi_configure_regs);
-
-int gsi_enable_fw(phys_addr_t gsi_base_addr, u32 gsi_size, enum gsi_ver ver)
-{
-	void __iomem *gsi_base;
-	uint32_t value;
-
-	if (ver <= GSI_VER_ERR || ver >= GSI_VER_MAX) {
-		GSIERR("Incorrect version %d\n", ver);
-		return -GSI_STATUS_ERROR;
-	}
-
-	gsi_base = ioremap_nocache(gsi_base_addr, gsi_size);
-	if (!gsi_base) {
-		GSIERR("ioremap failed\n");
-		return -GSI_STATUS_RES_ALLOC_FAILURE;
-	}
-
-	/* Enable the MCS and set to x2 clocks */
-	if (ver >= GSI_VER_1_2) {
-		value = ((1 << GSI_GSI_MCS_CFG_MCS_ENABLE_SHFT) &
-				GSI_GSI_MCS_CFG_MCS_ENABLE_BMSK);
-		gsi_writel(value, gsi_base + GSI_GSI_MCS_CFG_OFFS);
-
-		value = (((1 << GSI_GSI_CFG_GSI_ENABLE_SHFT) &
-				GSI_GSI_CFG_GSI_ENABLE_BMSK) |
-			((0 << GSI_GSI_CFG_MCS_ENABLE_SHFT) &
-				GSI_GSI_CFG_MCS_ENABLE_BMSK) |
-			((1 << GSI_GSI_CFG_DOUBLE_MCS_CLK_FREQ_SHFT) &
-				GSI_GSI_CFG_DOUBLE_MCS_CLK_FREQ_BMSK) |
-			((0 << GSI_GSI_CFG_UC_IS_MCS_SHFT) &
-				GSI_GSI_CFG_UC_IS_MCS_BMSK) |
-			((0 << GSI_GSI_CFG_GSI_PWR_CLPS_SHFT) &
-				GSI_GSI_CFG_GSI_PWR_CLPS_BMSK) |
-			((0 << GSI_GSI_CFG_BP_MTRIX_DISABLE_SHFT) &
-				GSI_GSI_CFG_BP_MTRIX_DISABLE_BMSK));
-	} else {
-		value = (((1 << GSI_GSI_CFG_GSI_ENABLE_SHFT) &
-				GSI_GSI_CFG_GSI_ENABLE_BMSK) |
-			((1 << GSI_GSI_CFG_MCS_ENABLE_SHFT) &
-				GSI_GSI_CFG_MCS_ENABLE_BMSK) |
-			((1 << GSI_GSI_CFG_DOUBLE_MCS_CLK_FREQ_SHFT) &
-				GSI_GSI_CFG_DOUBLE_MCS_CLK_FREQ_BMSK) |
-			((0 << GSI_GSI_CFG_UC_IS_MCS_SHFT) &
-				GSI_GSI_CFG_UC_IS_MCS_BMSK));
-	}
-
-	/* GSI frequency is peripheral frequency divided by 3 (2+1) */
-	if (ver >= GSI_VER_2_5)
-		value |= ((2 << GSI_V2_5_GSI_CFG_SLEEP_CLK_DIV_SHFT) &
-			GSI_V2_5_GSI_CFG_SLEEP_CLK_DIV_BMSK);
-	gsi_writel(value, gsi_base + GSI_GSI_CFG_OFFS);
-	iounmap(gsi_base);
-
-	return 0;
-
-}
-EXPORT_SYMBOL(gsi_enable_fw);
-
-void gsi_get_inst_ram_offset_and_size(unsigned long *base_offset,
-		unsigned long *size, enum gsi_ver ver)
-{
-	unsigned long maxn;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return;
-	}
-
-	switch (ver) {
-	case GSI_VER_1_0:
-	case GSI_VER_1_2:
-	case GSI_VER_1_3:
-		maxn = GSI_GSI_INST_RAM_n_MAXn;
-		break;
-	case GSI_VER_2_0:
-		maxn = GSI_V2_0_GSI_INST_RAM_n_MAXn;
-		break;
-	case GSI_VER_2_2:
-		maxn = GSI_V2_2_GSI_INST_RAM_n_MAXn;
-		break;
-	case GSI_VER_2_5:
-		maxn = GSI_V2_5_GSI_INST_RAM_n_MAXn;
-		break;
-	case GSI_VER_2_7:
-		maxn = GSI_V2_7_GSI_INST_RAM_n_MAXn;
-		break;
-	case GSI_VER_ERR:
-	case GSI_VER_MAX:
-	default:
-		GSIERR("GSI version is not supported %d\n", ver);
-		WARN_ON(1);
-		return;
-	}
-	if (size)
-		*size = GSI_GSI_INST_RAM_n_WORD_SZ * (maxn + 1);
-
-	if (base_offset) {
-		if (ver < GSI_VER_2_5)
-			*base_offset = GSI_GSI_INST_RAM_n_OFFS(0);
-		else
-			*base_offset = GSI_V2_5_GSI_INST_RAM_n_OFFS(0);
-	}
-}
-EXPORT_SYMBOL(gsi_get_inst_ram_offset_and_size);
-
-int gsi_halt_channel_ee(unsigned int chan_idx, unsigned int ee, int *code)
-{
-	enum gsi_generic_ee_cmd_opcode op = GSI_GEN_EE_CMD_HALT_CHANNEL;
-	uint32_t val;
-	int res;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (chan_idx >= gsi_ctx->max_ch || !code) {
-		GSIERR("bad params chan_idx=%d\n", chan_idx);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	mutex_lock(&gsi_ctx->mlock);
-	reinit_completion(&gsi_ctx->gen_ee_cmd_compl);
-
-	/* invalidate the response */
-	gsi_ctx->scratch.word0.val = gsi_readl(gsi_ctx->base +
-			GSI_EE_n_CNTXT_SCRATCH_0_OFFS(gsi_ctx->per.ee));
-	gsi_ctx->scratch.word0.s.generic_ee_cmd_return_code = 0;
-	gsi_writel(gsi_ctx->scratch.word0.val, gsi_ctx->base +
-			GSI_EE_n_CNTXT_SCRATCH_0_OFFS(gsi_ctx->per.ee));
-
-	gsi_ctx->gen_ee_cmd_dbg.halt_channel++;
-	val = (((op << GSI_EE_n_GSI_EE_GENERIC_CMD_OPCODE_SHFT) &
-		GSI_EE_n_GSI_EE_GENERIC_CMD_OPCODE_BMSK) |
-		((chan_idx << GSI_EE_n_GSI_EE_GENERIC_CMD_VIRT_CHAN_IDX_SHFT) &
-			GSI_EE_n_GSI_EE_GENERIC_CMD_VIRT_CHAN_IDX_BMSK) |
-		((ee << GSI_EE_n_GSI_EE_GENERIC_CMD_EE_SHFT) &
-			GSI_EE_n_GSI_EE_GENERIC_CMD_EE_BMSK));
-	gsi_writel(val, gsi_ctx->base +
-		GSI_EE_n_GSI_EE_GENERIC_CMD_OFFS(gsi_ctx->per.ee));
-
-	res = wait_for_completion_timeout(&gsi_ctx->gen_ee_cmd_compl,
-		msecs_to_jiffies(GSI_CMD_TIMEOUT));
-	if (res == 0) {
-		GSIERR("chan_idx=%u ee=%u timed out\n", chan_idx, ee);
-		res = -GSI_STATUS_TIMED_OUT;
-		goto free_lock;
-	}
-
-	gsi_ctx->scratch.word0.val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_CNTXT_SCRATCH_0_OFFS(gsi_ctx->per.ee));
-	if (gsi_ctx->scratch.word0.s.generic_ee_cmd_return_code ==
-		GSI_GEN_EE_CMD_RETURN_CODE_RETRY) {
-		GSIDBG("chan_idx=%u ee=%u busy try again\n", chan_idx, ee);
-		*code = GSI_GEN_EE_CMD_RETURN_CODE_RETRY;
-		res = -GSI_STATUS_AGAIN;
-		goto free_lock;
-	}
-	if (gsi_ctx->scratch.word0.s.generic_ee_cmd_return_code == 0) {
-		GSIERR("No response received\n");
-		res = -GSI_STATUS_ERROR;
-		goto free_lock;
-	}
-
-	res = GSI_STATUS_SUCCESS;
-	*code = gsi_ctx->scratch.word0.s.generic_ee_cmd_return_code;
-free_lock:
-	mutex_unlock(&gsi_ctx->mlock);
-
-	return res;
-}
-EXPORT_SYMBOL(gsi_halt_channel_ee);
-
-int gsi_alloc_channel_ee(unsigned int chan_idx, unsigned int ee, int *code)
-{
-	enum gsi_generic_ee_cmd_opcode op = GSI_GEN_EE_CMD_ALLOC_CHANNEL;
-	struct gsi_chan_ctx *ctx;
-	uint32_t val;
-	int res;
-
-	if (chan_idx >= gsi_ctx->max_ch || !code) {
-		GSIERR("bad params chan_idx=%d\n", chan_idx);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (ee == 0)
-		return gsi_alloc_ap_channel(chan_idx);
-
-	mutex_lock(&gsi_ctx->mlock);
-	reinit_completion(&gsi_ctx->gen_ee_cmd_compl);
-
-	/* invalidate the response */
-	gsi_ctx->scratch.word0.val = gsi_readl(gsi_ctx->base +
-			GSI_EE_n_CNTXT_SCRATCH_0_OFFS(gsi_ctx->per.ee));
-	gsi_ctx->scratch.word0.s.generic_ee_cmd_return_code = 0;
-	gsi_writel(gsi_ctx->scratch.word0.val, gsi_ctx->base +
-			GSI_EE_n_CNTXT_SCRATCH_0_OFFS(gsi_ctx->per.ee));
-
-	val = (((op << GSI_EE_n_GSI_EE_GENERIC_CMD_OPCODE_SHFT) &
-		GSI_EE_n_GSI_EE_GENERIC_CMD_OPCODE_BMSK) |
-		((chan_idx << GSI_EE_n_GSI_EE_GENERIC_CMD_VIRT_CHAN_IDX_SHFT) &
-			GSI_EE_n_GSI_EE_GENERIC_CMD_VIRT_CHAN_IDX_BMSK) |
-		((ee << GSI_EE_n_GSI_EE_GENERIC_CMD_EE_SHFT) &
-			GSI_EE_n_GSI_EE_GENERIC_CMD_EE_BMSK));
-	gsi_writel(val, gsi_ctx->base +
-		GSI_EE_n_GSI_EE_GENERIC_CMD_OFFS(gsi_ctx->per.ee));
-
-	res = wait_for_completion_timeout(&gsi_ctx->gen_ee_cmd_compl,
-		msecs_to_jiffies(GSI_CMD_TIMEOUT));
-	if (res == 0) {
-		GSIERR("chan_idx=%u ee=%u timed out\n", chan_idx, ee);
-		res = -GSI_STATUS_TIMED_OUT;
-		goto free_lock;
-	}
-
-	gsi_ctx->scratch.word0.val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_CNTXT_SCRATCH_0_OFFS(gsi_ctx->per.ee));
-	if (gsi_ctx->scratch.word0.s.generic_ee_cmd_return_code ==
-		GSI_GEN_EE_CMD_RETURN_CODE_OUT_OF_RESOURCES) {
-		GSIDBG("chan_idx=%u ee=%u out of resources\n", chan_idx, ee);
-		*code = GSI_GEN_EE_CMD_RETURN_CODE_OUT_OF_RESOURCES;
-		res = -GSI_STATUS_RES_ALLOC_FAILURE;
-		goto free_lock;
-	}
-	if (gsi_ctx->scratch.word0.s.generic_ee_cmd_return_code == 0) {
-		GSIERR("No response received\n");
-		res = -GSI_STATUS_ERROR;
-		goto free_lock;
-	}
-	if (ee == 0) {
-		ctx = &gsi_ctx->chan[chan_idx];
-		gsi_ctx->ch_dbg[chan_idx].ch_allocate++;
-	}
-	res = GSI_STATUS_SUCCESS;
-	*code = gsi_ctx->scratch.word0.s.generic_ee_cmd_return_code;
-free_lock:
-	mutex_unlock(&gsi_ctx->mlock);
-
-	return res;
-}
-EXPORT_SYMBOL(gsi_alloc_channel_ee);
-
-int gsi_map_virtual_ch_to_per_ep(u32 ee, u32 chan_num, u32 per_ep_index)
-{
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return -GSI_STATUS_NODEV;
-	}
-
-	if (!gsi_ctx->base) {
-		GSIERR("access to GSI HW has not been mapped\n");
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	gsi_writel(per_ep_index,
-		gsi_ctx->base +
-		GSI_V2_5_GSI_MAP_EE_n_CH_k_VP_TABLE_OFFS(chan_num, ee));
-
-	return 0;
-}
-EXPORT_SYMBOL(gsi_map_virtual_ch_to_per_ep);
-
-void gsi_wdi3_write_evt_ring_db(unsigned long evt_ring_hdl,
-	uint32_t db_addr_low, uint32_t db_addr_high)
-{
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return;
-	}
-
-	gsi_writel(db_addr_low, gsi_ctx->base +
-		GSI_EE_n_EV_CH_k_CNTXT_12_OFFS(evt_ring_hdl, gsi_ctx->per.ee));
-
-	gsi_writel(db_addr_high, gsi_ctx->base +
-		GSI_EE_n_EV_CH_k_CNTXT_13_OFFS(evt_ring_hdl, gsi_ctx->per.ee));
-}
-EXPORT_SYMBOL(gsi_wdi3_write_evt_ring_db);
-
-void gsi_wdi3_dump_register(unsigned long chan_hdl)
-{
-	uint32_t val;
-
-	if (!gsi_ctx) {
-		pr_err("%s:%d gsi context not allocated\n", __func__, __LINE__);
-		return;
-	}
-	GSIDBG("reg dump ch id %ld\n", chan_hdl);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_CNTXT_0_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-	GSIDBG("GSI_EE_n_GSI_CH_k_CNTXT_0_OFFS 0x%x\n", val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_CNTXT_1_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-	GSIDBG("GSI_EE_n_GSI_CH_k_CNTXT_1_OFFS 0x%x\n", val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_CNTXT_2_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-	GSIDBG("GSI_EE_n_GSI_CH_k_CNTXT_2_OFFS 0x%x\n", val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_CNTXT_3_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-	GSIDBG("GSI_EE_n_GSI_CH_k_CNTXT_3_OFFS 0x%x\n", val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_CNTXT_4_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-	GSIDBG("GSI_EE_n_GSI_CH_k_CNTXT_4_OFFS 0x%x\n", val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_CNTXT_5_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-	GSIDBG("GSI_EE_n_GSI_CH_k_CNTXT_5_OFFS 0x%x\n", val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_CNTXT_6_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-	GSIDBG("GSI_EE_n_GSI_CH_k_CNTXT_6_OFFS 0x%x\n", val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_CNTXT_7_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-	GSIDBG("GSI_EE_n_GSI_CH_k_CNTXT_7_OFFS 0x%x\n", val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-	GSIDBG("GSI_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_OFFS 0x%x\n", val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-	GSIDBG("GSI_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_OFFS 0x%x\n", val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_QOS_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-	GSIDBG("GSI_EE_n_GSI_CH_k_QOS_OFFS 0x%x\n", val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_SCRATCH_0_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-	GSIDBG("GSI_EE_n_GSI_CH_k_SCRATCH_0_OFFS 0x%x\n", val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_SCRATCH_1_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-	GSIDBG("GSI_EE_n_GSI_CH_k_SCRATCH_1_OFFS 0x%x\n", val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_SCRATCH_2_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-	GSIDBG("GSI_EE_n_GSI_CH_k_SCRATCH_2_OFFS 0x%x\n", val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_SCRATCH_3_OFFS(chan_hdl,
-			gsi_ctx->per.ee));
-	GSIDBG("GSI_EE_n_GSI_CH_k_SCRATCH_3_OFFS 0x%x\n", val);
-}
-EXPORT_SYMBOL(gsi_wdi3_dump_register);
-
-static int msm_gsi_probe(struct platform_device *pdev)
-{
-	struct device *dev = &pdev->dev;
-
-	pr_debug("gsi_probe\n");
-	gsi_ctx = devm_kzalloc(dev, sizeof(*gsi_ctx), GFP_KERNEL);
-	if (!gsi_ctx) {
-		dev_err(dev, "failed to allocated gsi context\n");
-		return -ENOMEM;
-	}
-
-	gsi_ctx->ipc_logbuf = ipc_log_context_create(GSI_IPC_LOG_PAGES,
-		"gsi", 0);
-	if (gsi_ctx->ipc_logbuf == NULL)
-		GSIERR("failed to create IPC log, continue...\n");
-
-	gsi_ctx->dev = dev;
-	init_completion(&gsi_ctx->gen_ee_cmd_compl);
-	gsi_debugfs_init();
-
-	return 0;
-}
-
-static struct platform_driver msm_gsi_driver = {
-	.probe          = msm_gsi_probe,
-	.driver		= {
-		.name	= "gsi",
-		.of_match_table = msm_gsi_match,
-	},
-};
-
-static struct platform_device *pdev;
-
-/**
- * Module Init.
- */
-static int __init gsi_init(void)
-{
-	int ret;
-
-	pr_debug("%s\n", __func__);
-
-	ret = platform_driver_register(&msm_gsi_driver);
-	if (ret < 0)
-		goto out;
-
-	if (running_emulation) {
-		pdev = platform_device_register_simple("gsi", -1, NULL, 0);
-		if (IS_ERR(pdev)) {
-			ret = PTR_ERR(pdev);
-			platform_driver_unregister(&msm_gsi_driver);
-			goto out;
-		}
-	}
-
-out:
-	return ret;
-}
-arch_initcall(gsi_init);
-
-/*
- * Module exit.
- */
-static void __exit gsi_exit(void)
-{
-	if (running_emulation && pdev)
-		platform_device_unregister(pdev);
-	platform_driver_unregister(&msm_gsi_driver);
-}
-module_exit(gsi_exit);
-
-MODULE_LICENSE("GPL v2");
-MODULE_DESCRIPTION("Generic Software Interface (GSI)");

gsi/gsi.h

@@ -1,345 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2015-2019, The Linux Foundation. All rights reserved.
- */
-
-#ifndef GSI_H
-#define GSI_H
-
-#include <linux/device.h>
-#include <linux/types.h>
-#include <linux/completion.h>
-#include <linux/mutex.h>
-#include <linux/spinlock.h>
-#include <linux/msm_gsi.h>
-#include <linux/errno.h>
-#include <linux/ipc_logging.h>
-
-/*
- * The following for adding code (ie. for EMULATION) not found on x86.
- */
-#if defined(CONFIG_IPA_EMULATION)
-# include "gsi_emulation_stubs.h"
-#endif
-
-#define GSI_ASSERT() \
-	BUG()
-
-#define GSI_CHAN_MAX      31
-#define GSI_EVT_RING_MAX  24
-#define GSI_NO_EVT_ERINDEX 31
-
-#define gsi_readl(c)	(readl(c))
-#define gsi_writel(v, c)	({ __iowmb(); writel_relaxed((v), (c)); })
-
-#define GSI_IPC_LOGGING(buf, fmt, args...) \
-	do { \
-		if (buf) \
-			ipc_log_string((buf), fmt, __func__, __LINE__, \
-				## args); \
-	} while (0)
-
-#define GSIDBG(fmt, args...) \
-	do { \
-		dev_dbg(gsi_ctx->dev, "%s:%d " fmt, __func__, __LINE__, \
-		## args);\
-		if (gsi_ctx) { \
-			GSI_IPC_LOGGING(gsi_ctx->ipc_logbuf, \
-				"%s:%d " fmt, ## args); \
-			GSI_IPC_LOGGING(gsi_ctx->ipc_logbuf_low, \
-				"%s:%d " fmt, ## args); \
-		} \
-	} while (0)
-
-#define GSIDBG_LOW(fmt, args...) \
-	do { \
-		dev_dbg(gsi_ctx->dev, "%s:%d " fmt, __func__, __LINE__, \
-		## args);\
-		if (gsi_ctx) { \
-			GSI_IPC_LOGGING(gsi_ctx->ipc_logbuf_low, \
-				"%s:%d " fmt, ## args); \
-		} \
-	} while (0)
-
-#define GSIERR(fmt, args...) \
-	do { \
-		dev_err(gsi_ctx->dev, "%s:%d " fmt, __func__, __LINE__, \
-		## args);\
-		if (gsi_ctx) { \
-			GSI_IPC_LOGGING(gsi_ctx->ipc_logbuf, \
-				"%s:%d " fmt, ## args); \
-			GSI_IPC_LOGGING(gsi_ctx->ipc_logbuf_low, \
-				"%s:%d " fmt, ## args); \
-		} \
-	} while (0)
-
-#define GSI_IPC_LOG_PAGES 50
-
-enum gsi_evt_ring_state {
-	GSI_EVT_RING_STATE_NOT_ALLOCATED = 0x0,
-	GSI_EVT_RING_STATE_ALLOCATED = 0x1,
-	GSI_EVT_RING_STATE_ERROR = 0xf
-};
-
-enum gsi_chan_state {
-	GSI_CHAN_STATE_NOT_ALLOCATED = 0x0,
-	GSI_CHAN_STATE_ALLOCATED = 0x1,
-	GSI_CHAN_STATE_STARTED = 0x2,
-	GSI_CHAN_STATE_STOPPED = 0x3,
-	GSI_CHAN_STATE_STOP_IN_PROC = 0x4,
-	GSI_CHAN_STATE_ERROR = 0xf
-};
-
-struct gsi_ring_ctx {
-	spinlock_t slock;
-	unsigned long base_va;
-	uint64_t base;
-	uint64_t wp;
-	uint64_t rp;
-	uint64_t wp_local;
-	uint64_t rp_local;
-	uint16_t len;
-	uint8_t elem_sz;
-	uint16_t max_num_elem;
-	uint64_t end;
-};
-
-struct gsi_chan_dp_stats {
-	unsigned long ch_below_lo;
-	unsigned long ch_below_hi;
-	unsigned long ch_above_hi;
-	unsigned long empty_time;
-	unsigned long last_timestamp;
-};
-
-struct gsi_chan_stats {
-	unsigned long queued;
-	unsigned long completed;
-	unsigned long callback_to_poll;
-	unsigned long poll_to_callback;
-	unsigned long poll_pending_irq;
-	unsigned long invalid_tre_error;
-	unsigned long poll_ok;
-	unsigned long poll_empty;
-	unsigned long userdata_in_use;
-	struct gsi_chan_dp_stats dp;
-};
-
-/**
- * struct gsi_user_data - user_data element pointed by the TRE
- * @valid: valid to be cleaned. if its true that means it is being used.
- *	false means its free to overwrite
- * @p: pointer to the user data array element
- */
-struct gsi_user_data {
-	bool valid;
-	void *p;
-};
-
-struct gsi_chan_ctx {
-	struct gsi_chan_props props;
-	enum gsi_chan_state state;
-	struct gsi_ring_ctx ring;
-	struct gsi_user_data *user_data;
-	struct gsi_evt_ctx *evtr;
-	struct mutex mlock;
-	struct completion compl;
-	bool allocated;
-	atomic_t poll_mode;
-	union __packed gsi_channel_scratch scratch;
-	struct gsi_chan_stats stats;
-	bool enable_dp_stats;
-	bool print_dp_stats;
-};
-
-struct gsi_evt_stats {
-	unsigned long completed;
-};
-
-struct gsi_evt_ctx {
-	struct gsi_evt_ring_props props;
-	enum gsi_evt_ring_state state;
-	uint8_t id;
-	struct gsi_ring_ctx ring;
-	struct mutex mlock;
-	struct completion compl;
-	struct gsi_chan_ctx *chan;
-	atomic_t chan_ref_cnt;
-	union __packed gsi_evt_scratch scratch;
-	struct gsi_evt_stats stats;
-};
-
-struct gsi_ee_scratch {
-	union __packed {
-		struct {
-			uint32_t inter_ee_cmd_return_code:3;
-			uint32_t resvd1:2;
-			uint32_t generic_ee_cmd_return_code:3;
-			uint32_t resvd2:7;
-			uint32_t max_usb_pkt_size:1;
-			uint32_t resvd3:8;
-			uint32_t mhi_base_chan_idx:8;
-		} s;
-		uint32_t val;
-	} word0;
-	uint32_t word1;
-};
-
-struct ch_debug_stats {
-	unsigned long ch_allocate;
-	unsigned long ch_start;
-	unsigned long ch_stop;
-	unsigned long ch_reset;
-	unsigned long ch_de_alloc;
-	unsigned long ch_db_stop;
-	unsigned long cmd_completed;
-};
-
-struct gsi_generic_ee_cmd_debug_stats {
-	unsigned long halt_channel;
-};
-
-struct gsi_ctx {
-	void __iomem *base;
-	struct device *dev;
-	struct gsi_per_props per;
-	bool per_registered;
-	struct gsi_chan_ctx chan[GSI_CHAN_MAX];
-	struct ch_debug_stats ch_dbg[GSI_CHAN_MAX];
-	struct gsi_evt_ctx evtr[GSI_EVT_RING_MAX];
-	struct gsi_generic_ee_cmd_debug_stats gen_ee_cmd_dbg;
-	struct mutex mlock;
-	spinlock_t slock;
-	unsigned long evt_bmap;
-	bool enabled;
-	atomic_t num_chan;
-	atomic_t num_evt_ring;
-	struct gsi_ee_scratch scratch;
-	int num_ch_dp_stats;
-	struct workqueue_struct *dp_stat_wq;
-	u32 max_ch;
-	u32 max_ev;
-	struct completion gen_ee_cmd_compl;
-	void *ipc_logbuf;
-	void *ipc_logbuf_low;
-	/*
-	 * The following used only on emulation systems.
-	 */
-	void __iomem *intcntrlr_base;
-	u32 intcntrlr_mem_size;
-	irq_handler_t intcntrlr_gsi_isr;
-	irq_handler_t intcntrlr_client_isr;
-};
-
-enum gsi_re_type {
-	GSI_RE_XFER = 0x2,
-	GSI_RE_IMMD_CMD = 0x3,
-	GSI_RE_NOP = 0x4,
-	GSI_RE_COAL = 0x8,
-};
-
-struct __packed gsi_tre {
-	uint64_t buffer_ptr;
-	uint16_t buf_len;
-	uint16_t resvd1;
-	uint16_t chain:1;
-	uint16_t resvd4:7;
-	uint16_t ieob:1;
-	uint16_t ieot:1;
-	uint16_t bei:1;
-	uint16_t resvd3:5;
-	uint8_t re_type;
-	uint8_t resvd2;
-};
-
-struct __packed gsi_gci_tre {
-	uint64_t buffer_ptr:41;
-	uint64_t resvd1:7;
-	uint64_t buf_len:16;
-	uint64_t cookie:40;
-	uint64_t resvd2:8;
-	uint64_t re_type:8;
-	uint64_t resvd3:8;
-};
-
-#define GSI_XFER_COMPL_TYPE_GCI 0x28
-
-struct __packed gsi_xfer_compl_evt {
-	union {
-		uint64_t xfer_ptr;
-		struct {
-			uint64_t cookie:40;
-			uint64_t resvd1:24;
-		};
-	};
-	uint16_t len;
-	uint8_t veid;
-	uint8_t code;  /* see gsi_chan_evt */
-	uint16_t resvd;
-	uint8_t type;
-	uint8_t chid;
-};
-
-enum gsi_err_type {
-	GSI_ERR_TYPE_GLOB = 0x1,
-	GSI_ERR_TYPE_CHAN = 0x2,
-	GSI_ERR_TYPE_EVT = 0x3,
-};
-
-enum gsi_err_code {
-	GSI_INVALID_TRE_ERR = 0x1,
-	GSI_OUT_OF_BUFFERS_ERR = 0x2,
-	GSI_OUT_OF_RESOURCES_ERR = 0x3,
-	GSI_UNSUPPORTED_INTER_EE_OP_ERR = 0x4,
-	GSI_EVT_RING_EMPTY_ERR = 0x5,
-	GSI_NON_ALLOCATED_EVT_ACCESS_ERR = 0x6,
-	GSI_HWO_1_ERR = 0x8
-};
-
-struct __packed gsi_log_err {
-	uint32_t arg3:4;
-	uint32_t arg2:4;
-	uint32_t arg1:4;
-	uint32_t code:4;
-	uint32_t resvd:3;
-	uint32_t virt_idx:5;
-	uint32_t err_type:4;
-	uint32_t ee:4;
-};
-
-enum gsi_ch_cmd_opcode {
-	GSI_CH_ALLOCATE = 0x0,
-	GSI_CH_START = 0x1,
-	GSI_CH_STOP = 0x2,
-	GSI_CH_RESET = 0x9,
-	GSI_CH_DE_ALLOC = 0xa,
-	GSI_CH_DB_STOP = 0xb,
-};
-
-enum gsi_evt_ch_cmd_opcode {
-	GSI_EVT_ALLOCATE = 0x0,
-	GSI_EVT_RESET = 0x9,
-	GSI_EVT_DE_ALLOC = 0xa,
-};
-
-enum gsi_generic_ee_cmd_opcode {
-	GSI_GEN_EE_CMD_HALT_CHANNEL = 0x1,
-	GSI_GEN_EE_CMD_ALLOC_CHANNEL = 0x2,
-};
-
-enum gsi_generic_ee_cmd_return_code {
-	GSI_GEN_EE_CMD_RETURN_CODE_SUCCESS = 0x1,
-	GSI_GEN_EE_CMD_RETURN_CODE_CHANNEL_NOT_RUNNING = 0x2,
-	GSI_GEN_EE_CMD_RETURN_CODE_INCORRECT_DIRECTION = 0x3,
-	GSI_GEN_EE_CMD_RETURN_CODE_INCORRECT_CHANNEL_TYPE = 0x4,
-	GSI_GEN_EE_CMD_RETURN_CODE_INCORRECT_CHANNEL_INDEX = 0x5,
-	GSI_GEN_EE_CMD_RETURN_CODE_RETRY = 0x6,
-	GSI_GEN_EE_CMD_RETURN_CODE_OUT_OF_RESOURCES = 0x7,
-};
-
-extern struct gsi_ctx *gsi_ctx;
-void gsi_debugfs_init(void);
-uint16_t gsi_find_idx_from_addr(struct gsi_ring_ctx *ctx, uint64_t addr);
-void gsi_update_ch_dp_stats(struct gsi_chan_ctx *ctx, uint16_t used);
-
-#endif

gsi/gsi_dbg.c

@@ -1,744 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2015-2019, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/completion.h>
-#include <linux/debugfs.h>
-#include <linux/dma-mapping.h>
-#include <linux/random.h>
-#include <linux/uaccess.h>
-#include <linux/msm_gsi.h>
-#include "gsi_reg.h"
-#include "gsi.h"
-
-#define TERR(fmt, args...) \
-		pr_err("%s:%d " fmt, __func__, __LINE__, ## args)
-#define TDBG(fmt, args...) \
-		pr_debug("%s:%d " fmt, __func__, __LINE__, ## args)
-#define PRT_STAT(fmt, args...) \
-		pr_err(fmt, ## args)
-
-static struct dentry *dent;
-static char dbg_buff[4096];
-static void *gsi_ipc_logbuf_low;
-
-static void gsi_wq_print_dp_stats(struct work_struct *work);
-static DECLARE_DELAYED_WORK(gsi_print_dp_stats_work, gsi_wq_print_dp_stats);
-static void gsi_wq_update_dp_stats(struct work_struct *work);
-static DECLARE_DELAYED_WORK(gsi_update_dp_stats_work, gsi_wq_update_dp_stats);
-
-static ssize_t gsi_dump_evt(struct file *file,
-		const char __user *buf, size_t count, loff_t *ppos)
-{
-	u32 arg1;
-	u32 arg2;
-	unsigned long missing;
-	char *sptr, *token;
-	uint32_t val;
-	struct gsi_evt_ctx *ctx;
-	uint16_t i;
-
-	if (count >= sizeof(dbg_buff))
-		return -EINVAL;
-
-	missing = copy_from_user(dbg_buff, buf, count);
-	if (missing)
-		return -EFAULT;
-
-	dbg_buff[count] = '\0';
-
-	sptr = dbg_buff;
-
-	token = strsep(&sptr, " ");
-	if (!token)
-		return -EINVAL;
-	if (kstrtou32(token, 0, &arg1))
-		return -EINVAL;
-
-	token = strsep(&sptr, " ");
-	if (!token)
-		return -EINVAL;
-	if (kstrtou32(token, 0, &arg2))
-		return -EINVAL;
-
-	TDBG("arg1=%u arg2=%u\n", arg1, arg2);
-
-	if (arg1 >= gsi_ctx->max_ev) {
-		TERR("invalid evt ring id %u\n", arg1);
-		return -EINVAL;
-	}
-
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_EV_CH_k_CNTXT_0_OFFS(arg1, gsi_ctx->per.ee));
-	TERR("EV%2d CTX0  0x%x\n", arg1, val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_EV_CH_k_CNTXT_1_OFFS(arg1, gsi_ctx->per.ee));
-	TERR("EV%2d CTX1  0x%x\n", arg1, val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_EV_CH_k_CNTXT_2_OFFS(arg1, gsi_ctx->per.ee));
-	TERR("EV%2d CTX2  0x%x\n", arg1, val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_EV_CH_k_CNTXT_3_OFFS(arg1, gsi_ctx->per.ee));
-	TERR("EV%2d CTX3  0x%x\n", arg1, val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_EV_CH_k_CNTXT_4_OFFS(arg1, gsi_ctx->per.ee));
-	TERR("EV%2d CTX4  0x%x\n", arg1, val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_EV_CH_k_CNTXT_5_OFFS(arg1, gsi_ctx->per.ee));
-	TERR("EV%2d CTX5  0x%x\n", arg1, val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_EV_CH_k_CNTXT_6_OFFS(arg1, gsi_ctx->per.ee));
-	TERR("EV%2d CTX6  0x%x\n", arg1, val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_EV_CH_k_CNTXT_7_OFFS(arg1, gsi_ctx->per.ee));
-	TERR("EV%2d CTX7  0x%x\n", arg1, val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_EV_CH_k_CNTXT_8_OFFS(arg1, gsi_ctx->per.ee));
-	TERR("EV%2d CTX8  0x%x\n", arg1, val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_EV_CH_k_CNTXT_9_OFFS(arg1, gsi_ctx->per.ee));
-	TERR("EV%2d CTX9  0x%x\n", arg1, val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_EV_CH_k_CNTXT_10_OFFS(arg1, gsi_ctx->per.ee));
-	TERR("EV%2d CTX10 0x%x\n", arg1, val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_EV_CH_k_CNTXT_11_OFFS(arg1, gsi_ctx->per.ee));
-	TERR("EV%2d CTX11 0x%x\n", arg1, val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_EV_CH_k_CNTXT_12_OFFS(arg1, gsi_ctx->per.ee));
-	TERR("EV%2d CTX12 0x%x\n", arg1, val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_EV_CH_k_CNTXT_13_OFFS(arg1, gsi_ctx->per.ee));
-	TERR("EV%2d CTX13 0x%x\n", arg1, val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_EV_CH_k_SCRATCH_0_OFFS(arg1, gsi_ctx->per.ee));
-	TERR("EV%2d SCR0  0x%x\n", arg1, val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_EV_CH_k_SCRATCH_1_OFFS(arg1, gsi_ctx->per.ee));
-	TERR("EV%2d SCR1  0x%x\n", arg1, val);
-
-	if (arg2) {
-		ctx = &gsi_ctx->evtr[arg1];
-
-		if (ctx->props.ring_base_vaddr) {
-			for (i = 0; i < ctx->props.ring_len / 16; i++)
-				TERR("EV%2d (0x%08llx) %08x %08x %08x %08x\n",
-				arg1, ctx->props.ring_base_addr + i * 16,
-				*(u32 *)((u8 *)ctx->props.ring_base_vaddr +
-					i * 16 + 0),
-				*(u32 *)((u8 *)ctx->props.ring_base_vaddr +
-					i * 16 + 4),
-				*(u32 *)((u8 *)ctx->props.ring_base_vaddr +
-					i * 16 + 8),
-				*(u32 *)((u8 *)ctx->props.ring_base_vaddr +
-					i * 16 + 12));
-		} else {
-			TERR("No VA supplied for event ring id %u\n", arg1);
-		}
-	}
-
-	return count;
-}
-
-static ssize_t gsi_dump_ch(struct file *file,
-		const char __user *buf, size_t count, loff_t *ppos)
-{
-	u32 arg1;
-	u32 arg2;
-	unsigned long missing;
-	char *sptr, *token;
-	uint32_t val;
-	struct gsi_chan_ctx *ctx;
-	uint16_t i;
-
-	if (count >= sizeof(dbg_buff))
-		return -EINVAL;
-
-	missing = copy_from_user(dbg_buff, buf, count);
-	if (missing)
-		return -EFAULT;
-
-	dbg_buff[count] = '\0';
-
-	sptr = dbg_buff;
-
-	token = strsep(&sptr, " ");
-	if (!token)
-		return -EINVAL;
-	if (kstrtou32(token, 0, &arg1))
-		return -EINVAL;
-
-	token = strsep(&sptr, " ");
-	if (!token)
-		return -EINVAL;
-	if (kstrtou32(token, 0, &arg2))
-		return -EINVAL;
-
-	TDBG("arg1=%u arg2=%u\n", arg1, arg2);
-
-	if (arg1 >= gsi_ctx->max_ch) {
-		TERR("invalid chan id %u\n", arg1);
-		return -EINVAL;
-	}
-
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_CNTXT_0_OFFS(arg1, gsi_ctx->per.ee));
-	TERR("CH%2d CTX0  0x%x\n", arg1, val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_CNTXT_1_OFFS(arg1, gsi_ctx->per.ee));
-	TERR("CH%2d CTX1  0x%x\n", arg1, val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_CNTXT_2_OFFS(arg1, gsi_ctx->per.ee));
-	TERR("CH%2d CTX2  0x%x\n", arg1, val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_CNTXT_3_OFFS(arg1, gsi_ctx->per.ee));
-	TERR("CH%2d CTX3  0x%x\n", arg1, val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_CNTXT_4_OFFS(arg1, gsi_ctx->per.ee));
-	TERR("CH%2d CTX4  0x%x\n", arg1, val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_CNTXT_5_OFFS(arg1, gsi_ctx->per.ee));
-	TERR("CH%2d CTX5  0x%x\n", arg1, val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_CNTXT_6_OFFS(arg1, gsi_ctx->per.ee));
-	TERR("CH%2d CTX6  0x%x\n", arg1, val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_CNTXT_7_OFFS(arg1, gsi_ctx->per.ee));
-	TERR("CH%2d CTX7  0x%x\n", arg1, val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_OFFS(arg1,
-			gsi_ctx->per.ee));
-	TERR("CH%2d REFRP 0x%x\n", arg1, val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_OFFS(arg1,
-			gsi_ctx->per.ee));
-	TERR("CH%2d REFWP 0x%x\n", arg1, val);
-	if (gsi_ctx->per.ver >= GSI_VER_2_5) {
-		val = gsi_readl(gsi_ctx->base +
-			GSI_V2_5_EE_n_GSI_CH_k_QOS_OFFS(arg1, gsi_ctx->per.ee));
-	} else {
-		val = gsi_readl(gsi_ctx->base +
-			GSI_EE_n_GSI_CH_k_QOS_OFFS(arg1, gsi_ctx->per.ee));
-	}
-	TERR("CH%2d QOS   0x%x\n", arg1, val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_SCRATCH_0_OFFS(arg1, gsi_ctx->per.ee));
-	TERR("CH%2d SCR0  0x%x\n", arg1, val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_SCRATCH_1_OFFS(arg1, gsi_ctx->per.ee));
-	TERR("CH%2d SCR1  0x%x\n", arg1, val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_SCRATCH_2_OFFS(arg1, gsi_ctx->per.ee));
-	TERR("CH%2d SCR2  0x%x\n", arg1, val);
-	val = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_SCRATCH_3_OFFS(arg1, gsi_ctx->per.ee));
-	TERR("CH%2d SCR3  0x%x\n", arg1, val);
-
-	if (arg2) {
-		ctx = &gsi_ctx->chan[arg1];
-
-		if (ctx->props.ring_base_vaddr) {
-			for (i = 0; i < ctx->props.ring_len / 16; i++)
-				TERR("CH%2d (0x%08llx) %08x %08x %08x %08x\n",
-				arg1, ctx->props.ring_base_addr + i * 16,
-				*(u32 *)((u8 *)ctx->props.ring_base_vaddr +
-					i * 16 + 0),
-				*(u32 *)((u8 *)ctx->props.ring_base_vaddr +
-					i * 16 + 4),
-				*(u32 *)((u8 *)ctx->props.ring_base_vaddr +
-					i * 16 + 8),
-				*(u32 *)((u8 *)ctx->props.ring_base_vaddr +
-					i * 16 + 12));
-		} else {
-			TERR("No VA supplied for chan id %u\n", arg1);
-		}
-	}
-
-	return count;
-}
-
-static void gsi_dump_ch_stats(struct gsi_chan_ctx *ctx)
-{
-	if (!ctx->allocated)
-		return;
-
-	PRT_STAT("CH%2d:\n", ctx->props.ch_id);
-	PRT_STAT("queued=%lu compl=%lu\n",
-		ctx->stats.queued,
-		ctx->stats.completed);
-	PRT_STAT("cb->poll=%lu poll->cb=%lu poll_pend_irq=%lu\n",
-		ctx->stats.callback_to_poll,
-		ctx->stats.poll_to_callback,
-		ctx->stats.poll_pending_irq);
-	PRT_STAT("invalid_tre_error=%lu\n",
-		ctx->stats.invalid_tre_error);
-	PRT_STAT("poll_ok=%lu poll_empty=%lu\n",
-		ctx->stats.poll_ok, ctx->stats.poll_empty);
-	if (ctx->evtr)
-		PRT_STAT("compl_evt=%lu\n",
-			ctx->evtr->stats.completed);
-	PRT_STAT("userdata_in_use=%lu\n", ctx->stats.userdata_in_use);
-
-	PRT_STAT("ch_below_lo=%lu\n", ctx->stats.dp.ch_below_lo);
-	PRT_STAT("ch_below_hi=%lu\n", ctx->stats.dp.ch_below_hi);
-	PRT_STAT("ch_above_hi=%lu\n", ctx->stats.dp.ch_above_hi);
-	PRT_STAT("time_empty=%lums\n", ctx->stats.dp.empty_time);
-	PRT_STAT("\n");
-}
-
-static ssize_t gsi_dump_stats(struct file *file,
-		const char __user *buf, size_t count, loff_t *ppos)
-{
-	int ch_id;
-	int min, max, ret;
-
-	ret = kstrtos32_from_user(buf, count, 0, &ch_id);
-	if (ret)
-		return ret;
-
-	if (ch_id == -1) {
-		min = 0;
-		max = gsi_ctx->max_ch;
-	} else if (ch_id < 0 || ch_id >= gsi_ctx->max_ch ||
-		   !gsi_ctx->chan[ch_id].allocated) {
-		goto error;
-	} else {
-		min = ch_id;
-		max = ch_id + 1;
-	}
-
-	for (ch_id = min; ch_id < max; ch_id++)
-		gsi_dump_ch_stats(&gsi_ctx->chan[ch_id]);
-
-	return count;
-error:
-	TERR("Usage: echo ch_id > stats. Use -1 for all\n");
-	return -EINVAL;
-}
-
-static int gsi_dbg_create_stats_wq(void)
-{
-	gsi_ctx->dp_stat_wq =
-		create_singlethread_workqueue("gsi_stat");
-	if (!gsi_ctx->dp_stat_wq) {
-		TERR("failed create workqueue\n");
-		return -ENOMEM;
-	}
-
-	return 0;
-}
-
-static void gsi_dbg_destroy_stats_wq(void)
-{
-	cancel_delayed_work_sync(&gsi_update_dp_stats_work);
-	cancel_delayed_work_sync(&gsi_print_dp_stats_work);
-	flush_workqueue(gsi_ctx->dp_stat_wq);
-	destroy_workqueue(gsi_ctx->dp_stat_wq);
-	gsi_ctx->dp_stat_wq = NULL;
-}
-
-static ssize_t gsi_enable_dp_stats(struct file *file,
-	const char __user *buf, size_t count, loff_t *ppos)
-{
-	int ch_id;
-	bool enable;
-	int ret;
-
-	if (count >= sizeof(dbg_buff))
-		goto error;
-
-	if (copy_from_user(dbg_buff, buf, count))
-		goto error;
-
-	dbg_buff[count] = '\0';
-
-	if (dbg_buff[0] != '+' && dbg_buff[0] != '-')
-		goto error;
-
-	enable = (dbg_buff[0] == '+');
-
-	if (kstrtos32(dbg_buff + 1, 0, &ch_id))
-		goto error;
-
-	if (ch_id < 0 || ch_id >= gsi_ctx->max_ch ||
-	    !gsi_ctx->chan[ch_id].allocated) {
-		goto error;
-	}
-
-	if (gsi_ctx->chan[ch_id].enable_dp_stats == enable) {
-		TERR("ch_%d: already enabled/disabled\n", ch_id);
-		return -EINVAL;
-	}
-	gsi_ctx->chan[ch_id].enable_dp_stats = enable;
-
-	if (enable)
-		gsi_ctx->num_ch_dp_stats++;
-	else
-		gsi_ctx->num_ch_dp_stats--;
-
-	if (enable) {
-		if (gsi_ctx->num_ch_dp_stats == 1) {
-			ret = gsi_dbg_create_stats_wq();
-			if (ret)
-				return ret;
-		}
-		cancel_delayed_work_sync(&gsi_update_dp_stats_work);
-		queue_delayed_work(gsi_ctx->dp_stat_wq,
-			&gsi_update_dp_stats_work, msecs_to_jiffies(10));
-	} else if (!enable && gsi_ctx->num_ch_dp_stats == 0) {
-		gsi_dbg_destroy_stats_wq();
-	}
-
-	return count;
-error:
-	TERR("Usage: echo [+-]ch_id > enable_dp_stats\n");
-	return -EINVAL;
-}
-
-static ssize_t gsi_set_max_elem_dp_stats(struct file *file,
-		const char __user *buf, size_t count, loff_t *ppos)
-{
-	u32 ch_id;
-	u32 max_elem;
-	unsigned long missing;
-	char *sptr, *token;
-
-
-	if (count >= sizeof(dbg_buff))
-		goto error;
-
-	missing = copy_from_user(dbg_buff, buf, count);
-	if (missing)
-		goto error;
-
-	dbg_buff[count] = '\0';
-
-	sptr = dbg_buff;
-
-	token = strsep(&sptr, " ");
-	if (!token) {
-		TERR("\n");
-		goto error;
-	}
-
-	if (kstrtou32(token, 0, &ch_id)) {
-		TERR("\n");
-		goto error;
-	}
-
-	token = strsep(&sptr, " ");
-	if (!token) {
-		/* get */
-		if (kstrtou32(dbg_buff, 0, &ch_id))
-			goto error;
-		if (ch_id >= gsi_ctx->max_ch)
-			goto error;
-		PRT_STAT("ch %d: max_re_expected=%d\n", ch_id,
-			gsi_ctx->chan[ch_id].props.max_re_expected);
-		return count;
-	}
-	if (kstrtou32(token, 0, &max_elem)) {
-		TERR("\n");
-		goto error;
-	}
-
-	TDBG("ch_id=%u max_elem=%u\n", ch_id, max_elem);
-
-	if (ch_id >= gsi_ctx->max_ch) {
-		TERR("invalid chan id %u\n", ch_id);
-		goto error;
-	}
-
-	gsi_ctx->chan[ch_id].props.max_re_expected = max_elem;
-
-	return count;
-
-error:
-	TERR("Usage: (set) echo <ch_id> <max_elem> > max_elem_dp_stats\n");
-	TERR("Usage: (get) echo <ch_id> > max_elem_dp_stats\n");
-	return -EINVAL;
-}
-
-static void gsi_wq_print_dp_stats(struct work_struct *work)
-{
-	int ch_id;
-
-	for (ch_id = 0; ch_id < gsi_ctx->max_ch; ch_id++) {
-		if (gsi_ctx->chan[ch_id].print_dp_stats)
-			gsi_dump_ch_stats(&gsi_ctx->chan[ch_id]);
-	}
-
-	queue_delayed_work(gsi_ctx->dp_stat_wq, &gsi_print_dp_stats_work,
-		msecs_to_jiffies(1000));
-}
-
-static void gsi_dbg_update_ch_dp_stats(struct gsi_chan_ctx *ctx)
-{
-	uint16_t start_hw;
-	uint16_t end_hw;
-	uint64_t rp_hw;
-	uint64_t wp_hw;
-	int ee = gsi_ctx->per.ee;
-	uint16_t used_hw;
-
-	rp_hw = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_CNTXT_4_OFFS(ctx->props.ch_id, ee));
-	rp_hw |= ((uint64_t)gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_CNTXT_5_OFFS(ctx->props.ch_id, ee)))
-		<< 32;
-
-	wp_hw = gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_CNTXT_6_OFFS(ctx->props.ch_id, ee));
-	wp_hw |= ((uint64_t)gsi_readl(gsi_ctx->base +
-		GSI_EE_n_GSI_CH_k_CNTXT_7_OFFS(ctx->props.ch_id, ee)))
-		<< 32;
-
-	start_hw = gsi_find_idx_from_addr(&ctx->ring, rp_hw);
-	end_hw = gsi_find_idx_from_addr(&ctx->ring, wp_hw);
-
-	if (end_hw >= start_hw)
-		used_hw = end_hw - start_hw;
-	else
-		used_hw = ctx->ring.max_num_elem + 1 - (start_hw - end_hw);
-
-	TDBG("ch %d used %d\n", ctx->props.ch_id, used_hw);
-	gsi_update_ch_dp_stats(ctx, used_hw);
-}
-
-static void gsi_wq_update_dp_stats(struct work_struct *work)
-{
-	int ch_id;
-
-	for (ch_id = 0; ch_id < gsi_ctx->max_ch; ch_id++) {
-		if (gsi_ctx->chan[ch_id].allocated &&
-		    gsi_ctx->chan[ch_id].enable_dp_stats)
-			gsi_dbg_update_ch_dp_stats(&gsi_ctx->chan[ch_id]);
-	}
-
-	queue_delayed_work(gsi_ctx->dp_stat_wq, &gsi_update_dp_stats_work,
-		msecs_to_jiffies(10));
-}
-
-
-static ssize_t gsi_rst_stats(struct file *file,
-		const char __user *buf, size_t count, loff_t *ppos)
-{
-	int ch_id;
-	int min, max, ret;
-
-	ret = kstrtos32_from_user(buf, count, 0, &ch_id);
-	if (ret)
-		return ret;
-
-	if (ch_id == -1) {
-		min = 0;
-		max = gsi_ctx->max_ch;
-	} else if (ch_id < 0 || ch_id >= gsi_ctx->max_ch ||
-		   !gsi_ctx->chan[ch_id].allocated) {
-		goto error;
-	} else {
-		min = ch_id;
-		max = ch_id + 1;
-	}
-
-	for (ch_id = min; ch_id < max; ch_id++)
-		memset(&gsi_ctx->chan[ch_id].stats, 0,
-			sizeof(gsi_ctx->chan[ch_id].stats));
-
-	return count;
-error:
-	TERR("Usage: echo ch_id > rst_stats. Use -1 for all\n");
-	return -EINVAL;
-}
-
-static ssize_t gsi_print_dp_stats(struct file *file,
-	const char __user *buf, size_t count, loff_t *ppos)
-{
-	int ch_id;
-	bool enable;
-	int ret;
-
-	if (count >= sizeof(dbg_buff))
-		goto error;
-
-	if (copy_from_user(dbg_buff, buf, count))
-		goto error;
-
-	dbg_buff[count] = '\0';
-
-	if (dbg_buff[0] != '+' && dbg_buff[0] != '-')
-		goto error;
-
-	enable = (dbg_buff[0] == '+');
-
-	if (kstrtos32(dbg_buff + 1, 0, &ch_id))
-		goto error;
-
-	if (ch_id < 0 || ch_id >= gsi_ctx->max_ch ||
-	    !gsi_ctx->chan[ch_id].allocated) {
-		goto error;
-	}
-
-	if (gsi_ctx->chan[ch_id].print_dp_stats == enable) {
-		TERR("ch_%d: already enabled/disabled\n", ch_id);
-		return -EINVAL;
-	}
-	gsi_ctx->chan[ch_id].print_dp_stats = enable;
-
-	if (enable)
-		gsi_ctx->num_ch_dp_stats++;
-	else
-		gsi_ctx->num_ch_dp_stats--;
-
-	if (enable) {
-		if (gsi_ctx->num_ch_dp_stats == 1) {
-			ret = gsi_dbg_create_stats_wq();
-			if (ret)
-				return ret;
-		}
-		cancel_delayed_work_sync(&gsi_print_dp_stats_work);
-		queue_delayed_work(gsi_ctx->dp_stat_wq,
-			&gsi_print_dp_stats_work, msecs_to_jiffies(10));
-	} else if (!enable && gsi_ctx->num_ch_dp_stats == 0) {
-		gsi_dbg_destroy_stats_wq();
-	}
-
-	return count;
-error:
-	TERR("Usage: echo [+-]ch_id > print_dp_stats\n");
-	return -EINVAL;
-}
-
-static ssize_t gsi_enable_ipc_low(struct file *file,
-	const char __user *ubuf, size_t count, loff_t *ppos)
-{
-	s8 option = 0;
-	int ret;
-
-	ret = kstrtos8_from_user(ubuf, count, 0, &option);
-	if (ret)
-		return ret;
-
-	mutex_lock(&gsi_ctx->mlock);
-	if (option) {
-		if (!gsi_ipc_logbuf_low) {
-			gsi_ipc_logbuf_low =
-				ipc_log_context_create(GSI_IPC_LOG_PAGES,
-					"gsi_low", 0);
-			if (gsi_ipc_logbuf_low == NULL)
-				TERR("failed to get ipc_logbuf_low\n");
-		}
-		gsi_ctx->ipc_logbuf_low = gsi_ipc_logbuf_low;
-	} else {
-		gsi_ctx->ipc_logbuf_low = NULL;
-	}
-	mutex_unlock(&gsi_ctx->mlock);
-
-	return count;
-}
-
-const struct file_operations gsi_ev_dump_ops = {
-	.write = gsi_dump_evt,
-};
-
-const struct file_operations gsi_ch_dump_ops = {
-	.write = gsi_dump_ch,
-};
-
-const struct file_operations gsi_stats_ops = {
-	.write = gsi_dump_stats,
-};
-
-const struct file_operations gsi_enable_dp_stats_ops = {
-	.write = gsi_enable_dp_stats,
-};
-
-const struct file_operations gsi_max_elem_dp_stats_ops = {
-	.write = gsi_set_max_elem_dp_stats,
-};
-
-const struct file_operations gsi_rst_stats_ops = {
-	.write = gsi_rst_stats,
-};
-
-const struct file_operations gsi_print_dp_stats_ops = {
-	.write = gsi_print_dp_stats,
-};
-
-const struct file_operations gsi_ipc_low_ops = {
-	.write = gsi_enable_ipc_low,
-};
-
-void gsi_debugfs_init(void)
-{
-	static struct dentry *dfile;
-	const mode_t write_only_mode = 0220;
-
-	dent = debugfs_create_dir("gsi", 0);
-	if (IS_ERR(dent)) {
-		TERR("fail to create dir\n");
-		return;
-	}
-
-	dfile = debugfs_create_file("ev_dump", write_only_mode,
-			dent, 0, &gsi_ev_dump_ops);
-	if (!dfile || IS_ERR(dfile)) {
-		TERR("fail to create ev_dump file\n");
-		goto fail;
-	}
-
-	dfile = debugfs_create_file("ch_dump", write_only_mode,
-			dent, 0, &gsi_ch_dump_ops);
-	if (!dfile || IS_ERR(dfile)) {
-		TERR("fail to create ch_dump file\n");
-		goto fail;
-	}
-
-	dfile = debugfs_create_file("stats", write_only_mode, dent,
-			0, &gsi_stats_ops);
-	if (!dfile || IS_ERR(dfile)) {
-		TERR("fail to create stats file\n");
-		goto fail;
-	}
-
-	dfile = debugfs_create_file("enable_dp_stats", write_only_mode, dent,
-			0, &gsi_enable_dp_stats_ops);
-	if (!dfile || IS_ERR(dfile)) {
-		TERR("fail to create stats file\n");
-		goto fail;
-	}
-
-	dfile = debugfs_create_file("max_elem_dp_stats", write_only_mode,
-		dent, 0, &gsi_max_elem_dp_stats_ops);
-	if (!dfile || IS_ERR(dfile)) {
-		TERR("fail to create stats file\n");
-		goto fail;
-	}
-
-	dfile = debugfs_create_file("rst_stats", write_only_mode,
-		dent, 0, &gsi_rst_stats_ops);
-	if (!dfile || IS_ERR(dfile)) {
-		TERR("fail to create stats file\n");
-		goto fail;
-	}
-
-	dfile = debugfs_create_file("print_dp_stats",
-		write_only_mode, dent, 0, &gsi_print_dp_stats_ops);
-	if (!dfile || IS_ERR(dfile)) {
-		TERR("fail to create stats file\n");
-		goto fail;
-	}
-
-	dfile = debugfs_create_file("ipc_low", write_only_mode,
-		dent, 0, &gsi_ipc_low_ops);
-	if (!dfile || IS_ERR(dfile)) {
-		TERR("could not create ipc_low\n");
-		goto fail;
-	}
-
-	return;
-fail:
-	debugfs_remove_recursive(dent);
-}
-

gsi/gsi_emulation.c

@@ -1,227 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2018, The Linux Foundation. All rights reserved.
- */
-
-#include "gsi_emulation.h"
-
-/*
- * *****************************************************************************
- * The following used to set up the EMULATION interrupt controller...
- * *****************************************************************************
- */
-int setup_emulator_cntrlr(
-	void __iomem *intcntrlr_base,
-	u32           intcntrlr_mem_size)
-{
-	uint32_t val, ver, intrCnt, rangeCnt, range;
-
-	val = gsi_emu_readl(intcntrlr_base + GE_INT_CTL_VER_CNT);
-
-	intrCnt  = val & 0xFFFF;
-	ver      = (val >> 16) & 0xFFFF;
-	rangeCnt = intrCnt / 32;
-
-	GSIDBG(
-	    "CTL_VER_CNT reg val(0x%x) intr cnt(%u) cntrlr ver(0x%x) rangeCnt(%u)\n",
-	    val, intrCnt, ver, rangeCnt);
-
-	/*
-	 * Verify the interrupt controller version
-	 */
-	if (ver == 0 || ver == 0xFFFF || ver < DEO_IC_INT_CTL_VER_MIN) {
-		GSIERR(
-		  "Error: invalid interrupt controller version 0x%x\n",
-		  ver);
-		return -GSI_STATUS_INVALID_PARAMS;
-	}
-
-	/*
-	 * Verify the interrupt count
-	 *
-	 * NOTE: intrCnt must be at least one block and multiple of 32
-	 */
-	if ((intrCnt % 32) != 0) {
-		GSIERR(
-		  "Invalid interrupt count read from HW 0x%04x\n",
-		  intrCnt);
-		return -GSI_STATUS_ERROR;
-	}
-
-	/*
-	 * Calculate number of ranges used, each range handles 32 int lines
-	 */
-	if (rangeCnt > DEO_IC_MAX_RANGE_CNT) {
-		GSIERR(
-		  "SW interrupt limit(%u) passed, increase DEO_IC_MAX_RANGE_CNT(%u)\n",
-		  rangeCnt,
-		  DEO_IC_MAX_RANGE_CNT);
-		return -GSI_STATUS_ERROR;
-	}
-
-	/*
-	 * Let's take the last register offset minus the first
-	 * register offset (ie. range) and compare it to the interrupt
-	 * controller's dtsi defined memory size.  The range better
-	 * fit within the size.
-	 */
-	val = GE_SOFT_INT_n(rangeCnt-1) - GE_INT_CTL_VER_CNT;
-	if (val > intcntrlr_mem_size) {
-		GSIERR(
-		    "Interrupt controller register range (%u) exceeds dtsi provisioned size (%u)\n",
-		    val, intcntrlr_mem_size);
-		return -GSI_STATUS_ERROR;
-	}
-
-	/*
-	 * The following will disable the emulators interrupt controller,
-	 * so that we can config it...
-	 */
-	GSIDBG("Writing GE_INT_MASTER_ENABLE\n");
-	gsi_emu_writel(
-		0x0,
-		intcntrlr_base + GE_INT_MASTER_ENABLE);
-
-	/*
-	 * Init register maps of all ranges
-	 */
-	for (range = 0; range < rangeCnt; range++) {
-		/*
-		 * Disable all int sources by setting all enable clear bits
-		 */
-		GSIDBG("Writing GE_INT_ENABLE_CLEAR_n(%u)\n", range);
-		gsi_emu_writel(
-		    0xFFFFFFFF,
-		    intcntrlr_base + GE_INT_ENABLE_CLEAR_n(range));
-
-		/*
-		 * Clear all raw statuses
-		 */
-		GSIDBG("Writing GE_INT_CLEAR_n(%u)\n", range);
-		gsi_emu_writel(
-		    0xFFFFFFFF,
-		    intcntrlr_base + GE_INT_CLEAR_n(range));
-
-		/*
-		 * Init all int types
-		 */
-		GSIDBG("Writing GE_INT_TYPE_n(%u)\n", range);
-		gsi_emu_writel(
-		    0x0,
-		    intcntrlr_base + GE_INT_TYPE_n(range));
-	}
-
-	/*
-	 * The following tells the interrupt controller to interrupt us
-	 * when it sees interupts from ipa and/or gsi.
-	 *
-	 * Interrupts:
-	 * ===================================================================
-	 * DUT0                       [  63 :   16 ]
-	 * ipa_irq                                        [ 3 : 0 ] <---HERE
-	 * ipa_gsi_bam_irq                                [ 7 : 4 ] <---HERE
-	 * ipa_bam_apu_sec_error_irq                      [ 8 ]
-	 * ipa_bam_apu_non_sec_error_irq                  [ 9 ]
-	 * ipa_bam_xpu2_msa_intr                          [ 10 ]
-	 * ipa_vmidmt_nsgcfgirpt                          [ 11 ]
-	 * ipa_vmidmt_nsgirpt                             [ 12 ]
-	 * ipa_vmidmt_gcfgirpt                            [ 13 ]
-	 * ipa_vmidmt_girpt                               [ 14 ]
-	 * bam_xpu3_qad_non_secure_intr_sp                [ 15 ]
-	 */
-	GSIDBG("Writing GE_INT_ENABLE_n(0)\n");
-	gsi_emu_writel(
-	    0x00FF, /* See <---HERE above */
-	    intcntrlr_base + GE_INT_ENABLE_n(0));
-
-	/*
-	 * The following will enable the IC post config...
-	 */
-	GSIDBG("Writing GE_INT_MASTER_ENABLE\n");
-	gsi_emu_writel(
-	    0x1,
-	    intcntrlr_base + GE_INT_MASTER_ENABLE);
-
-	return 0;
-}
-
-/*
- * *****************************************************************************
- * The following for EMULATION hard irq...
- * *****************************************************************************
- */
-irqreturn_t emulator_hard_irq_isr(
-	int   irq,
-	void *ctxt)
-{
-	struct gsi_ctx *gsi_ctx_ptr = (struct gsi_ctx *) ctxt;
-
-	uint32_t val;
-
-	val = gsi_emu_readl(gsi_ctx_ptr->intcntrlr_base + GE_INT_MASTER_STATUS);
-
-	/*
-	 * If bit zero is set, interrupt is for us, hence return IRQ_NONE
-	 * when it's not set...
-	 */
-	if (!(val & 0x00000001))
-		return IRQ_NONE;
-
-	/*
-	 * The following will mask (ie. turn off) future interrupts from
-	 * the emulator's interrupt controller. It wil stay this way until
-	 * we turn back on...which will be done in the bottom half
-	 * (ie. emulator_soft_irq_isr)...
-	 */
-	gsi_emu_writel(
-		0x0,
-		gsi_ctx_ptr->intcntrlr_base + GE_INT_OUT_ENABLE);
-
-	return IRQ_WAKE_THREAD;
-}
-
-/*
- * *****************************************************************************
- * The following for EMULATION soft irq...
- * *****************************************************************************
- */
-irqreturn_t emulator_soft_irq_isr(
-	int   irq,
-	void *ctxt)
-{
-	struct gsi_ctx *gsi_ctx_ptr = (struct gsi_ctx *) ctxt;
-
-	irqreturn_t retVal = IRQ_HANDLED;
-	uint32_t	val;
-
-	val = gsi_emu_readl(gsi_ctx_ptr->intcntrlr_base + GE_IRQ_STATUS_n(0));
-
-	GSIDBG("Got irq(%d) with status(0x%08X)\n", irq, val);
-
-	if (val & 0xF0 && gsi_ctx_ptr->intcntrlr_gsi_isr) {
-		GSIDBG("Got gsi interrupt\n");
-		retVal = gsi_ctx_ptr->intcntrlr_gsi_isr(irq, ctxt);
-	}
-
-	if (val & 0x0F && gsi_ctx_ptr->intcntrlr_client_isr) {
-		GSIDBG("Got ipa interrupt\n");
-		retVal = gsi_ctx_ptr->intcntrlr_client_isr(irq, 0);
-	}
-
-	/*
-	 * The following will clear the interrupts...
-	 */
-	gsi_emu_writel(
-		0xFFFFFFFF,
-		gsi_ctx_ptr->intcntrlr_base + GE_INT_CLEAR_n(0));
-
-	/*
-	 * The following will unmask (ie. turn on) future interrupts from
-	 * the emulator's interrupt controller...
-	 */
-	gsi_emu_writel(
-		0x1,
-		gsi_ctx_ptr->intcntrlr_base + GE_INT_OUT_ENABLE);
-
-	return retVal;
-}

gsi/gsi_emulation.h

@@ -1,186 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
- */
-
-#if !defined(_GSI_EMULATION_H_)
-# define _GSI_EMULATION_H_
-
-# include <linux/interrupt.h>
-
-# include "gsi.h"
-# include "gsi_reg.h"
-# include "gsi_emulation_stubs.h"
-
-# define gsi_emu_readl(c)     (readl(c))
-# define gsi_emu_writel(v, c) ({ __iowmb(); writel_relaxed((v), (c)); })
-
-# define CNTRLR_BASE 0
-
-/*
- * The following file contains definitions and declarations that are
- * germane only to the IPA emulation system, which is run from an X86
- * environment.  Declaration's for non-X86 (ie. arm) are merely stubs
- * to facilitate compile and link.
- *
- * Interrupt controller registers.
- * Descriptions taken from the EMULATION interrupt controller SWI.
- * - There is only one Master Enable register
- * - Each group of 32 interrupt lines (range) is controlled by 8 registers,
- *   which are consecutive in memory:
- *      GE_INT_ENABLE_n
- *      GE_INT_ENABLE_CLEAR_n
- *      GE_INT_ENABLE_SET_n
- *      GE_INT_TYPE_n
- *      GE_IRQ_STATUS_n
- *      GE_RAW_STATUS_n
- *      GE_INT_CLEAR_n
- *      GE_SOFT_INT_n
- * - After the above 8 registers, there are the registers of the next
- *   group (range) of 32 interrupt lines, and so on.
- */
-
-/** @brief The interrupt controller version and interrupt count register.
- *         Specifies interrupt controller version (upper 16 bits) and the
- *         number of interrupt lines supported by HW (lower 16 bits).
- */
-# define GE_INT_CTL_VER_CNT              \
-	(CNTRLR_BASE + 0x0000)
-
-/** @brief Enable or disable physical IRQ output signal to the system,
- *         not affecting any status registers.
- *
- *         0x0 : DISABLE IRQ output disabled
- *         0x1 : ENABLE  IRQ output enabled
- */
-# define GE_INT_OUT_ENABLE               \
-	(CNTRLR_BASE + 0x0004)
-
-/** @brief The IRQ master enable register.
- *         Bit #0: IRQ_ENABLE, set 0 to disable, 1 to enable.
- */
-# define GE_INT_MASTER_ENABLE            \
-	(CNTRLR_BASE + 0x0008)
-
-# define GE_INT_MASTER_STATUS            \
-	(CNTRLR_BASE + 0x000C)
-
-/** @brief Each bit disables (bit=0, default) or enables (bit=1) the
- *         corresponding interrupt source
- */
-# define GE_INT_ENABLE_n(n)              \
-	(CNTRLR_BASE + 0x0010 + 0x20 * (n))
-
-/** @brief Write bit=1 to clear (to 0) the corresponding bit(s) in INT_ENABLE.
- *         Does nothing for bit=0
- */
-# define GE_INT_ENABLE_CLEAR_n(n)        \
-	(CNTRLR_BASE + 0x0014 + 0x20 * (n))
-
-/** @brief Write bit=1 to set (to 1) the corresponding bit(s) in INT_ENABLE.
- *         Does nothing for bit=0
- */
-# define GE_INT_ENABLE_SET_n(n)          \
-	(CNTRLR_BASE + 0x0018 + 0x20 * (n))
-
-/** @brief Select level (bit=0, default) or edge (bit=1) sensitive input
- *         detection logic for each corresponding interrupt source
- */
-# define GE_INT_TYPE_n(n)                \
-	(CNTRLR_BASE + 0x001C + 0x20 * (n))
-
-/** @brief Shows the interrupt sources captured in RAW_STATUS that have been
- *         steered to irq_n by INT_SELECT. Interrupts must also be enabled by
- *         INT_ENABLE and MASTER_ENABLE. Read only register.
- *         Bit values: 1=active, 0=inactive
- */
-# define GE_IRQ_STATUS_n(n)                      \
-	(CNTRLR_BASE + 0x0020 + 0x20 * (n))
-
-/** @brief Shows the interrupt sources that have been latched by the input
- *         logic of the Interrupt Controller. Read only register.
- *         Bit values: 1=active, 0=inactive
- */
-# define GE_RAW_STATUS_n(n)                      \
-	(CNTRLR_BASE + 0x0024 + 0x20 * (n))
-
-/** @brief Write bit=1 to clear the corresponding bit(s) in RAW_STATUS.
- *         Does nothing for bit=0
- */
-# define GE_INT_CLEAR_n(n)               \
-	(CNTRLR_BASE + 0x0028 + 0x20 * (n))
-
-/** @brief Write bit=1 to set the corresponding bit(s) in RAW_STATUS.
- *         Does nothing for bit=0.
- *  @note  Only functional for edge detected interrupts
- */
-# define GE_SOFT_INT_n(n)                        \
-	(CNTRLR_BASE + 0x002C + 0x20 * (n))
-
-/** @brief Maximal number of ranges in SW. Each range supports 32 interrupt
- *         lines. If HW is extended considerably, increase this value
- */
-# define DEO_IC_MAX_RANGE_CNT            8
-
-/** @brief Size of the registers of one range in memory, in bytes */
-# define DEO_IC_RANGE_MEM_SIZE           32  /* SWI: 8 registers, no gaps */
-
-/** @brief Minimal Interrupt controller HW version */
-# define DEO_IC_INT_CTL_VER_MIN          0x0102
-
-
-#if defined(CONFIG_IPA_EMULATION) /* declarations to follow */
-
-/*
- * *****************************************************************************
- * The following used to set up the EMULATION interrupt controller...
- * *****************************************************************************
- */
-int setup_emulator_cntrlr(
-	void __iomem *intcntrlr_base,
-	u32           intcntrlr_mem_size);
-
-/*
- * *****************************************************************************
- * The following for EMULATION hard irq...
- * *****************************************************************************
- */
-irqreturn_t emulator_hard_irq_isr(
-	int   irq,
-	void *ctxt);
-
-/*
- * *****************************************************************************
- * The following for EMULATION soft irq...
- * *****************************************************************************
- */
-irqreturn_t emulator_soft_irq_isr(
-	int   irq,
-	void *ctxt);
-
-# else /* #if !defined(CONFIG_IPA_EMULATION) then definitions to follow */
-
-static inline int setup_emulator_cntrlr(
-	void __iomem *intcntrlr_base,
-	u32           intcntrlr_mem_size)
-{
-	return 0;
-}
-
-static inline irqreturn_t emulator_hard_irq_isr(
-	int   irq,
-	void *ctxt)
-{
-	return IRQ_NONE;
-}
-
-static inline irqreturn_t emulator_soft_irq_isr(
-	int   irq,
-	void *ctxt)
-{
-	return IRQ_HANDLED;
-}
-
-# endif /* #if defined(CONFIG_IPA_EMULATION) */
-
-#endif /* #if !defined(_GSI_EMULATION_H_) */

gsi/gsi_emulation_stubs.h

@@ -1,12 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
- */
-
-#if !defined(_GSI_EMULATION_STUBS_H_)
-# define _GSI_EMULATION_STUBS_H_
-
-# include <asm/barrier.h>
-# define __iowmb()       wmb() /* used in gsi.h */
-
-#endif /* #if !defined(_GSI_EMULATION_STUBS_H_) */

gsi/gsi_reg.h

@@ -1,30 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
- */
-
-#ifndef __GSI_REG_H__
-#define __GSI_REG_H__
-
-enum gsi_register_ver {
-	GSI_REGISTER_VER_1 = 0,
-	GSI_REGISTER_VER_2 = 1,
-	GSI_REGISTER_MAX,
-};
-
-#ifdef GSI_REGISTER_VER_CURRENT
-#error GSI_REGISTER_VER_CURRENT already defined
-#endif
-
-#ifdef CONFIG_GSI_REGISTER_VERSION_2
-#include "gsi_reg_v2.h"
-#define GSI_REGISTER_VER_CURRENT GSI_REGISTER_VER_2
-#endif
-
-/* The default is V1 */
-#ifndef GSI_REGISTER_VER_CURRENT
-#include "gsi_reg_v1.h"
-#define GSI_REGISTER_VER_CURRENT GSI_REGISTER_VER_1
-#endif
-
-#endif /* __GSI_REG_H__ */

gsi/gsi_reg_v1.h

@@ -1,1098 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2015-2019, The Linux Foundation. All rights reserved.
- */
-
-#ifndef __GSI_REG_V1_H__
-#define __GSI_REG_V1_H__
-
-#define GSI_GSI_REG_BASE_OFFS 0
-
-#define GSI_GSI_CFG_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x00000000)
-#define GSI_V2_5_GSI_CFG_SLEEP_CLK_DIV_BMSK 0xf00
-#define GSI_V2_5_GSI_CFG_SLEEP_CLK_DIV_SHFT 0x8
-#define GSI_GSI_CFG_BP_MTRIX_DISABLE_BMSK 0x20
-#define GSI_GSI_CFG_BP_MTRIX_DISABLE_SHFT 0x5
-#define GSI_GSI_CFG_GSI_PWR_CLPS_BMSK 0x10
-#define GSI_GSI_CFG_GSI_PWR_CLPS_SHFT 0x4
-#define GSI_GSI_CFG_UC_IS_MCS_BMSK 0x8
-#define GSI_GSI_CFG_UC_IS_MCS_SHFT 0x3
-#define GSI_GSI_CFG_DOUBLE_MCS_CLK_FREQ_BMSK 0x4
-#define GSI_GSI_CFG_DOUBLE_MCS_CLK_FREQ_SHFT 0x2
-#define GSI_GSI_CFG_MCS_ENABLE_BMSK 0x2
-#define GSI_GSI_CFG_MCS_ENABLE_SHFT 0x1
-#define GSI_GSI_CFG_GSI_ENABLE_BMSK 0x1
-#define GSI_GSI_CFG_GSI_ENABLE_SHFT 0x0
-
-#define GSI_GSI_MCS_CFG_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000B000)
-#define GSI_GSI_MCS_CFG_MCS_ENABLE_BMSK 0x1
-#define GSI_GSI_MCS_CFG_MCS_ENABLE_SHFT 0x0
-
-#define GSI_GSI_PERIPH_BASE_ADDR_LSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x00000018)
-#define GSI_GSI_PERIPH_BASE_ADDR_LSB_RMSK 0xffffffff
-#define GSI_GSI_PERIPH_BASE_ADDR_LSB_BASE_ADDR_BMSK 0xffffffff
-#define GSI_GSI_PERIPH_BASE_ADDR_LSB_BASE_ADDR_SHFT 0x0
-
-#define GSI_GSI_PERIPH_BASE_ADDR_MSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000001c)
-#define GSI_GSI_PERIPH_BASE_ADDR_MSB_RMSK 0xffffffff
-#define GSI_GSI_PERIPH_BASE_ADDR_MSB_BASE_ADDR_BMSK 0xffffffff
-#define GSI_GSI_PERIPH_BASE_ADDR_MSB_BASE_ADDR_SHFT 0x0
-
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_LSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000a0)
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_LSB_RMSK 0x3ffc1047
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_LSB_TLV_INT_BMSK 0x3f000000
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_LSB_TLV_INT_SHFT 0x18
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_LSB_CSR_INT_BMSK 0xfc0000
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_LSB_CSR_INT_SHFT 0x12
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_LSB_INT_END_INT_BMSK 0x1000
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_LSB_INT_END_INT_SHFT 0xc
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_LSB_EV_ENG_INT_BMSK 0x40
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_LSB_EV_ENG_INT_SHFT 0x6
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_LSB_REE_INT_BMSK 0x7
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_LSB_REE_INT_SHFT 0x0
-
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_MSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000a4)
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_MSB_RMSK 0xfc3041
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_MSB_UCONTROLLER_INT_BMSK 0xfc0000
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_MSB_UCONTROLLER_INT_SHFT 0x12
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_MSB_RD_WR_INT_BMSK 0x3000
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_MSB_RD_WR_INT_SHFT 0xc
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_MSB_DB_ENG_INT_BMSK 0x40
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_MSB_DB_ENG_INT_SHFT 0x6
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_MSB_TIMER_INT_BMSK 0x1
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_MSB_TIMER_INT_SHFT 0x0
-
-#define GSI_IC_GEN_EVNT_BCK_PRS_LSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000a8)
-#define GSI_IC_GEN_EVNT_BCK_PRS_LSB_RMSK 0x3ffc1047
-#define GSI_IC_GEN_EVNT_BCK_PRS_LSB_TLV_INT_BMSK 0x3f000000
-#define GSI_IC_GEN_EVNT_BCK_PRS_LSB_TLV_INT_SHFT 0x18
-#define GSI_IC_GEN_EVNT_BCK_PRS_LSB_CSR_INT_BMSK 0xfc0000
-#define GSI_IC_GEN_EVNT_BCK_PRS_LSB_CSR_INT_SHFT 0x12
-#define GSI_IC_GEN_EVNT_BCK_PRS_LSB_INT_END_INT_BMSK 0x1000
-#define GSI_IC_GEN_EVNT_BCK_PRS_LSB_INT_END_INT_SHFT 0xc
-#define GSI_IC_GEN_EVNT_BCK_PRS_LSB_EV_ENG_INT_BMSK 0x40
-#define GSI_IC_GEN_EVNT_BCK_PRS_LSB_EV_ENG_INT_SHFT 0x6
-#define GSI_IC_GEN_EVNT_BCK_PRS_LSB_REE_INT_BMSK 0x7
-#define GSI_IC_GEN_EVNT_BCK_PRS_LSB_REE_INT_SHFT 0x0
-
-#define GSI_IC_GEN_EVNT_BCK_PRS_MSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000ac)
-#define GSI_IC_GEN_EVNT_BCK_PRS_MSB_RMSK 0xfc3041
-#define GSI_IC_GEN_EVNT_BCK_PRS_MSB_UCONTROLLER_INT_BMSK 0xfc0000
-#define GSI_IC_GEN_EVNT_BCK_PRS_MSB_UCONTROLLER_INT_SHFT 0x12
-#define GSI_IC_GEN_EVNT_BCK_PRS_MSB_RD_WR_INT_BMSK 0x3000
-#define GSI_IC_GEN_EVNT_BCK_PRS_MSB_RD_WR_INT_SHFT 0xc
-#define GSI_IC_GEN_EVNT_BCK_PRS_MSB_DB_ENG_INT_BMSK 0x40
-#define GSI_IC_GEN_EVNT_BCK_PRS_MSB_DB_ENG_INT_SHFT 0x6
-#define GSI_IC_GEN_EVNT_BCK_PRS_MSB_TIMER_INT_BMSK 0x1
-#define GSI_IC_GEN_EVNT_BCK_PRS_MSB_TIMER_INT_SHFT 0x0
-
-#define GSI_IC_GEN_INT_BCK_PRS_LSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000b0)
-#define GSI_IC_GEN_INT_BCK_PRS_LSB_RMSK 0x3ffc1047
-#define GSI_IC_GEN_INT_BCK_PRS_LSB_TLV_INT_BMSK 0x3f000000
-#define GSI_IC_GEN_INT_BCK_PRS_LSB_TLV_INT_SHFT 0x18
-#define GSI_IC_GEN_INT_BCK_PRS_LSB_CSR_INT_BMSK 0xfc0000
-#define GSI_IC_GEN_INT_BCK_PRS_LSB_CSR_INT_SHFT 0x12
-#define GSI_IC_GEN_INT_BCK_PRS_LSB_INT_END_INT_BMSK 0x1000
-#define GSI_IC_GEN_INT_BCK_PRS_LSB_INT_END_INT_SHFT 0xc
-#define GSI_IC_GEN_INT_BCK_PRS_LSB_EV_ENG_INT_BMSK 0x40
-#define GSI_IC_GEN_INT_BCK_PRS_LSB_EV_ENG_INT_SHFT 0x6
-#define GSI_IC_GEN_INT_BCK_PRS_LSB_REE_INT_BMSK 0x7
-#define GSI_IC_GEN_INT_BCK_PRS_LSB_REE_INT_SHFT 0x0
-
-#define GSI_IC_GEN_INT_BCK_PRS_MSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000b4)
-#define GSI_IC_GEN_INT_BCK_PRS_MSB_RMSK 0xfc3041
-#define GSI_IC_GEN_INT_BCK_PRS_MSB_UCONTROLLER_INT_BMSK 0xfc0000
-#define GSI_IC_GEN_INT_BCK_PRS_MSB_UCONTROLLER_INT_SHFT 0x12
-#define GSI_IC_GEN_INT_BCK_PRS_MSB_RD_WR_INT_BMSK 0x3000
-#define GSI_IC_GEN_INT_BCK_PRS_MSB_RD_WR_INT_SHFT 0xc
-#define GSI_IC_GEN_INT_BCK_PRS_MSB_DB_ENG_INT_BMSK 0x40
-#define GSI_IC_GEN_INT_BCK_PRS_MSB_DB_ENG_INT_SHFT 0x6
-#define GSI_IC_GEN_INT_BCK_PRS_MSB_TIMER_INT_BMSK 0x1
-#define GSI_IC_GEN_INT_BCK_PRS_MSB_TIMER_INT_SHFT 0x0
-
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_LSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000b8)
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_LSB_RMSK 0x3ffc1047
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_LSB_TLV_INT_BMSK 0x3f000000
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_LSB_TLV_INT_SHFT 0x18
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_LSB_CSR_INT_BMSK 0xfc0000
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_LSB_CSR_INT_SHFT 0x12
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_LSB_INT_END_INT_BMSK 0x1000
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_LSB_INT_END_INT_SHFT 0xc
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_LSB_EV_ENG_INT_BMSK 0x40
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_LSB_EV_ENG_INT_SHFT 0x6
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_LSB_REE_INT_BMSK 0x7
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_LSB_REE_INT_SHFT 0x0
-
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_MSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000bc)
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_MSB_RMSK 0xfc3041
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_MSB_UCONTROLLER_INT_BMSK 0xfc0000
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_MSB_UCONTROLLER_INT_SHFT 0x12
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_MSB_RD_WR_INT_BMSK 0x3000
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_MSB_RD_WR_INT_SHFT 0xc
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_MSB_DB_ENG_INT_BMSK 0x40
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_MSB_DB_ENG_INT_SHFT 0x6
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_MSB_TIMER_INT_BMSK 0x1
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_MSB_TIMER_INT_SHFT 0x0
-
-#define GSI_IC_PROCESS_DESC_BCK_PRS_LSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000c0)
-#define GSI_IC_PROCESS_DESC_BCK_PRS_LSB_RMSK 0x3ffc1047
-#define GSI_IC_PROCESS_DESC_BCK_PRS_LSB_TLV_INT_BMSK 0x3f000000
-#define GSI_IC_PROCESS_DESC_BCK_PRS_LSB_TLV_INT_SHFT 0x18
-#define GSI_IC_PROCESS_DESC_BCK_PRS_LSB_CSR_INT_BMSK 0xfc0000
-#define GSI_IC_PROCESS_DESC_BCK_PRS_LSB_CSR_INT_SHFT 0x12
-#define GSI_IC_PROCESS_DESC_BCK_PRS_LSB_INT_END_INT_BMSK 0x1000
-#define GSI_IC_PROCESS_DESC_BCK_PRS_LSB_INT_END_INT_SHFT 0xc
-#define GSI_IC_PROCESS_DESC_BCK_PRS_LSB_EV_ENG_INT_BMSK 0x40
-#define GSI_IC_PROCESS_DESC_BCK_PRS_LSB_EV_ENG_INT_SHFT 0x6
-#define GSI_IC_PROCESS_DESC_BCK_PRS_LSB_REE_INT_BMSK 0x7
-#define GSI_IC_PROCESS_DESC_BCK_PRS_LSB_REE_INT_SHFT 0x0
-
-#define GSI_IC_PROCESS_DESC_BCK_PRS_MSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000c4)
-#define GSI_IC_PROCESS_DESC_BCK_PRS_MSB_RMSK 0xfc3041
-#define GSI_IC_PROCESS_DESC_BCK_PRS_MSB_UCONTROLLER_INT_BMSK 0xfc0000
-#define GSI_IC_PROCESS_DESC_BCK_PRS_MSB_UCONTROLLER_INT_SHFT 0x12
-#define GSI_IC_PROCESS_DESC_BCK_PRS_MSB_RD_WR_INT_BMSK 0x3000
-#define GSI_IC_PROCESS_DESC_BCK_PRS_MSB_RD_WR_INT_SHFT 0xc
-#define GSI_IC_PROCESS_DESC_BCK_PRS_MSB_DB_ENG_INT_BMSK 0x40
-#define GSI_IC_PROCESS_DESC_BCK_PRS_MSB_DB_ENG_INT_SHFT 0x6
-#define GSI_IC_PROCESS_DESC_BCK_PRS_MSB_TIMER_INT_BMSK 0x1
-#define GSI_IC_PROCESS_DESC_BCK_PRS_MSB_TIMER_INT_SHFT 0x0
-
-#define GSI_IC_TLV_STOP_BCK_PRS_LSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000c8)
-#define GSI_IC_TLV_STOP_BCK_PRS_LSB_RMSK 0x3ffc1047
-#define GSI_IC_TLV_STOP_BCK_PRS_LSB_TLV_INT_BMSK 0x3f000000
-#define GSI_IC_TLV_STOP_BCK_PRS_LSB_TLV_INT_SHFT 0x18
-#define GSI_IC_TLV_STOP_BCK_PRS_LSB_CSR_INT_BMSK 0xfc0000
-#define GSI_IC_TLV_STOP_BCK_PRS_LSB_CSR_INT_SHFT 0x12
-#define GSI_IC_TLV_STOP_BCK_PRS_LSB_INT_END_INT_BMSK 0x1000
-#define GSI_IC_TLV_STOP_BCK_PRS_LSB_INT_END_INT_SHFT 0xc
-#define GSI_IC_TLV_STOP_BCK_PRS_LSB_EV_ENG_INT_BMSK 0x40
-#define GSI_IC_TLV_STOP_BCK_PRS_LSB_EV_ENG_INT_SHFT 0x6
-#define GSI_IC_TLV_STOP_BCK_PRS_LSB_REE_INT_BMSK 0x7
-#define GSI_IC_TLV_STOP_BCK_PRS_LSB_REE_INT_SHFT 0x0
-
-#define GSI_IC_TLV_STOP_BCK_PRS_MSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000cc)
-#define GSI_IC_TLV_STOP_BCK_PRS_MSB_RMSK 0xfc3041
-#define GSI_IC_TLV_STOP_BCK_PRS_MSB_UCONTROLLER_INT_BMSK 0xfc0000
-#define GSI_IC_TLV_STOP_BCK_PRS_MSB_UCONTROLLER_INT_SHFT 0x12
-#define GSI_IC_TLV_STOP_BCK_PRS_MSB_RD_WR_INT_BMSK 0x3000
-#define GSI_IC_TLV_STOP_BCK_PRS_MSB_RD_WR_INT_SHFT 0xc
-#define GSI_IC_TLV_STOP_BCK_PRS_MSB_DB_ENG_INT_BMSK 0x40
-#define GSI_IC_TLV_STOP_BCK_PRS_MSB_DB_ENG_INT_SHFT 0x6
-#define GSI_IC_TLV_STOP_BCK_PRS_MSB_TIMER_INT_BMSK 0x1
-#define GSI_IC_TLV_STOP_BCK_PRS_MSB_TIMER_INT_SHFT 0x0
-
-#define GSI_IC_TLV_RESET_BCK_PRS_LSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000d0)
-#define GSI_IC_TLV_RESET_BCK_PRS_LSB_RMSK 0x3ffc1047
-#define GSI_IC_TLV_RESET_BCK_PRS_LSB_TLV_INT_BMSK 0x3f000000
-#define GSI_IC_TLV_RESET_BCK_PRS_LSB_TLV_INT_SHFT 0x18
-#define GSI_IC_TLV_RESET_BCK_PRS_LSB_CSR_INT_BMSK 0xfc0000
-#define GSI_IC_TLV_RESET_BCK_PRS_LSB_CSR_INT_SHFT 0x12
-#define GSI_IC_TLV_RESET_BCK_PRS_LSB_INT_END_INT_BMSK 0x1000
-#define GSI_IC_TLV_RESET_BCK_PRS_LSB_INT_END_INT_SHFT 0xc
-#define GSI_IC_TLV_RESET_BCK_PRS_LSB_EV_ENG_INT_BMSK 0x40
-#define GSI_IC_TLV_RESET_BCK_PRS_LSB_EV_ENG_INT_SHFT 0x6
-#define GSI_IC_TLV_RESET_BCK_PRS_LSB_REE_INT_BMSK 0x7
-#define GSI_IC_TLV_RESET_BCK_PRS_LSB_REE_INT_SHFT 0x0
-
-#define GSI_IC_TLV_RESET_BCK_PRS_MSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000d4)
-#define GSI_IC_TLV_RESET_BCK_PRS_MSB_RMSK 0xfc3041
-#define GSI_IC_TLV_RESET_BCK_PRS_MSB_UCONTROLLER_INT_BMSK 0xfc0000
-#define GSI_IC_TLV_RESET_BCK_PRS_MSB_UCONTROLLER_INT_SHFT 0x12
-#define GSI_IC_TLV_RESET_BCK_PRS_MSB_RD_WR_INT_BMSK 0x3000
-#define GSI_IC_TLV_RESET_BCK_PRS_MSB_RD_WR_INT_SHFT 0xc
-#define GSI_IC_TLV_RESET_BCK_PRS_MSB_DB_ENG_INT_BMSK 0x40
-#define GSI_IC_TLV_RESET_BCK_PRS_MSB_DB_ENG_INT_SHFT 0x6
-#define GSI_IC_TLV_RESET_BCK_PRS_MSB_TIMER_INT_BMSK 0x1
-#define GSI_IC_TLV_RESET_BCK_PRS_MSB_TIMER_INT_SHFT 0x0
-
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_LSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000d8)
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_LSB_RMSK 0x3ffc1047
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_LSB_TLV_INT_BMSK 0x3f000000
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_LSB_TLV_INT_SHFT 0x18
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_LSB_CSR_INT_BMSK 0xfc0000
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_LSB_CSR_INT_SHFT 0x12
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_LSB_INT_END_INT_BMSK 0x1000
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_LSB_INT_END_INT_SHFT 0xc
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_LSB_EV_ENG_INT_BMSK 0x40
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_LSB_EV_ENG_INT_SHFT 0x6
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_LSB_REE_INT_BMSK 0x7
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_LSB_REE_INT_SHFT 0x0
-
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_MSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000dc)
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_MSB_RMSK 0xfc3041
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_MSB_UCONTROLLER_INT_BMSK 0xfc0000
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_MSB_UCONTROLLER_INT_SHFT 0x12
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_MSB_RD_WR_INT_BMSK 0x3000
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_MSB_RD_WR_INT_SHFT 0xc
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_MSB_DB_ENG_INT_BMSK 0x40
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_MSB_DB_ENG_INT_SHFT 0x6
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_MSB_TIMER_INT_BMSK 0x1
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_MSB_TIMER_INT_SHFT 0x0
-
-#define GSI_IC_READ_BCK_PRS_LSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000e0)
-#define GSI_IC_READ_BCK_PRS_LSB_RMSK 0x3ffc1047
-#define GSI_IC_READ_BCK_PRS_LSB_TLV_INT_BMSK 0x3f000000
-#define GSI_IC_READ_BCK_PRS_LSB_TLV_INT_SHFT 0x18
-#define GSI_IC_READ_BCK_PRS_LSB_CSR_INT_BMSK 0xfc0000
-#define GSI_IC_READ_BCK_PRS_LSB_CSR_INT_SHFT 0x12
-#define GSI_IC_READ_BCK_PRS_LSB_INT_END_INT_BMSK 0x1000
-#define GSI_IC_READ_BCK_PRS_LSB_INT_END_INT_SHFT 0xc
-#define GSI_IC_READ_BCK_PRS_LSB_EV_ENG_INT_BMSK 0x40
-#define GSI_IC_READ_BCK_PRS_LSB_EV_ENG_INT_SHFT 0x6
-#define GSI_IC_READ_BCK_PRS_LSB_REE_INT_BMSK 0x7
-#define GSI_IC_READ_BCK_PRS_LSB_REE_INT_SHFT 0x0
-
-#define GSI_IC_READ_BCK_PRS_MSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000e4)
-#define GSI_IC_READ_BCK_PRS_MSB_RMSK 0xfc3041
-#define GSI_IC_READ_BCK_PRS_MSB_UCONTROLLER_INT_BMSK 0xfc0000
-#define GSI_IC_READ_BCK_PRS_MSB_UCONTROLLER_INT_SHFT 0x12
-#define GSI_IC_READ_BCK_PRS_MSB_RD_WR_INT_BMSK 0x3000
-#define GSI_IC_READ_BCK_PRS_MSB_RD_WR_INT_SHFT 0xc
-#define GSI_IC_READ_BCK_PRS_MSB_DB_ENG_INT_BMSK 0x40
-#define GSI_IC_READ_BCK_PRS_MSB_DB_ENG_INT_SHFT 0x6
-#define GSI_IC_READ_BCK_PRS_MSB_TIMER_INT_BMSK 0x1
-#define GSI_IC_READ_BCK_PRS_MSB_TIMER_INT_SHFT 0x0
-
-#define GSI_IC_WRITE_BCK_PRS_LSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000e8)
-#define GSI_IC_WRITE_BCK_PRS_LSB_RMSK 0x3ffc1047
-#define GSI_IC_WRITE_BCK_PRS_LSB_TLV_INT_BMSK 0x3f000000
-#define GSI_IC_WRITE_BCK_PRS_LSB_TLV_INT_SHFT 0x18
-#define GSI_IC_WRITE_BCK_PRS_LSB_CSR_INT_BMSK 0xfc0000
-#define GSI_IC_WRITE_BCK_PRS_LSB_CSR_INT_SHFT 0x12
-#define GSI_IC_WRITE_BCK_PRS_LSB_INT_END_INT_BMSK 0x1000
-#define GSI_IC_WRITE_BCK_PRS_LSB_INT_END_INT_SHFT 0xc
-#define GSI_IC_WRITE_BCK_PRS_LSB_EV_ENG_INT_BMSK 0x40
-#define GSI_IC_WRITE_BCK_PRS_LSB_EV_ENG_INT_SHFT 0x6
-#define GSI_IC_WRITE_BCK_PRS_LSB_REE_INT_BMSK 0x7
-#define GSI_IC_WRITE_BCK_PRS_LSB_REE_INT_SHFT 0x0
-
-#define GSI_IC_WRITE_BCK_PRS_MSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000ec)
-#define GSI_IC_WRITE_BCK_PRS_MSB_RMSK 0xfc3041
-#define GSI_IC_WRITE_BCK_PRS_MSB_UCONTROLLER_INT_BMSK 0xfc0000
-#define GSI_IC_WRITE_BCK_PRS_MSB_UCONTROLLER_INT_SHFT 0x12
-#define GSI_IC_WRITE_BCK_PRS_MSB_RD_WR_INT_BMSK 0x3000
-#define GSI_IC_WRITE_BCK_PRS_MSB_RD_WR_INT_SHFT 0xc
-#define GSI_IC_WRITE_BCK_PRS_MSB_DB_ENG_INT_BMSK 0x40
-#define GSI_IC_WRITE_BCK_PRS_MSB_DB_ENG_INT_SHFT 0x6
-#define GSI_IC_WRITE_BCK_PRS_MSB_TIMER_INT_BMSK 0x1
-#define GSI_IC_WRITE_BCK_PRS_MSB_TIMER_INT_SHFT 0x0
-
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_LSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000f0)
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_LSB_RMSK 0x3ffc1047
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_LSB_TLV_INT_BMSK 0x3f000000
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_LSB_TLV_INT_SHFT 0x18
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_LSB_CSR_INT_BMSK 0xfc0000
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_LSB_CSR_INT_SHFT 0x12
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_LSB_INT_END_INT_BMSK 0x1000
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_LSB_INT_END_INT_SHFT 0xc
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_LSB_EV_ENG_INT_BMSK 0x40
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_LSB_EV_ENG_INT_SHFT 0x6
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_LSB_REE_INT_BMSK 0x7
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_LSB_REE_INT_SHFT 0x0
-
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_MSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000f4)
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_MSB_RMSK 0xfc3041
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_MSB_UCONTROLLER_INT_BMSK 0xfc0000
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_MSB_UCONTROLLER_INT_SHFT 0x12
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_MSB_RD_WR_INT_BMSK 0x3000
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_MSB_RD_WR_INT_SHFT 0xc
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_MSB_DB_ENG_INT_BMSK 0x40
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_MSB_DB_ENG_INT_SHFT 0x6
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_MSB_TIMER_INT_BMSK 0x1
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_MSB_TIMER_INT_SHFT 0x0
-
-#define GSI_GSI_IRAM_PTR_CH_CMD_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x00000400)
-#define GSI_GSI_IRAM_PTR_CH_CMD_RMSK 0xfff
-#define GSI_GSI_IRAM_PTR_CH_CMD_IRAM_PTR_BMSK 0xfff
-#define GSI_GSI_IRAM_PTR_CH_CMD_IRAM_PTR_SHFT 0x0
-
-#define GSI_GSI_IRAM_PTR_EE_GENERIC_CMD_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x00000404)
-#define GSI_GSI_IRAM_PTR_EE_GENERIC_CMD_RMSK 0xfff
-#define GSI_GSI_IRAM_PTR_EE_GENERIC_CMD_IRAM_PTR_BMSK 0xfff
-#define GSI_GSI_IRAM_PTR_EE_GENERIC_CMD_IRAM_PTR_SHFT 0x0
-
-#define GSI_V2_5_GSI_IRAM_PTR_TLV_CH_NOT_FULL_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x00000408)
-#define GSI_V2_5_GSI_IRAM_PTR_TLV_CH_NOT_FULL_RMSK 0xfff
-#define GSI_V2_5_GSI_IRAM_PTR_TLV_CH_NOT_FULL_IRAM_PTR_BMSK 0xfff
-#define GSI_V2_5_GSI_IRAM_PTR_TLV_CH_NOT_FULL_IRAM_PTR_SHFT 0x0
-
-#define GSI_GSI_IRAM_PTR_CH_DB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x00000418)
-#define GSI_GSI_IRAM_PTR_CH_DB_RMSK 0xfff
-#define GSI_GSI_IRAM_PTR_CH_DB_IRAM_PTR_BMSK 0xfff
-#define GSI_GSI_IRAM_PTR_CH_DB_IRAM_PTR_SHFT 0x0
-
-#define GSI_GSI_IRAM_PTR_EV_DB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000041c)
-#define GSI_GSI_IRAM_PTR_EV_DB_RMSK 0xfff
-#define GSI_GSI_IRAM_PTR_EV_DB_IRAM_PTR_BMSK 0xfff
-#define GSI_GSI_IRAM_PTR_EV_DB_IRAM_PTR_SHFT 0x0
-
-#define GSI_GSI_IRAM_PTR_NEW_RE_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x00000420)
-#define GSI_GSI_IRAM_PTR_NEW_RE_RMSK 0xfff
-#define GSI_GSI_IRAM_PTR_NEW_RE_IRAM_PTR_BMSK 0xfff
-#define GSI_GSI_IRAM_PTR_NEW_RE_IRAM_PTR_SHFT 0x0
-
-#define GSI_GSI_IRAM_PTR_CH_DIS_COMP_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x00000424)
-#define GSI_GSI_IRAM_PTR_CH_DIS_COMP_RMSK 0xfff
-#define GSI_GSI_IRAM_PTR_CH_DIS_COMP_IRAM_PTR_BMSK 0xfff
-#define GSI_GSI_IRAM_PTR_CH_DIS_COMP_IRAM_PTR_SHFT 0x0
-
-#define GSI_GSI_IRAM_PTR_CH_EMPTY_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x00000428)
-#define GSI_GSI_IRAM_PTR_CH_EMPTY_RMSK 0xfff
-#define GSI_GSI_IRAM_PTR_CH_EMPTY_IRAM_PTR_BMSK 0xfff
-#define GSI_GSI_IRAM_PTR_CH_EMPTY_IRAM_PTR_SHFT 0x0
-
-#define GSI_GSI_IRAM_PTR_EVENT_GEN_COMP_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000042c)
-#define GSI_GSI_IRAM_PTR_EVENT_GEN_COMP_RMSK 0xfff
-#define GSI_GSI_IRAM_PTR_EVENT_GEN_COMP_IRAM_PTR_BMSK 0xfff
-#define GSI_GSI_IRAM_PTR_EVENT_GEN_COMP_IRAM_PTR_SHFT 0x0
-
-#define GSI_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_0_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x00000430)
-#define GSI_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_0_RMSK 0xfff
-#define GSI_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_0_IRAM_PTR_BMSK 0xfff
-#define GSI_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_0_IRAM_PTR_SHFT 0x0
-
-#define GSI_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_2_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x00000434)
-#define GSI_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_2_RMSK 0xfff
-#define GSI_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_2_IRAM_PTR_BMSK 0xfff
-#define GSI_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_2_IRAM_PTR_SHFT 0x0
-
-#define GSI_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_1_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x00000438)
-#define GSI_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_1_RMSK 0xfff
-#define GSI_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_1_IRAM_PTR_BMSK 0xfff
-#define GSI_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_1_IRAM_PTR_SHFT 0x0
-
-#define GSI_GSI_IRAM_PTR_TIMER_EXPIRED_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000043c)
-#define GSI_GSI_IRAM_PTR_TIMER_EXPIRED_RMSK 0xfff
-#define GSI_GSI_IRAM_PTR_TIMER_EXPIRED_IRAM_PTR_BMSK 0xfff
-#define GSI_GSI_IRAM_PTR_TIMER_EXPIRED_IRAM_PTR_SHFT 0x0
-
-#define GSI_GSI_IRAM_PTR_WRITE_ENG_COMP_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x00000440)
-#define GSI_GSI_IRAM_PTR_WRITE_ENG_COMP_RMSK 0xfff
-#define GSI_GSI_IRAM_PTR_WRITE_ENG_COMP_IRAM_PTR_BMSK 0xfff
-#define GSI_GSI_IRAM_PTR_WRITE_ENG_COMP_IRAM_PTR_SHFT 0x0
-
-#define GSI_GSI_IRAM_PTR_READ_ENG_COMP_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x00000444)
-#define GSI_GSI_IRAM_PTR_READ_ENG_COMP_RMSK 0xfff
-#define GSI_GSI_IRAM_PTR_READ_ENG_COMP_IRAM_PTR_BMSK 0xfff
-#define GSI_GSI_IRAM_PTR_READ_ENG_COMP_IRAM_PTR_SHFT 0x0
-
-#define GSI_GSI_IRAM_PTR_UC_GP_INT_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x00000448)
-#define GSI_GSI_IRAM_PTR_UC_GP_INT_RMSK 0xfff
-#define GSI_GSI_IRAM_PTR_UC_GP_INT_IRAM_PTR_BMSK 0xfff
-#define GSI_GSI_IRAM_PTR_UC_GP_INT_IRAM_PTR_SHFT 0x0
-
-/* Real H/W register name is with STOPPED with single P */
-#define GSI_GSI_IRAM_PTR_INT_MOD_STOPPED_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000044c)
-#define GSI_GSI_IRAM_PTR_INT_MOD_STOPPED_RMSK 0xfff
-#define GSI_GSI_IRAM_PTR_INT_MOD_STOPPED_IRAM_PTR_BMSK 0xfff
-#define GSI_GSI_IRAM_PTR_INT_MOD_STOPPED_IRAM_PTR_SHFT 0x0
-
-#define GSI_GSI_INST_RAM_n_WORD_SZ 0x4
-#define GSI_GSI_INST_RAM_n_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00004000 + GSI_GSI_INST_RAM_n_WORD_SZ * (n))
-#define GSI_V2_5_GSI_INST_RAM_n_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001b000 + GSI_GSI_INST_RAM_n_WORD_SZ * (n))
-#define GSI_GSI_INST_RAM_n_RMSK 0xffffffff
-#define GSI_GSI_INST_RAM_n_MAXn 4095
-#define GSI_V2_0_GSI_INST_RAM_n_MAXn 6143
-#define GSI_V2_2_GSI_INST_RAM_n_MAXn 4095
-#define GSI_V2_5_GSI_INST_RAM_n_MAXn 8191
-#define GSI_V2_7_GSI_INST_RAM_n_MAXn 5119
-
-#define GSI_GSI_INST_RAM_n_INST_BYTE_3_BMSK 0xff000000
-#define GSI_GSI_INST_RAM_n_INST_BYTE_3_SHFT 0x18
-#define GSI_GSI_INST_RAM_n_INST_BYTE_2_BMSK 0xff0000
-#define GSI_GSI_INST_RAM_n_INST_BYTE_2_SHFT 0x10
-#define GSI_GSI_INST_RAM_n_INST_BYTE_1_BMSK 0xff00
-#define GSI_GSI_INST_RAM_n_INST_BYTE_1_SHFT 0x8
-#define GSI_GSI_INST_RAM_n_INST_BYTE_0_BMSK 0xff
-#define GSI_GSI_INST_RAM_n_INST_BYTE_0_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_k_CNTXT_0_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001c000 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_GSI_CH_k_CNTXT_0_ELEMENT_SIZE_BMSK 0xff000000
-#define GSI_EE_n_GSI_CH_k_CNTXT_0_ELEMENT_SIZE_SHFT 0x18
-#define GSI_EE_n_GSI_CH_k_CNTXT_0_CHSTATE_BMSK 0xf00000
-#define GSI_EE_n_GSI_CH_k_CNTXT_0_CHSTATE_SHFT 0x14
-#define GSI_EE_n_GSI_CH_k_CNTXT_0_ERINDEX_BMSK 0x7c000
-#define GSI_EE_n_GSI_CH_k_CNTXT_0_ERINDEX_SHFT 0xe
-#define GSI_V2_5_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_PROTOCOL_MSB_BMSK 0x2000
-#define GSI_V2_5_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_PROTOCOL_MSB_SHFT 0xd
-#define GSI_EE_n_GSI_CH_k_CNTXT_0_CHID_BMSK 0x1f00
-#define GSI_EE_n_GSI_CH_k_CNTXT_0_CHID_SHFT 0x8
-#define GSI_EE_n_GSI_CH_k_CNTXT_0_EE_BMSK 0xf0
-#define GSI_EE_n_GSI_CH_k_CNTXT_0_EE_SHFT 0x4
-#define GSI_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_DIR_BMSK 0x8
-#define GSI_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_DIR_SHFT 0x3
-#define GSI_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_PROTOCOL_BMSK 0x7
-#define GSI_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_PROTOCOL_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_k_CNTXT_1_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001c004 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_GSI_CH_k_CNTXT_1_R_LENGTH_BMSK 0xffff
-#define GSI_EE_n_GSI_CH_k_CNTXT_1_R_LENGTH_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_k_CNTXT_2_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001c008 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_GSI_CH_k_CNTXT_2_R_BASE_ADDR_LSBS_BMSK 0xffffffff
-#define GSI_EE_n_GSI_CH_k_CNTXT_2_R_BASE_ADDR_LSBS_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_k_CNTXT_3_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001c00c + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_GSI_CH_k_CNTXT_3_R_BASE_ADDR_MSBS_BMSK 0xffffffff
-#define GSI_EE_n_GSI_CH_k_CNTXT_3_R_BASE_ADDR_MSBS_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_k_CNTXT_4_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001c010 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_GSI_CH_k_CNTXT_4_READ_PTR_LSB_BMSK 0xffffffff
-#define GSI_EE_n_GSI_CH_k_CNTXT_4_READ_PTR_LSB_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_k_CNTXT_5_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001c014 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_GSI_CH_k_CNTXT_5_READ_PTR_MSB_BMSK 0xffffffff
-#define GSI_EE_n_GSI_CH_k_CNTXT_5_READ_PTR_MSB_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_k_CNTXT_6_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001c018 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_GSI_CH_k_CNTXT_6_WRITE_PTR_LSB_BMSK 0xffffffff
-#define GSI_EE_n_GSI_CH_k_CNTXT_6_WRITE_PTR_LSB_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_k_CNTXT_7_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001c01c + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_GSI_CH_k_CNTXT_7_WRITE_PTR_MSB_BMSK 0xffffffff
-#define GSI_EE_n_GSI_CH_k_CNTXT_7_WRITE_PTR_MSB_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001c054 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_RMSK 0xffff
-#define GSI_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_MAXk 30
-#define GSI_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_MAXn 3
-#define GSI_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_READ_PTR_BMSK 0xffff
-#define GSI_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_READ_PTR_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001c058 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_RMSK 0xffff
-#define GSI_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_MAXk 30
-#define GSI_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_MAXn 3
-#define GSI_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_RE_INTR_DB_BMSK 0xffff
-#define GSI_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_RE_INTR_DB_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_k_QOS_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001c05c + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_GSI_CH_k_QOS_RMSK 0x303
-#define GSI_EE_n_GSI_CH_k_QOS_MAXk 30
-#define GSI_EE_n_GSI_CH_k_QOS_MAXn 3
-#define GSI_EE_n_GSI_CH_k_QOS_USE_ESCAPE_BUF_ONLY_BMSK 0x400
-#define GSI_EE_n_GSI_CH_k_QOS_USE_ESCAPE_BUF_ONLY_SHFT 0xa
-#define GSI_EE_n_GSI_CH_k_QOS_USE_DB_ENG_BMSK 0x200
-#define GSI_EE_n_GSI_CH_k_QOS_USE_DB_ENG_SHFT 0x9
-#define GSI_EE_n_GSI_CH_k_QOS_MAX_PREFETCH_BMSK 0x100
-#define GSI_EE_n_GSI_CH_k_QOS_MAX_PREFETCH_SHFT 0x8
-#define GSI_EE_n_GSI_CH_k_QOS_WRR_WEIGHT_BMSK 0xf
-#define GSI_EE_n_GSI_CH_k_QOS_WRR_WEIGHT_SHFT 0x0
-
-#define GSI_V2_5_EE_n_GSI_CH_k_QOS_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000f05c + 0x4000 * (n) + 0x80 * (k))
-#define GSI_V2_5_EE_n_GSI_CH_k_QOS_EMPTY_LVL_THRSHOLD_BMSK 0xff0000
-#define GSI_V2_5_EE_n_GSI_CH_k_QOS_EMPTY_LVL_THRSHOLD_SHFT 0x10
-#define GSI_V2_5_EE_n_GSI_CH_k_QOS_PREFETCH_MODE_BMSK 0x3c00
-#define GSI_V2_5_EE_n_GSI_CH_k_QOS_PREFETCH_MODE_SHFT 0xa
-#define GSI_V2_5_EE_n_GSI_CH_k_QOS_USE_DB_ENG_BMSK 0x200
-#define GSI_V2_5_EE_n_GSI_CH_k_QOS_USE_DB_ENG_SHFT 0x9
-#define GSI_V2_5_EE_n_GSI_CH_k_QOS_MAX_PREFETCH_BMSK 0x100
-#define GSI_V2_5_EE_n_GSI_CH_k_QOS_MAX_PREFETCH_SHFT 0x8
-#define GSI_V2_5_EE_n_GSI_CH_k_QOS_WRR_WEIGHT_BMSK 0xf
-#define GSI_V2_5_EE_n_GSI_CH_k_QOS_WRR_WEIGHT_SHFT 0x0
-
-
-#define GSI_EE_n_GSI_CH_k_SCRATCH_0_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001c060 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_GSI_CH_k_SCRATCH_0_SCRATCH_BMSK 0xffffffff
-#define GSI_EE_n_GSI_CH_k_SCRATCH_0_SCRATCH_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_k_SCRATCH_1_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001c064 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_GSI_CH_k_SCRATCH_1_SCRATCH_BMSK 0xffffffff
-#define GSI_EE_n_GSI_CH_k_SCRATCH_1_SCRATCH_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_k_SCRATCH_2_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001c068 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_GSI_CH_k_SCRATCH_2_SCRATCH_BMSK 0xffffffff
-#define GSI_EE_n_GSI_CH_k_SCRATCH_2_SCRATCH_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_k_SCRATCH_3_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001c06c + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_GSI_CH_k_SCRATCH_3_SCRATCH_BMSK 0xffffffff
-#define GSI_EE_n_GSI_CH_k_SCRATCH_3_SCRATCH_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_CNTXT_0_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001d000 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_EV_CH_k_CNTXT_0_ELEMENT_SIZE_BMSK 0xff000000
-#define GSI_EE_n_EV_CH_k_CNTXT_0_ELEMENT_SIZE_SHFT 0x18
-#define GSI_EE_n_EV_CH_k_CNTXT_0_CHSTATE_BMSK 0xf00000
-#define GSI_EE_n_EV_CH_k_CNTXT_0_CHSTATE_SHFT 0x14
-#define GSI_EE_n_EV_CH_k_CNTXT_0_INTYPE_BMSK 0x10000
-#define GSI_EE_n_EV_CH_k_CNTXT_0_INTYPE_SHFT 0x10
-#define GSI_EE_n_EV_CH_k_CNTXT_0_EVCHID_BMSK 0xff00
-#define GSI_EE_n_EV_CH_k_CNTXT_0_EVCHID_SHFT 0x8
-#define GSI_EE_n_EV_CH_k_CNTXT_0_EE_BMSK 0xf0
-#define GSI_EE_n_EV_CH_k_CNTXT_0_EE_SHFT 0x4
-#define GSI_EE_n_EV_CH_k_CNTXT_0_CHTYPE_BMSK 0xf
-#define GSI_EE_n_EV_CH_k_CNTXT_0_CHTYPE_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_CNTXT_1_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001d004 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_EV_CH_k_CNTXT_1_R_LENGTH_BMSK 0xffff
-#define GSI_EE_n_EV_CH_k_CNTXT_1_R_LENGTH_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_CNTXT_2_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001d008 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_EV_CH_k_CNTXT_2_R_BASE_ADDR_LSBS_BMSK 0xffffffff
-#define GSI_EE_n_EV_CH_k_CNTXT_2_R_BASE_ADDR_LSBS_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_CNTXT_3_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001d00c + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_EV_CH_k_CNTXT_3_R_BASE_ADDR_MSBS_BMSK 0xffffffff
-#define GSI_EE_n_EV_CH_k_CNTXT_3_R_BASE_ADDR_MSBS_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_CNTXT_4_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001d010 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_EV_CH_k_CNTXT_4_READ_PTR_LSB_BMSK 0xffffffff
-#define GSI_EE_n_EV_CH_k_CNTXT_4_READ_PTR_LSB_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_CNTXT_5_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001d014 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_EV_CH_k_CNTXT_5_READ_PTR_MSB_BMSK 0xffffffff
-#define GSI_EE_n_EV_CH_k_CNTXT_5_READ_PTR_MSB_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_CNTXT_6_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001d018 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_EV_CH_k_CNTXT_6_WRITE_PTR_LSB_BMSK 0xffffffff
-#define GSI_EE_n_EV_CH_k_CNTXT_6_WRITE_PTR_LSB_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_CNTXT_7_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001d01c + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_EV_CH_k_CNTXT_7_WRITE_PTR_MSB_BMSK 0xffffffff
-#define GSI_EE_n_EV_CH_k_CNTXT_7_WRITE_PTR_MSB_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_CNTXT_8_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001d020 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_EV_CH_k_CNTXT_8_INT_MOD_CNT_BMSK 0xff000000
-#define GSI_EE_n_EV_CH_k_CNTXT_8_INT_MOD_CNT_SHFT 0x18
-#define GSI_EE_n_EV_CH_k_CNTXT_8_INT_MODC_BMSK 0xff0000
-#define GSI_EE_n_EV_CH_k_CNTXT_8_INT_MODC_SHFT 0x10
-#define GSI_EE_n_EV_CH_k_CNTXT_8_INT_MODT_BMSK 0xffff
-#define GSI_EE_n_EV_CH_k_CNTXT_8_INT_MODT_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_CNTXT_9_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001d024 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_EV_CH_k_CNTXT_9_INTVEC_BMSK 0xffffffff
-#define GSI_EE_n_EV_CH_k_CNTXT_9_INTVEC_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_CNTXT_10_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001d028 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_EV_CH_k_CNTXT_10_MSI_ADDR_LSB_BMSK 0xffffffff
-#define GSI_EE_n_EV_CH_k_CNTXT_10_MSI_ADDR_LSB_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_CNTXT_11_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001d02c + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_EV_CH_k_CNTXT_11_MSI_ADDR_MSB_BMSK 0xffffffff
-#define GSI_EE_n_EV_CH_k_CNTXT_11_MSI_ADDR_MSB_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_CNTXT_12_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001d030 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_EV_CH_k_CNTXT_12_RP_UPDATE_ADDR_LSB_BMSK 0xffffffff
-#define GSI_EE_n_EV_CH_k_CNTXT_12_RP_UPDATE_ADDR_LSB_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_CNTXT_13_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001d034 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_EV_CH_k_CNTXT_13_RP_UPDATE_ADDR_MSB_BMSK 0xffffffff
-#define GSI_EE_n_EV_CH_k_CNTXT_13_RP_UPDATE_ADDR_MSB_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_SCRATCH_0_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001d048 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_EV_CH_k_SCRATCH_0_SCRATCH_BMSK 0xffffffff
-#define GSI_EE_n_EV_CH_k_SCRATCH_0_SCRATCH_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_SCRATCH_1_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001d04c + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_EV_CH_k_SCRATCH_1_SCRATCH_BMSK 0xffffffff
-#define GSI_EE_n_EV_CH_k_SCRATCH_1_SCRATCH_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_k_DOORBELL_0_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001e000 + 0x4000 * (n) + 0x8 * (k))
-#define GSI_EE_n_GSI_CH_k_DOORBELL_0_WRITE_PTR_LSB_BMSK 0xffffffff
-#define GSI_EE_n_GSI_CH_k_DOORBELL_0_WRITE_PTR_LSB_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_k_DOORBELL_1_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001e004 + 0x4000 * (n) + 0x8 * (k))
-#define GSI_EE_n_GSI_CH_k_DOORBELL_1_WRITE_PTR_MSB_BMSK 0xffffffff
-#define GSI_EE_n_GSI_CH_k_DOORBELL_1_WRITE_PTR_MSB_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_DOORBELL_0_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001e100 + 0x4000 * (n) + 0x8 * (k))
-#define GSI_EE_n_EV_CH_k_DOORBELL_0_WRITE_PTR_LSB_BMSK 0xffffffff
-#define GSI_EE_n_EV_CH_k_DOORBELL_0_WRITE_PTR_LSB_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_DOORBELL_1_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001e104 + 0x4000 * (n) + 0x8 * (k))
-#define GSI_EE_n_EV_CH_k_DOORBELL_1_WRITE_PTR_MSB_BMSK 0xffffffff
-#define GSI_EE_n_EV_CH_k_DOORBELL_1_WRITE_PTR_MSB_SHFT 0x0
-
-#define GSI_EE_n_GSI_STATUS_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f000 + 0x4000 * (n))
-#define GSI_EE_n_GSI_STATUS_ENABLED_BMSK 0x1
-#define GSI_EE_n_GSI_STATUS_ENABLED_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_CMD_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f008 + 0x4000 * (n))
-#define GSI_EE_n_GSI_CH_CMD_OPCODE_BMSK 0xff000000
-#define GSI_EE_n_GSI_CH_CMD_OPCODE_SHFT 0x18
-#define GSI_EE_n_GSI_CH_CMD_CHID_BMSK 0xff
-#define GSI_EE_n_GSI_CH_CMD_CHID_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_CMD_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f010 + 0x4000 * (n))
-#define GSI_EE_n_EV_CH_CMD_OPCODE_BMSK 0xff000000
-#define GSI_EE_n_EV_CH_CMD_OPCODE_SHFT 0x18
-#define GSI_EE_n_EV_CH_CMD_CHID_BMSK 0xff
-#define GSI_EE_n_EV_CH_CMD_CHID_SHFT 0x0
-
-#define GSI_EE_n_GSI_EE_GENERIC_CMD_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f018 + 0x4000 * (n))
-#define GSI_EE_n_GSI_EE_GENERIC_CMD_OPCODE_BMSK 0x1f
-#define GSI_EE_n_GSI_EE_GENERIC_CMD_OPCODE_SHFT 0x0
-#define GSI_EE_n_GSI_EE_GENERIC_CMD_VIRT_CHAN_IDX_BMSK 0x3e0
-#define GSI_EE_n_GSI_EE_GENERIC_CMD_VIRT_CHAN_IDX_SHFT 0x5
-#define GSI_EE_n_GSI_EE_GENERIC_CMD_EE_BMSK 0x3c00
-#define GSI_EE_n_GSI_EE_GENERIC_CMD_EE_SHFT 0xa
-
-#define GSI_V1_0_EE_n_GSI_HW_PARAM_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f040 + 0x4000 * (n))
-#define GSI_V1_0_EE_n_GSI_HW_PARAM_PERIPH_SEC_GRP_BMSK 0x7c000000
-#define GSI_V1_0_EE_n_GSI_HW_PARAM_PERIPH_SEC_GRP_SHFT 0x1a
-#define GSI_V1_0_EE_n_GSI_HW_PARAM_USE_AXI_M_BMSK 0x2000000
-#define GSI_V1_0_EE_n_GSI_HW_PARAM_USE_AXI_M_SHFT 0x19
-#define GSI_V1_0_EE_n_GSI_HW_PARAM_PERIPH_CONF_ADDR_BUS_W_BMSK 0x1f00000
-#define GSI_V1_0_EE_n_GSI_HW_PARAM_PERIPH_CONF_ADDR_BUS_W_SHFT 0x14
-#define GSI_V1_0_EE_n_GSI_HW_PARAM_NUM_EES_BMSK 0xf0000
-#define GSI_V1_0_EE_n_GSI_HW_PARAM_NUM_EES_SHFT 0x10
-#define GSI_V1_0_EE_n_GSI_HW_PARAM_GSI_CH_NUM_BMSK 0xff00
-#define GSI_V1_0_EE_n_GSI_HW_PARAM_GSI_CH_NUM_SHFT 0x8
-#define GSI_V1_0_EE_n_GSI_HW_PARAM_GSI_EV_CH_NUM_BMSK 0xff
-#define GSI_V1_0_EE_n_GSI_HW_PARAM_GSI_EV_CH_NUM_SHFT 0x0
-
-#define GSI_V1_2_EE_n_GSI_HW_PARAM_0_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f038 + 0x4000 * (n))
-#define GSI_V1_2_EE_n_GSI_HW_PARAM_0_USE_AXI_M_BMSK 0x80000000
-#define GSI_V1_2_EE_n_GSI_HW_PARAM_0_USE_AXI_M_SHFT 0x1f
-#define GSI_V1_2_EE_n_GSI_HW_PARAM_0_PERIPH_SEC_GRP_BMSK 0x7c000000
-#define GSI_V1_2_EE_n_GSI_HW_PARAM_0_PERIPH_SEC_GRP_SHFT 0x1a
-#define GSI_V1_2_EE_n_GSI_HW_PARAM_0_PERIPH_CONF_ADDR_BUS_W_BMSK 0x3e00000
-#define GSI_V1_2_EE_n_GSI_HW_PARAM_0_PERIPH_CONF_ADDR_BUS_W_SHFT 0x15
-#define GSI_V1_2_EE_n_GSI_HW_PARAM_0_NUM_EES_BMSK 0x1f0000
-#define GSI_V1_2_EE_n_GSI_HW_PARAM_0_NUM_EES_SHFT 0x10
-#define GSI_V1_2_EE_n_GSI_HW_PARAM_0_GSI_CH_NUM_BMSK 0xff00
-#define GSI_V1_2_EE_n_GSI_HW_PARAM_0_GSI_CH_NUM_SHFT 0x8
-#define GSI_V1_2_EE_n_GSI_HW_PARAM_0_GSI_EV_CH_NUM_BMSK 0xff
-#define GSI_V1_2_EE_n_GSI_HW_PARAM_0_GSI_EV_CH_NUM_SHFT 0x0
-
-#define GSI_V1_3_EE_n_GSI_HW_PARAM_2_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f040 + 0x4000 * (n))
-#define GSI_V1_3_EE_n_GSI_HW_PARAM_2_RMSK 0x7fff
-#define GSI_V1_3_EE_n_GSI_HW_PARAM_2_MAXn 2
-#define GSI_V1_3_EE_n_GSI_HW_PARAM_2_GSI_CH_FULL_LOGIC_BMSK 0x4000
-#define GSI_V1_3_EE_n_GSI_HW_PARAM_2_GSI_CH_FULL_LOGIC_SHFT 0xe
-#define GSI_V1_3_EE_n_GSI_HW_PARAM_2_GSI_CH_PEND_TRANSLATE_BMSK 0x2000
-#define GSI_V1_3_EE_n_GSI_HW_PARAM_2_GSI_CH_PEND_TRANSLATE_SHFT 0xd
-#define GSI_V1_3_EE_n_GSI_HW_PARAM_2_GSI_NUM_EV_PER_EE_BMSK 0x1f00
-#define GSI_V1_3_EE_n_GSI_HW_PARAM_2_GSI_NUM_EV_PER_EE_SHFT 0x8
-#define GSI_V1_3_EE_n_GSI_HW_PARAM_2_GSI_NUM_CH_PER_EE_BMSK 0xf8
-#define GSI_V1_3_EE_n_GSI_HW_PARAM_2_GSI_NUM_CH_PER_EE_SHFT 0x3
-#define GSI_V1_3_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_BMSK 0x7
-#define GSI_V1_3_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_SHFT 0x0
-#define GSI_V1_3_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_ONE_KB_FVAL 0x0
-#define GSI_V1_3_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_TWO_KB_FVAL 0x1
-
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f040 + 0x4000 * (n))
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_RMSK 0x7fff
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_MAXn 2
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_SDMA_N_IOVEC_BMSK 0x38000000
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_SDMA_N_IOVEC_SHFT 0x1b
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_SDMA_MAX_BURST_BMSK 0x7F80000
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_SDMA_MAX_BURST_SHFT 0x13
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_SDMA_N_INT_BMSK 0x70000
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_SDMA_N_INT_SHFT 0x10
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_USE_SDMA_BMSK 0x8000
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_USE_SDMA_SHFT 0xf
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_CH_FULL_LOGIC_BMSK 0x4000
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_CH_FULL_LOGIC_SHFT 0xe
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_CH_PEND_TRANSLATE_BMSK 0x2000
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_CH_PEND_TRANSLATE_SHFT 0xd
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_NUM_EV_PER_EE_BMSK 0x1f00
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_NUM_EV_PER_EE_SHFT 0x8
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_NUM_CH_PER_EE_BMSK 0xf8
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_NUM_CH_PER_EE_SHFT 0x3
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_BMSK 0x7
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_SHFT 0x0
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_ONE_KB_FVAL 0x0
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_TWO_KB_FVAL 0x1
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_TWO_N_HALF_KB_FVAL 0x2
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_THREE_KB_FVAL 0x3
-
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_OFFS(n) \
-		(GSI_GSI_REG_BASE_OFFS + 0x0001f040 + 0x4000 * (n))
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_USE_INTER_EE_BMSK 0x8000
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_USE_INTER_EE_SHFT 0xf
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_USE_RD_WR_ENG_BMSK 0x4000
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_USE_RD_WR_ENG_SHFT 0xE
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_RMSK 0x7fff
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_MAXn 2
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_SDMA_N_IOVEC_BMSK 0x38000000
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_SDMA_N_IOVEC_SHFT 0x1b
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_SDMA_MAX_BURST_BMSK 0x7F80000
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_SDMA_MAX_BURST_SHFT 0x13
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_SDMA_N_INT_BMSK 0x70000
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_SDMA_N_INT_SHFT 0x10
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_USE_SDMA_BMSK 0x8000
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_USE_SDMA_SHFT 0xf
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_CH_FULL_LOGIC_BMSK 0x4000
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_CH_FULL_LOGIC_SHFT 0xe
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_CH_PEND_TRANSLATE_BMSK 0x2000
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_CH_PEND_TRANSLATE_SHFT 0xd
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_NUM_EV_PER_EE_BMSK 0x1f00
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_NUM_EV_PER_EE_SHFT 0x8
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_NUM_CH_PER_EE_BMSK 0xf8
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_NUM_CH_PER_EE_SHFT 0x3
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_BMSK 0x7
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_SHFT 0x0
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_ONE_KB_FVAL 0x0
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_TWO_KB_FVAL 0x1
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_TWO_N_HALF_KB_FVAL 0x2
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_THREE_KB_FVAL 0x3
-
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_OFFS(n) \
-		(GSI_GSI_REG_BASE_OFFS + 0x00012040 + 0x4000 * (n))
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_USE_INTER_EE_BMSK 0x8000
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_USE_INTER_EE_SHFT 0xf
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_USE_RD_WR_ENG_BMSK 0x4000
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_USE_RD_WR_ENG_SHFT 0xE
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_RMSK 0x7fff
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_MAXn 2
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_SDMA_N_IOVEC_BMSK 0x38000000
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_SDMA_N_IOVEC_SHFT 0x1b
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_SDMA_MAX_BURST_BMSK 0x7F80000
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_SDMA_MAX_BURST_SHFT 0x13
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_SDMA_N_INT_BMSK 0x70000
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_SDMA_N_INT_SHFT 0x10
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_USE_SDMA_BMSK 0x8000
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_USE_SDMA_SHFT 0xf
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_CH_FULL_LOGIC_BMSK 0x4000
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_CH_FULL_LOGIC_SHFT 0xe
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_CH_PEND_TRANSLATE_BMSK 0x2000
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_CH_PEND_TRANSLATE_SHFT 0xd
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_NUM_EV_PER_EE_BMSK 0x1f00
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_NUM_EV_PER_EE_SHFT 0x8
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_NUM_CH_PER_EE_BMSK 0xf8
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_NUM_CH_PER_EE_SHFT 0x3
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_BMSK 0x7
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_SHFT 0x0
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_ONE_KB_FVAL 0x0
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_TWO_KB_FVAL 0x1
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_TWO_N_HALF_KB_FVAL 0x2
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_THREE_KB_FVAL 0x3
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_THREE_N_HALF_KB_FVAL 0x4
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_FOUR_KB_FVAL 0x5
-
-#define GSI_EE_n_GSI_SW_VERSION_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f044 + 0x4000 * (n))
-#define GSI_EE_n_GSI_SW_VERSION_MAJOR_BMSK 0xf0000000
-#define GSI_EE_n_GSI_SW_VERSION_MAJOR_SHFT 0x1c
-#define GSI_EE_n_GSI_SW_VERSION_MINOR_BMSK 0xfff0000
-#define GSI_EE_n_GSI_SW_VERSION_MINOR_SHFT 0x10
-#define GSI_EE_n_GSI_SW_VERSION_STEP_BMSK 0xffff
-#define GSI_EE_n_GSI_SW_VERSION_STEP_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_TYPE_IRQ_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f080 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_TYPE_IRQ_GENERAL_BMSK 0x40
-#define GSI_EE_n_CNTXT_TYPE_IRQ_GENERAL_SHFT 0x6
-#define GSI_EE_n_CNTXT_TYPE_IRQ_INTER_EE_EV_CTRL_BMSK 0x20
-#define GSI_EE_n_CNTXT_TYPE_IRQ_INTER_EE_EV_CTRL_SHFT 0x5
-#define GSI_EE_n_CNTXT_TYPE_IRQ_INTER_EE_CH_CTRL_BMSK 0x10
-#define GSI_EE_n_CNTXT_TYPE_IRQ_INTER_EE_CH_CTRL_SHFT 0x4
-#define GSI_EE_n_CNTXT_TYPE_IRQ_IEOB_BMSK 0x8
-#define GSI_EE_n_CNTXT_TYPE_IRQ_IEOB_SHFT 0x3
-#define GSI_EE_n_CNTXT_TYPE_IRQ_GLOB_EE_BMSK 0x4
-#define GSI_EE_n_CNTXT_TYPE_IRQ_GLOB_EE_SHFT 0x2
-#define GSI_EE_n_CNTXT_TYPE_IRQ_EV_CTRL_BMSK 0x2
-#define GSI_EE_n_CNTXT_TYPE_IRQ_EV_CTRL_SHFT 0x1
-#define GSI_EE_n_CNTXT_TYPE_IRQ_CH_CTRL_BMSK 0x1
-#define GSI_EE_n_CNTXT_TYPE_IRQ_CH_CTRL_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_TYPE_IRQ_MSK_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f088 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_TYPE_IRQ_MSK_GENERAL_BMSK 0x40
-#define GSI_EE_n_CNTXT_TYPE_IRQ_MSK_GENERAL_SHFT 0x6
-#define GSI_EE_n_CNTXT_TYPE_IRQ_MSK_INTER_EE_EV_CTRL_BMSK 0x20
-#define GSI_EE_n_CNTXT_TYPE_IRQ_MSK_INTER_EE_EV_CTRL_SHFT 0x5
-#define GSI_EE_n_CNTXT_TYPE_IRQ_MSK_INTER_EE_CH_CTRL_BMSK 0x10
-#define GSI_EE_n_CNTXT_TYPE_IRQ_MSK_INTER_EE_CH_CTRL_SHFT 0x4
-#define GSI_EE_n_CNTXT_TYPE_IRQ_MSK_IEOB_BMSK 0x8
-#define GSI_EE_n_CNTXT_TYPE_IRQ_MSK_IEOB_SHFT 0x3
-#define GSI_EE_n_CNTXT_TYPE_IRQ_MSK_GLOB_EE_BMSK 0x4
-#define GSI_EE_n_CNTXT_TYPE_IRQ_MSK_GLOB_EE_SHFT 0x2
-#define GSI_EE_n_CNTXT_TYPE_IRQ_MSK_EV_CTRL_BMSK 0x2
-#define GSI_EE_n_CNTXT_TYPE_IRQ_MSK_EV_CTRL_SHFT 0x1
-#define GSI_EE_n_CNTXT_TYPE_IRQ_MSK_CH_CTRL_BMSK 0x1
-#define GSI_EE_n_CNTXT_TYPE_IRQ_MSK_CH_CTRL_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_SRC_GSI_CH_IRQ_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f090 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_SRC_GSI_CH_IRQ_GSI_CH_BIT_MAP_BMSK 0xffffffff
-#define GSI_EE_n_CNTXT_SRC_GSI_CH_IRQ_GSI_CH_BIT_MAP_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_SRC_EV_CH_IRQ_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f094 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_SRC_EV_CH_IRQ_EV_CH_BIT_MAP_BMSK 0xffffffff
-#define GSI_EE_n_CNTXT_SRC_EV_CH_IRQ_EV_CH_BIT_MAP_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f098 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_GSI_CH_BIT_MAP_MSK_BMSK 0x1ffff
-#define GSI_V2_5_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_GSI_CH_BIT_MAP_MSK_BMSK 0x7fffff
-#define GSI_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_GSI_CH_BIT_MAP_MSK_SHFT 0x0
-#define GSI_V2_5_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_GSI_CH_BIT_MAP_MSK_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f09c + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_EV_CH_BIT_MAP_MSK_BMSK 0xfff
-#define GSI_V2_5_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_EV_CH_BIT_MAP_MSK_BMSK 0xfffff
-#define GSI_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_EV_CH_BIT_MAP_MSK_SHFT 0x0
-#define GSI_V2_5_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_EV_CH_BIT_MAP_MSK_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_SRC_GSI_CH_IRQ_CLR_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f0a0 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_SRC_GSI_CH_IRQ_CLR_GSI_CH_BIT_MAP_BMSK 0xffffffff
-#define GSI_EE_n_CNTXT_SRC_GSI_CH_IRQ_CLR_GSI_CH_BIT_MAP_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_SRC_EV_CH_IRQ_CLR_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f0a4 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_SRC_EV_CH_IRQ_CLR_EV_CH_BIT_MAP_BMSK 0xffffffff
-#define GSI_EE_n_CNTXT_SRC_EV_CH_IRQ_CLR_EV_CH_BIT_MAP_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_SRC_IEOB_IRQ_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f0b0 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_SRC_IEOB_IRQ_EV_CH_BIT_MAP_BMSK 0xffffffff
-#define GSI_EE_n_CNTXT_SRC_IEOB_IRQ_EV_CH_BIT_MAP_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f0b8 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_EV_CH_BIT_MAP_MSK_BMSK 0xfff
-#define GSI_V2_5_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_EV_CH_BIT_MAP_MSK_BMSK 0xfffff
-#define GSI_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_EV_CH_BIT_MAP_MSK_SHFT 0x0
-#define GSI_V2_5_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_EV_CH_BIT_MAP_MSK_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_SRC_IEOB_IRQ_CLR_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f0c0 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_SRC_IEOB_IRQ_CLR_EV_CH_BIT_MAP_BMSK 0xffffffff
-#define GSI_EE_n_CNTXT_SRC_IEOB_IRQ_CLR_EV_CH_BIT_MAP_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_GLOB_IRQ_STTS_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f100 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_GLOB_IRQ_STTS_GP_INT3_BMSK 0x8
-#define GSI_EE_n_CNTXT_GLOB_IRQ_STTS_GP_INT3_SHFT 0x3
-#define GSI_EE_n_CNTXT_GLOB_IRQ_STTS_GP_INT2_BMSK 0x4
-#define GSI_EE_n_CNTXT_GLOB_IRQ_STTS_GP_INT2_SHFT 0x2
-#define GSI_EE_n_CNTXT_GLOB_IRQ_STTS_GP_INT1_BMSK 0x2
-#define GSI_EE_n_CNTXT_GLOB_IRQ_STTS_GP_INT1_SHFT 0x1
-#define GSI_EE_n_CNTXT_GLOB_IRQ_STTS_ERROR_INT_BMSK 0x1
-#define GSI_EE_n_CNTXT_GLOB_IRQ_STTS_ERROR_INT_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_GLOB_IRQ_EN_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f108 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_GLOB_IRQ_EN_GP_INT3_BMSK 0x8
-#define GSI_EE_n_CNTXT_GLOB_IRQ_EN_GP_INT3_SHFT 0x3
-#define GSI_EE_n_CNTXT_GLOB_IRQ_EN_GP_INT2_BMSK 0x4
-#define GSI_EE_n_CNTXT_GLOB_IRQ_EN_GP_INT2_SHFT 0x2
-#define GSI_EE_n_CNTXT_GLOB_IRQ_EN_GP_INT1_BMSK 0x2
-#define GSI_EE_n_CNTXT_GLOB_IRQ_EN_GP_INT1_SHFT 0x1
-#define GSI_EE_n_CNTXT_GLOB_IRQ_EN_ERROR_INT_BMSK 0x1
-#define GSI_EE_n_CNTXT_GLOB_IRQ_EN_ERROR_INT_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_GLOB_IRQ_CLR_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f110 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_GLOB_IRQ_CLR_GP_INT3_BMSK 0x8
-#define GSI_EE_n_CNTXT_GLOB_IRQ_CLR_GP_INT3_SHFT 0x3
-#define GSI_EE_n_CNTXT_GLOB_IRQ_CLR_GP_INT2_BMSK 0x4
-#define GSI_EE_n_CNTXT_GLOB_IRQ_CLR_GP_INT2_SHFT 0x2
-#define GSI_EE_n_CNTXT_GLOB_IRQ_CLR_GP_INT1_BMSK 0x2
-#define GSI_EE_n_CNTXT_GLOB_IRQ_CLR_GP_INT1_SHFT 0x1
-#define GSI_EE_n_CNTXT_GLOB_IRQ_CLR_ERROR_INT_BMSK 0x1
-#define GSI_EE_n_CNTXT_GLOB_IRQ_CLR_ERROR_INT_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_GSI_IRQ_STTS_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f118 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_GSI_IRQ_STTS_GSI_MCS_STACK_OVRFLOW_BMSK 0x8
-#define GSI_EE_n_CNTXT_GSI_IRQ_STTS_GSI_MCS_STACK_OVRFLOW_SHFT 0x3
-#define GSI_EE_n_CNTXT_GSI_IRQ_STTS_GSI_CMD_FIFO_OVRFLOW_BMSK 0x4
-#define GSI_EE_n_CNTXT_GSI_IRQ_STTS_GSI_CMD_FIFO_OVRFLOW_SHFT 0x2
-#define GSI_EE_n_CNTXT_GSI_IRQ_STTS_GSI_BUS_ERROR_BMSK 0x2
-#define GSI_EE_n_CNTXT_GSI_IRQ_STTS_GSI_BUS_ERROR_SHFT 0x1
-#define GSI_EE_n_CNTXT_GSI_IRQ_STTS_GSI_BREAK_POINT_BMSK 0x1
-#define GSI_EE_n_CNTXT_GSI_IRQ_STTS_GSI_BREAK_POINT_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_GSI_IRQ_EN_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f120 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_GSI_IRQ_EN_GSI_MCS_STACK_OVRFLOW_BMSK 0x8
-#define GSI_EE_n_CNTXT_GSI_IRQ_EN_GSI_MCS_STACK_OVRFLOW_SHFT 0x3
-#define GSI_EE_n_CNTXT_GSI_IRQ_EN_GSI_CMD_FIFO_OVRFLOW_BMSK 0x4
-#define GSI_EE_n_CNTXT_GSI_IRQ_EN_GSI_CMD_FIFO_OVRFLOW_SHFT 0x2
-#define GSI_EE_n_CNTXT_GSI_IRQ_EN_GSI_BUS_ERROR_BMSK 0x2
-#define GSI_EE_n_CNTXT_GSI_IRQ_EN_GSI_BUS_ERROR_SHFT 0x1
-#define GSI_EE_n_CNTXT_GSI_IRQ_EN_GSI_BREAK_POINT_BMSK 0x1
-#define GSI_EE_n_CNTXT_GSI_IRQ_EN_GSI_BREAK_POINT_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_GSI_IRQ_CLR_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f128 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_GSI_IRQ_CLR_GSI_MCS_STACK_OVRFLOW_BMSK 0x8
-#define GSI_EE_n_CNTXT_GSI_IRQ_CLR_GSI_MCS_STACK_OVRFLOW_SHFT 0x3
-#define GSI_EE_n_CNTXT_GSI_IRQ_CLR_GSI_CMD_FIFO_OVRFLOW_BMSK 0x4
-#define GSI_EE_n_CNTXT_GSI_IRQ_CLR_GSI_CMD_FIFO_OVRFLOW_SHFT 0x2
-#define GSI_EE_n_CNTXT_GSI_IRQ_CLR_GSI_BUS_ERROR_BMSK 0x2
-#define GSI_EE_n_CNTXT_GSI_IRQ_CLR_GSI_BUS_ERROR_SHFT 0x1
-#define GSI_EE_n_CNTXT_GSI_IRQ_CLR_GSI_BREAK_POINT_BMSK 0x1
-#define GSI_EE_n_CNTXT_GSI_IRQ_CLR_GSI_BREAK_POINT_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_MSI_BASE_LSB(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f188 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_MSI_BASE_MSB(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f18c + 0x4000 * (n))
-
-#define GSI_EE_n_CNTXT_INTSET_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f180 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_INTSET_INTYPE_BMSK 0x1
-#define GSI_EE_n_CNTXT_INTSET_INTYPE_SHFT 0x0
-
-#define GSI_EE_n_ERROR_LOG_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f200 + 0x4000 * (n))
-#define GSI_EE_n_ERROR_LOG_TODO_BMSK 0xffffffff
-#define GSI_EE_n_ERROR_LOG_TODO_SHFT 0x0
-
-#define GSI_EE_n_ERROR_LOG_CLR_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f210 + 0x4000 * (n))
-#define GSI_EE_n_ERROR_LOG_CLR_TODO_BMSK 0xffffffff
-#define GSI_EE_n_ERROR_LOG_CLR_TODO_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_SCRATCH_0_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f400 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_SCRATCH_0_SCRATCH_BMSK 0xffffffff
-#define GSI_EE_n_CNTXT_SCRATCH_0_SCRATCH_SHFT 0x0
-
-#define GSI_INTER_EE_n_SRC_GSI_CH_IRQ_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000c018 + 0x1000 * (n))
-#define GSI_INTER_EE_n_SRC_GSI_CH_IRQ_GSI_CH_BIT_MAP_BMSK 0xffffffff
-#define GSI_INTER_EE_n_SRC_GSI_CH_IRQ_GSI_CH_BIT_MAP_SHFT 0x0
-
-#define GSI_INTER_EE_n_SRC_EV_CH_IRQ_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000c01c + 0x1000 * (n))
-#define GSI_INTER_EE_n_SRC_EV_CH_IRQ_EV_CH_BIT_MAP_BMSK 0xffffffff
-#define GSI_INTER_EE_n_SRC_EV_CH_IRQ_EV_CH_BIT_MAP_SHFT 0x0
-
-#define GSI_INTER_EE_n_SRC_GSI_CH_IRQ_CLR_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000c028 + 0x1000 * (n))
-#define GSI_INTER_EE_n_SRC_GSI_CH_IRQ_CLR_GSI_CH_BIT_MAP_BMSK 0xffffffff
-#define GSI_INTER_EE_n_SRC_GSI_CH_IRQ_CLR_GSI_CH_BIT_MAP_SHFT 0x0
-
-#define GSI_INTER_EE_n_SRC_EV_CH_IRQ_CLR_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000c02c + 0x1000 * (n))
-#define GSI_INTER_EE_n_SRC_EV_CH_IRQ_CLR_EV_CH_BIT_MAP_BMSK 0xffffffff
-#define GSI_INTER_EE_n_SRC_EV_CH_IRQ_CLR_EV_CH_BIT_MAP_SHFT 0x0
-
-#define GSI_V2_5_GSI_MAP_EE_n_CH_k_VP_TABLE_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00003800 + 0x80 * (n) + 0x4 * (k))
-#define GSI_V2_5_GSI_MAP_EE_n_CH_k_VP_TABLE_VALID_BMSK 0x20
-#define GSI_V2_5_GSI_MAP_EE_n_CH_k_VP_TABLE_VALID_SHFT 0x5
-#define GSI_V2_5_GSI_MAP_EE_n_CH_k_VP_TABLE_PHY_CH_BMSK 0x1f
-#define GSI_V2_5_GSI_MAP_EE_n_CH_k_VP_TABLE_PHY_CH_SHFT 0x0
-
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_OFFS(n) \
-			(GSI_GSI_REG_BASE_OFFS + 0x00012040 + 0x4000 * (n))
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_USE_INTER_EE_BMSK 0x8000
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_USE_INTER_EE_SHFT 0xf
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_USE_RD_WR_ENG_BMSK 0x4000
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_USE_RD_WR_ENG_SHFT 0xE
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_RMSK 0x7fff
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_MAXn 2
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_SDMA_N_IOVEC_BMSK 0x38000000
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_SDMA_N_IOVEC_SHFT 0x1b
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_SDMA_MAX_BURST_BMSK 0x7F80000
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_SDMA_MAX_BURST_SHFT 0x13
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_SDMA_N_INT_BMSK 0x70000
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_SDMA_N_INT_SHFT 0x10
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_USE_SDMA_BMSK 0x8000
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_USE_SDMA_SHFT 0xf
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_CH_FULL_LOGIC_BMSK 0x4000
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_CH_FULL_LOGIC_SHFT 0xe
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_CH_PEND_TRANSLATE_BMSK 0x2000
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_CH_PEND_TRANSLATE_SHFT 0xd
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_NUM_EV_PER_EE_BMSK 0x1f00
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_NUM_EV_PER_EE_SHFT 0x8
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_NUM_CH_PER_EE_BMSK 0xf8
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_NUM_CH_PER_EE_SHFT 0x3
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_BMSK 0x7
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_SHFT 0x0
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_ONE_KB_FVAL 0x0
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_TWO_KB_FVAL 0x1
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_TWO_N_HALF_KB_FVAL 0x2
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_THREE_KB_FVAL 0x3
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_THREE_N_HALF_KB_FVAL 0x4
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_FOUR_KB_FVAL 0x5
-
-#endif /* __GSI_REG_V1_H__ */

gsi/gsi_reg_v2.h

@@ -1,1099 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2015-2019, The Linux Foundation. All rights reserved.
- */
-
-#ifndef __GSI_REG_V2_H__
-#define __GSI_REG_V2_H__
-
-#define GSI_GSI_REG_BASE_OFFS 0
-
-#define GSI_GSI_CFG_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x00000000)
-#define GSI_V2_5_GSI_CFG_SLEEP_CLK_DIV_BMSK 0xf00
-#define GSI_V2_5_GSI_CFG_SLEEP_CLK_DIV_SHFT 0x8
-#define GSI_GSI_CFG_BP_MTRIX_DISABLE_BMSK 0x20
-#define GSI_GSI_CFG_BP_MTRIX_DISABLE_SHFT 0x5
-#define GSI_GSI_CFG_GSI_PWR_CLPS_BMSK 0x10
-#define GSI_GSI_CFG_GSI_PWR_CLPS_SHFT 0x4
-#define GSI_GSI_CFG_UC_IS_MCS_BMSK 0x8
-#define GSI_GSI_CFG_UC_IS_MCS_SHFT 0x3
-#define GSI_GSI_CFG_DOUBLE_MCS_CLK_FREQ_BMSK 0x4
-#define GSI_GSI_CFG_DOUBLE_MCS_CLK_FREQ_SHFT 0x2
-#define GSI_GSI_CFG_MCS_ENABLE_BMSK 0x2
-#define GSI_GSI_CFG_MCS_ENABLE_SHFT 0x1
-#define GSI_GSI_CFG_GSI_ENABLE_BMSK 0x1
-#define GSI_GSI_CFG_GSI_ENABLE_SHFT 0x0
-
-#define GSI_GSI_MCS_CFG_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000B000)
-#define GSI_GSI_MCS_CFG_MCS_ENABLE_BMSK 0x1
-#define GSI_GSI_MCS_CFG_MCS_ENABLE_SHFT 0x0
-
-#define GSI_GSI_PERIPH_BASE_ADDR_LSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x00000018)
-#define GSI_GSI_PERIPH_BASE_ADDR_LSB_RMSK 0xffffffff
-#define GSI_GSI_PERIPH_BASE_ADDR_LSB_BASE_ADDR_BMSK 0xffffffff
-#define GSI_GSI_PERIPH_BASE_ADDR_LSB_BASE_ADDR_SHFT 0x0
-
-#define GSI_GSI_PERIPH_BASE_ADDR_MSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000001c)
-#define GSI_GSI_PERIPH_BASE_ADDR_MSB_RMSK 0xffffffff
-#define GSI_GSI_PERIPH_BASE_ADDR_MSB_BASE_ADDR_BMSK 0xffffffff
-#define GSI_GSI_PERIPH_BASE_ADDR_MSB_BASE_ADDR_SHFT 0x0
-
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_LSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000a0)
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_LSB_RMSK 0x3ffc1047
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_LSB_TLV_INT_BMSK 0x3f000000
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_LSB_TLV_INT_SHFT 0x18
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_LSB_CSR_INT_BMSK 0xfc0000
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_LSB_CSR_INT_SHFT 0x12
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_LSB_INT_END_INT_BMSK 0x1000
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_LSB_INT_END_INT_SHFT 0xc
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_LSB_EV_ENG_INT_BMSK 0x40
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_LSB_EV_ENG_INT_SHFT 0x6
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_LSB_REE_INT_BMSK 0x7
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_LSB_REE_INT_SHFT 0x0
-
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_MSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000a4)
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_MSB_RMSK 0xfc3041
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_MSB_UCONTROLLER_INT_BMSK 0xfc0000
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_MSB_UCONTROLLER_INT_SHFT 0x12
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_MSB_RD_WR_INT_BMSK 0x3000
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_MSB_RD_WR_INT_SHFT 0xc
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_MSB_DB_ENG_INT_BMSK 0x40
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_MSB_DB_ENG_INT_SHFT 0x6
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_MSB_TIMER_INT_BMSK 0x1
-#define GSI_IC_DISABLE_CHNL_BCK_PRS_MSB_TIMER_INT_SHFT 0x0
-
-#define GSI_IC_GEN_EVNT_BCK_PRS_LSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000a8)
-#define GSI_IC_GEN_EVNT_BCK_PRS_LSB_RMSK 0x3ffc1047
-#define GSI_IC_GEN_EVNT_BCK_PRS_LSB_TLV_INT_BMSK 0x3f000000
-#define GSI_IC_GEN_EVNT_BCK_PRS_LSB_TLV_INT_SHFT 0x18
-#define GSI_IC_GEN_EVNT_BCK_PRS_LSB_CSR_INT_BMSK 0xfc0000
-#define GSI_IC_GEN_EVNT_BCK_PRS_LSB_CSR_INT_SHFT 0x12
-#define GSI_IC_GEN_EVNT_BCK_PRS_LSB_INT_END_INT_BMSK 0x1000
-#define GSI_IC_GEN_EVNT_BCK_PRS_LSB_INT_END_INT_SHFT 0xc
-#define GSI_IC_GEN_EVNT_BCK_PRS_LSB_EV_ENG_INT_BMSK 0x40
-#define GSI_IC_GEN_EVNT_BCK_PRS_LSB_EV_ENG_INT_SHFT 0x6
-#define GSI_IC_GEN_EVNT_BCK_PRS_LSB_REE_INT_BMSK 0x7
-#define GSI_IC_GEN_EVNT_BCK_PRS_LSB_REE_INT_SHFT 0x0
-
-#define GSI_IC_GEN_EVNT_BCK_PRS_MSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000ac)
-#define GSI_IC_GEN_EVNT_BCK_PRS_MSB_RMSK 0xfc3041
-#define GSI_IC_GEN_EVNT_BCK_PRS_MSB_UCONTROLLER_INT_BMSK 0xfc0000
-#define GSI_IC_GEN_EVNT_BCK_PRS_MSB_UCONTROLLER_INT_SHFT 0x12
-#define GSI_IC_GEN_EVNT_BCK_PRS_MSB_RD_WR_INT_BMSK 0x3000
-#define GSI_IC_GEN_EVNT_BCK_PRS_MSB_RD_WR_INT_SHFT 0xc
-#define GSI_IC_GEN_EVNT_BCK_PRS_MSB_DB_ENG_INT_BMSK 0x40
-#define GSI_IC_GEN_EVNT_BCK_PRS_MSB_DB_ENG_INT_SHFT 0x6
-#define GSI_IC_GEN_EVNT_BCK_PRS_MSB_TIMER_INT_BMSK 0x1
-#define GSI_IC_GEN_EVNT_BCK_PRS_MSB_TIMER_INT_SHFT 0x0
-
-#define GSI_IC_GEN_INT_BCK_PRS_LSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000b0)
-#define GSI_IC_GEN_INT_BCK_PRS_LSB_RMSK 0x3ffc1047
-#define GSI_IC_GEN_INT_BCK_PRS_LSB_TLV_INT_BMSK 0x3f000000
-#define GSI_IC_GEN_INT_BCK_PRS_LSB_TLV_INT_SHFT 0x18
-#define GSI_IC_GEN_INT_BCK_PRS_LSB_CSR_INT_BMSK 0xfc0000
-#define GSI_IC_GEN_INT_BCK_PRS_LSB_CSR_INT_SHFT 0x12
-#define GSI_IC_GEN_INT_BCK_PRS_LSB_INT_END_INT_BMSK 0x1000
-#define GSI_IC_GEN_INT_BCK_PRS_LSB_INT_END_INT_SHFT 0xc
-#define GSI_IC_GEN_INT_BCK_PRS_LSB_EV_ENG_INT_BMSK 0x40
-#define GSI_IC_GEN_INT_BCK_PRS_LSB_EV_ENG_INT_SHFT 0x6
-#define GSI_IC_GEN_INT_BCK_PRS_LSB_REE_INT_BMSK 0x7
-#define GSI_IC_GEN_INT_BCK_PRS_LSB_REE_INT_SHFT 0x0
-
-#define GSI_IC_GEN_INT_BCK_PRS_MSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000b4)
-#define GSI_IC_GEN_INT_BCK_PRS_MSB_RMSK 0xfc3041
-#define GSI_IC_GEN_INT_BCK_PRS_MSB_UCONTROLLER_INT_BMSK 0xfc0000
-#define GSI_IC_GEN_INT_BCK_PRS_MSB_UCONTROLLER_INT_SHFT 0x12
-#define GSI_IC_GEN_INT_BCK_PRS_MSB_RD_WR_INT_BMSK 0x3000
-#define GSI_IC_GEN_INT_BCK_PRS_MSB_RD_WR_INT_SHFT 0xc
-#define GSI_IC_GEN_INT_BCK_PRS_MSB_DB_ENG_INT_BMSK 0x40
-#define GSI_IC_GEN_INT_BCK_PRS_MSB_DB_ENG_INT_SHFT 0x6
-#define GSI_IC_GEN_INT_BCK_PRS_MSB_TIMER_INT_BMSK 0x1
-#define GSI_IC_GEN_INT_BCK_PRS_MSB_TIMER_INT_SHFT 0x0
-
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_LSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000b8)
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_LSB_RMSK 0x3ffc1047
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_LSB_TLV_INT_BMSK 0x3f000000
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_LSB_TLV_INT_SHFT 0x18
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_LSB_CSR_INT_BMSK 0xfc0000
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_LSB_CSR_INT_SHFT 0x12
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_LSB_INT_END_INT_BMSK 0x1000
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_LSB_INT_END_INT_SHFT 0xc
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_LSB_EV_ENG_INT_BMSK 0x40
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_LSB_EV_ENG_INT_SHFT 0x6
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_LSB_REE_INT_BMSK 0x7
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_LSB_REE_INT_SHFT 0x0
-
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_MSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000bc)
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_MSB_RMSK 0xfc3041
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_MSB_UCONTROLLER_INT_BMSK 0xfc0000
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_MSB_UCONTROLLER_INT_SHFT 0x12
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_MSB_RD_WR_INT_BMSK 0x3000
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_MSB_RD_WR_INT_SHFT 0xc
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_MSB_DB_ENG_INT_BMSK 0x40
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_MSB_DB_ENG_INT_SHFT 0x6
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_MSB_TIMER_INT_BMSK 0x1
-#define GSI_IC_STOP_INT_MOD_BCK_PRS_MSB_TIMER_INT_SHFT 0x0
-
-#define GSI_IC_PROCESS_DESC_BCK_PRS_LSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000c0)
-#define GSI_IC_PROCESS_DESC_BCK_PRS_LSB_RMSK 0x3ffc1047
-#define GSI_IC_PROCESS_DESC_BCK_PRS_LSB_TLV_INT_BMSK 0x3f000000
-#define GSI_IC_PROCESS_DESC_BCK_PRS_LSB_TLV_INT_SHFT 0x18
-#define GSI_IC_PROCESS_DESC_BCK_PRS_LSB_CSR_INT_BMSK 0xfc0000
-#define GSI_IC_PROCESS_DESC_BCK_PRS_LSB_CSR_INT_SHFT 0x12
-#define GSI_IC_PROCESS_DESC_BCK_PRS_LSB_INT_END_INT_BMSK 0x1000
-#define GSI_IC_PROCESS_DESC_BCK_PRS_LSB_INT_END_INT_SHFT 0xc
-#define GSI_IC_PROCESS_DESC_BCK_PRS_LSB_EV_ENG_INT_BMSK 0x40
-#define GSI_IC_PROCESS_DESC_BCK_PRS_LSB_EV_ENG_INT_SHFT 0x6
-#define GSI_IC_PROCESS_DESC_BCK_PRS_LSB_REE_INT_BMSK 0x7
-#define GSI_IC_PROCESS_DESC_BCK_PRS_LSB_REE_INT_SHFT 0x0
-
-#define GSI_IC_PROCESS_DESC_BCK_PRS_MSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000c4)
-#define GSI_IC_PROCESS_DESC_BCK_PRS_MSB_RMSK 0xfc3041
-#define GSI_IC_PROCESS_DESC_BCK_PRS_MSB_UCONTROLLER_INT_BMSK 0xfc0000
-#define GSI_IC_PROCESS_DESC_BCK_PRS_MSB_UCONTROLLER_INT_SHFT 0x12
-#define GSI_IC_PROCESS_DESC_BCK_PRS_MSB_RD_WR_INT_BMSK 0x3000
-#define GSI_IC_PROCESS_DESC_BCK_PRS_MSB_RD_WR_INT_SHFT 0xc
-#define GSI_IC_PROCESS_DESC_BCK_PRS_MSB_DB_ENG_INT_BMSK 0x40
-#define GSI_IC_PROCESS_DESC_BCK_PRS_MSB_DB_ENG_INT_SHFT 0x6
-#define GSI_IC_PROCESS_DESC_BCK_PRS_MSB_TIMER_INT_BMSK 0x1
-#define GSI_IC_PROCESS_DESC_BCK_PRS_MSB_TIMER_INT_SHFT 0x0
-
-#define GSI_IC_TLV_STOP_BCK_PRS_LSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000c8)
-#define GSI_IC_TLV_STOP_BCK_PRS_LSB_RMSK 0x3ffc1047
-#define GSI_IC_TLV_STOP_BCK_PRS_LSB_TLV_INT_BMSK 0x3f000000
-#define GSI_IC_TLV_STOP_BCK_PRS_LSB_TLV_INT_SHFT 0x18
-#define GSI_IC_TLV_STOP_BCK_PRS_LSB_CSR_INT_BMSK 0xfc0000
-#define GSI_IC_TLV_STOP_BCK_PRS_LSB_CSR_INT_SHFT 0x12
-#define GSI_IC_TLV_STOP_BCK_PRS_LSB_INT_END_INT_BMSK 0x1000
-#define GSI_IC_TLV_STOP_BCK_PRS_LSB_INT_END_INT_SHFT 0xc
-#define GSI_IC_TLV_STOP_BCK_PRS_LSB_EV_ENG_INT_BMSK 0x40
-#define GSI_IC_TLV_STOP_BCK_PRS_LSB_EV_ENG_INT_SHFT 0x6
-#define GSI_IC_TLV_STOP_BCK_PRS_LSB_REE_INT_BMSK 0x7
-#define GSI_IC_TLV_STOP_BCK_PRS_LSB_REE_INT_SHFT 0x0
-
-#define GSI_IC_TLV_STOP_BCK_PRS_MSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000cc)
-#define GSI_IC_TLV_STOP_BCK_PRS_MSB_RMSK 0xfc3041
-#define GSI_IC_TLV_STOP_BCK_PRS_MSB_UCONTROLLER_INT_BMSK 0xfc0000
-#define GSI_IC_TLV_STOP_BCK_PRS_MSB_UCONTROLLER_INT_SHFT 0x12
-#define GSI_IC_TLV_STOP_BCK_PRS_MSB_RD_WR_INT_BMSK 0x3000
-#define GSI_IC_TLV_STOP_BCK_PRS_MSB_RD_WR_INT_SHFT 0xc
-#define GSI_IC_TLV_STOP_BCK_PRS_MSB_DB_ENG_INT_BMSK 0x40
-#define GSI_IC_TLV_STOP_BCK_PRS_MSB_DB_ENG_INT_SHFT 0x6
-#define GSI_IC_TLV_STOP_BCK_PRS_MSB_TIMER_INT_BMSK 0x1
-#define GSI_IC_TLV_STOP_BCK_PRS_MSB_TIMER_INT_SHFT 0x0
-
-#define GSI_IC_TLV_RESET_BCK_PRS_LSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000d0)
-#define GSI_IC_TLV_RESET_BCK_PRS_LSB_RMSK 0x3ffc1047
-#define GSI_IC_TLV_RESET_BCK_PRS_LSB_TLV_INT_BMSK 0x3f000000
-#define GSI_IC_TLV_RESET_BCK_PRS_LSB_TLV_INT_SHFT 0x18
-#define GSI_IC_TLV_RESET_BCK_PRS_LSB_CSR_INT_BMSK 0xfc0000
-#define GSI_IC_TLV_RESET_BCK_PRS_LSB_CSR_INT_SHFT 0x12
-#define GSI_IC_TLV_RESET_BCK_PRS_LSB_INT_END_INT_BMSK 0x1000
-#define GSI_IC_TLV_RESET_BCK_PRS_LSB_INT_END_INT_SHFT 0xc
-#define GSI_IC_TLV_RESET_BCK_PRS_LSB_EV_ENG_INT_BMSK 0x40
-#define GSI_IC_TLV_RESET_BCK_PRS_LSB_EV_ENG_INT_SHFT 0x6
-#define GSI_IC_TLV_RESET_BCK_PRS_LSB_REE_INT_BMSK 0x7
-#define GSI_IC_TLV_RESET_BCK_PRS_LSB_REE_INT_SHFT 0x0
-
-#define GSI_IC_TLV_RESET_BCK_PRS_MSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000d4)
-#define GSI_IC_TLV_RESET_BCK_PRS_MSB_RMSK 0xfc3041
-#define GSI_IC_TLV_RESET_BCK_PRS_MSB_UCONTROLLER_INT_BMSK 0xfc0000
-#define GSI_IC_TLV_RESET_BCK_PRS_MSB_UCONTROLLER_INT_SHFT 0x12
-#define GSI_IC_TLV_RESET_BCK_PRS_MSB_RD_WR_INT_BMSK 0x3000
-#define GSI_IC_TLV_RESET_BCK_PRS_MSB_RD_WR_INT_SHFT 0xc
-#define GSI_IC_TLV_RESET_BCK_PRS_MSB_DB_ENG_INT_BMSK 0x40
-#define GSI_IC_TLV_RESET_BCK_PRS_MSB_DB_ENG_INT_SHFT 0x6
-#define GSI_IC_TLV_RESET_BCK_PRS_MSB_TIMER_INT_BMSK 0x1
-#define GSI_IC_TLV_RESET_BCK_PRS_MSB_TIMER_INT_SHFT 0x0
-
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_LSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000d8)
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_LSB_RMSK 0x3ffc1047
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_LSB_TLV_INT_BMSK 0x3f000000
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_LSB_TLV_INT_SHFT 0x18
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_LSB_CSR_INT_BMSK 0xfc0000
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_LSB_CSR_INT_SHFT 0x12
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_LSB_INT_END_INT_BMSK 0x1000
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_LSB_INT_END_INT_SHFT 0xc
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_LSB_EV_ENG_INT_BMSK 0x40
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_LSB_EV_ENG_INT_SHFT 0x6
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_LSB_REE_INT_BMSK 0x7
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_LSB_REE_INT_SHFT 0x0
-
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_MSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000dc)
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_MSB_RMSK 0xfc3041
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_MSB_UCONTROLLER_INT_BMSK 0xfc0000
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_MSB_UCONTROLLER_INT_SHFT 0x12
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_MSB_RD_WR_INT_BMSK 0x3000
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_MSB_RD_WR_INT_SHFT 0xc
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_MSB_DB_ENG_INT_BMSK 0x40
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_MSB_DB_ENG_INT_SHFT 0x6
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_MSB_TIMER_INT_BMSK 0x1
-#define GSI_IC_RGSTR_TIMER_BCK_PRS_MSB_TIMER_INT_SHFT 0x0
-
-#define GSI_IC_READ_BCK_PRS_LSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000e0)
-#define GSI_IC_READ_BCK_PRS_LSB_RMSK 0x3ffc1047
-#define GSI_IC_READ_BCK_PRS_LSB_TLV_INT_BMSK 0x3f000000
-#define GSI_IC_READ_BCK_PRS_LSB_TLV_INT_SHFT 0x18
-#define GSI_IC_READ_BCK_PRS_LSB_CSR_INT_BMSK 0xfc0000
-#define GSI_IC_READ_BCK_PRS_LSB_CSR_INT_SHFT 0x12
-#define GSI_IC_READ_BCK_PRS_LSB_INT_END_INT_BMSK 0x1000
-#define GSI_IC_READ_BCK_PRS_LSB_INT_END_INT_SHFT 0xc
-#define GSI_IC_READ_BCK_PRS_LSB_EV_ENG_INT_BMSK 0x40
-#define GSI_IC_READ_BCK_PRS_LSB_EV_ENG_INT_SHFT 0x6
-#define GSI_IC_READ_BCK_PRS_LSB_REE_INT_BMSK 0x7
-#define GSI_IC_READ_BCK_PRS_LSB_REE_INT_SHFT 0x0
-
-#define GSI_IC_READ_BCK_PRS_MSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000e4)
-#define GSI_IC_READ_BCK_PRS_MSB_RMSK 0xfc3041
-#define GSI_IC_READ_BCK_PRS_MSB_UCONTROLLER_INT_BMSK 0xfc0000
-#define GSI_IC_READ_BCK_PRS_MSB_UCONTROLLER_INT_SHFT 0x12
-#define GSI_IC_READ_BCK_PRS_MSB_RD_WR_INT_BMSK 0x3000
-#define GSI_IC_READ_BCK_PRS_MSB_RD_WR_INT_SHFT 0xc
-#define GSI_IC_READ_BCK_PRS_MSB_DB_ENG_INT_BMSK 0x40
-#define GSI_IC_READ_BCK_PRS_MSB_DB_ENG_INT_SHFT 0x6
-#define GSI_IC_READ_BCK_PRS_MSB_TIMER_INT_BMSK 0x1
-#define GSI_IC_READ_BCK_PRS_MSB_TIMER_INT_SHFT 0x0
-
-#define GSI_IC_WRITE_BCK_PRS_LSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000e8)
-#define GSI_IC_WRITE_BCK_PRS_LSB_RMSK 0x3ffc1047
-#define GSI_IC_WRITE_BCK_PRS_LSB_TLV_INT_BMSK 0x3f000000
-#define GSI_IC_WRITE_BCK_PRS_LSB_TLV_INT_SHFT 0x18
-#define GSI_IC_WRITE_BCK_PRS_LSB_CSR_INT_BMSK 0xfc0000
-#define GSI_IC_WRITE_BCK_PRS_LSB_CSR_INT_SHFT 0x12
-#define GSI_IC_WRITE_BCK_PRS_LSB_INT_END_INT_BMSK 0x1000
-#define GSI_IC_WRITE_BCK_PRS_LSB_INT_END_INT_SHFT 0xc
-#define GSI_IC_WRITE_BCK_PRS_LSB_EV_ENG_INT_BMSK 0x40
-#define GSI_IC_WRITE_BCK_PRS_LSB_EV_ENG_INT_SHFT 0x6
-#define GSI_IC_WRITE_BCK_PRS_LSB_REE_INT_BMSK 0x7
-#define GSI_IC_WRITE_BCK_PRS_LSB_REE_INT_SHFT 0x0
-
-#define GSI_IC_WRITE_BCK_PRS_MSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000ec)
-#define GSI_IC_WRITE_BCK_PRS_MSB_RMSK 0xfc3041
-#define GSI_IC_WRITE_BCK_PRS_MSB_UCONTROLLER_INT_BMSK 0xfc0000
-#define GSI_IC_WRITE_BCK_PRS_MSB_UCONTROLLER_INT_SHFT 0x12
-#define GSI_IC_WRITE_BCK_PRS_MSB_RD_WR_INT_BMSK 0x3000
-#define GSI_IC_WRITE_BCK_PRS_MSB_RD_WR_INT_SHFT 0xc
-#define GSI_IC_WRITE_BCK_PRS_MSB_DB_ENG_INT_BMSK 0x40
-#define GSI_IC_WRITE_BCK_PRS_MSB_DB_ENG_INT_SHFT 0x6
-#define GSI_IC_WRITE_BCK_PRS_MSB_TIMER_INT_BMSK 0x1
-#define GSI_IC_WRITE_BCK_PRS_MSB_TIMER_INT_SHFT 0x0
-
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_LSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000f0)
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_LSB_RMSK 0x3ffc1047
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_LSB_TLV_INT_BMSK 0x3f000000
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_LSB_TLV_INT_SHFT 0x18
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_LSB_CSR_INT_BMSK 0xfc0000
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_LSB_CSR_INT_SHFT 0x12
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_LSB_INT_END_INT_BMSK 0x1000
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_LSB_INT_END_INT_SHFT 0xc
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_LSB_EV_ENG_INT_BMSK 0x40
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_LSB_EV_ENG_INT_SHFT 0x6
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_LSB_REE_INT_BMSK 0x7
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_LSB_REE_INT_SHFT 0x0
-
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_MSB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x000000f4)
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_MSB_RMSK 0xfc3041
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_MSB_UCONTROLLER_INT_BMSK 0xfc0000
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_MSB_UCONTROLLER_INT_SHFT 0x12
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_MSB_RD_WR_INT_BMSK 0x3000
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_MSB_RD_WR_INT_SHFT 0xc
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_MSB_DB_ENG_INT_BMSK 0x40
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_MSB_DB_ENG_INT_SHFT 0x6
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_MSB_TIMER_INT_BMSK 0x1
-#define GSI_IC_UCONTROLLER_GPR_BCK_PRS_MSB_TIMER_INT_SHFT 0x0
-
-#define GSI_GSI_IRAM_PTR_CH_CMD_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x00000400)
-#define GSI_GSI_IRAM_PTR_CH_CMD_RMSK 0xfff
-#define GSI_GSI_IRAM_PTR_CH_CMD_IRAM_PTR_BMSK 0xfff
-#define GSI_GSI_IRAM_PTR_CH_CMD_IRAM_PTR_SHFT 0x0
-
-#define GSI_GSI_IRAM_PTR_EE_GENERIC_CMD_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x00000404)
-#define GSI_GSI_IRAM_PTR_EE_GENERIC_CMD_RMSK 0xfff
-#define GSI_GSI_IRAM_PTR_EE_GENERIC_CMD_IRAM_PTR_BMSK 0xfff
-#define GSI_GSI_IRAM_PTR_EE_GENERIC_CMD_IRAM_PTR_SHFT 0x0
-
-#define GSI_V2_5_GSI_IRAM_PTR_TLV_CH_NOT_FULL_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x00000408)
-#define GSI_V2_5_GSI_IRAM_PTR_TLV_CH_NOT_FULL_RMSK 0xfff
-#define GSI_V2_5_GSI_IRAM_PTR_TLV_CH_NOT_FULL_IRAM_PTR_BMSK 0xfff
-#define GSI_V2_5_GSI_IRAM_PTR_TLV_CH_NOT_FULL_IRAM_PTR_SHFT 0x0
-
-#define GSI_GSI_IRAM_PTR_CH_DB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x00000418)
-#define GSI_GSI_IRAM_PTR_CH_DB_RMSK 0xfff
-#define GSI_GSI_IRAM_PTR_CH_DB_IRAM_PTR_BMSK 0xfff
-#define GSI_GSI_IRAM_PTR_CH_DB_IRAM_PTR_SHFT 0x0
-
-#define GSI_GSI_IRAM_PTR_EV_DB_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000041c)
-#define GSI_GSI_IRAM_PTR_EV_DB_RMSK 0xfff
-#define GSI_GSI_IRAM_PTR_EV_DB_IRAM_PTR_BMSK 0xfff
-#define GSI_GSI_IRAM_PTR_EV_DB_IRAM_PTR_SHFT 0x0
-
-#define GSI_GSI_IRAM_PTR_NEW_RE_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x00000420)
-#define GSI_GSI_IRAM_PTR_NEW_RE_RMSK 0xfff
-#define GSI_GSI_IRAM_PTR_NEW_RE_IRAM_PTR_BMSK 0xfff
-#define GSI_GSI_IRAM_PTR_NEW_RE_IRAM_PTR_SHFT 0x0
-
-#define GSI_GSI_IRAM_PTR_CH_DIS_COMP_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x00000424)
-#define GSI_GSI_IRAM_PTR_CH_DIS_COMP_RMSK 0xfff
-#define GSI_GSI_IRAM_PTR_CH_DIS_COMP_IRAM_PTR_BMSK 0xfff
-#define GSI_GSI_IRAM_PTR_CH_DIS_COMP_IRAM_PTR_SHFT 0x0
-
-#define GSI_GSI_IRAM_PTR_CH_EMPTY_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x00000428)
-#define GSI_GSI_IRAM_PTR_CH_EMPTY_RMSK 0xfff
-#define GSI_GSI_IRAM_PTR_CH_EMPTY_IRAM_PTR_BMSK 0xfff
-#define GSI_GSI_IRAM_PTR_CH_EMPTY_IRAM_PTR_SHFT 0x0
-
-#define GSI_GSI_IRAM_PTR_EVENT_GEN_COMP_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000042c)
-#define GSI_GSI_IRAM_PTR_EVENT_GEN_COMP_RMSK 0xfff
-#define GSI_GSI_IRAM_PTR_EVENT_GEN_COMP_IRAM_PTR_BMSK 0xfff
-#define GSI_GSI_IRAM_PTR_EVENT_GEN_COMP_IRAM_PTR_SHFT 0x0
-
-#define GSI_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_0_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x00000430)
-#define GSI_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_0_RMSK 0xfff
-#define GSI_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_0_IRAM_PTR_BMSK 0xfff
-#define GSI_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_0_IRAM_PTR_SHFT 0x0
-
-#define GSI_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_2_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x00000434)
-#define GSI_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_2_RMSK 0xfff
-#define GSI_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_2_IRAM_PTR_BMSK 0xfff
-#define GSI_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_2_IRAM_PTR_SHFT 0x0
-
-#define GSI_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_1_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x00000438)
-#define GSI_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_1_RMSK 0xfff
-#define GSI_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_1_IRAM_PTR_BMSK 0xfff
-#define GSI_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_1_IRAM_PTR_SHFT 0x0
-
-#define GSI_GSI_IRAM_PTR_TIMER_EXPIRED_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000043c)
-#define GSI_GSI_IRAM_PTR_TIMER_EXPIRED_RMSK 0xfff
-#define GSI_GSI_IRAM_PTR_TIMER_EXPIRED_IRAM_PTR_BMSK 0xfff
-#define GSI_GSI_IRAM_PTR_TIMER_EXPIRED_IRAM_PTR_SHFT 0x0
-
-#define GSI_GSI_IRAM_PTR_WRITE_ENG_COMP_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x00000440)
-#define GSI_GSI_IRAM_PTR_WRITE_ENG_COMP_RMSK 0xfff
-#define GSI_GSI_IRAM_PTR_WRITE_ENG_COMP_IRAM_PTR_BMSK 0xfff
-#define GSI_GSI_IRAM_PTR_WRITE_ENG_COMP_IRAM_PTR_SHFT 0x0
-
-#define GSI_GSI_IRAM_PTR_READ_ENG_COMP_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x00000444)
-#define GSI_GSI_IRAM_PTR_READ_ENG_COMP_RMSK 0xfff
-#define GSI_GSI_IRAM_PTR_READ_ENG_COMP_IRAM_PTR_BMSK 0xfff
-#define GSI_GSI_IRAM_PTR_READ_ENG_COMP_IRAM_PTR_SHFT 0x0
-
-#define GSI_GSI_IRAM_PTR_UC_GP_INT_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x00000448)
-#define GSI_GSI_IRAM_PTR_UC_GP_INT_RMSK 0xfff
-#define GSI_GSI_IRAM_PTR_UC_GP_INT_IRAM_PTR_BMSK 0xfff
-#define GSI_GSI_IRAM_PTR_UC_GP_INT_IRAM_PTR_SHFT 0x0
-
-/* Real H/W register name is with STOPPED with single P */
-#define GSI_GSI_IRAM_PTR_INT_MOD_STOPPED_OFFS \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000044c)
-#define GSI_GSI_IRAM_PTR_INT_MOD_STOPPED_RMSK 0xfff
-#define GSI_GSI_IRAM_PTR_INT_MOD_STOPPED_IRAM_PTR_BMSK 0xfff
-#define GSI_GSI_IRAM_PTR_INT_MOD_STOPPED_IRAM_PTR_SHFT 0x0
-
-#define GSI_GSI_INST_RAM_n_WORD_SZ 0x4
-#define GSI_GSI_INST_RAM_n_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00004000 + GSI_GSI_INST_RAM_n_WORD_SZ * (n))
-#define GSI_V2_5_GSI_INST_RAM_n_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001b000 + GSI_GSI_INST_RAM_n_WORD_SZ * (n))
-#define GSI_GSI_INST_RAM_n_RMSK 0xffffffff
-#define GSI_GSI_INST_RAM_n_MAXn 4095
-#define GSI_V2_0_GSI_INST_RAM_n_MAXn 6143
-#define GSI_V2_2_GSI_INST_RAM_n_MAXn 4095
-#define GSI_V2_5_GSI_INST_RAM_n_MAXn 8191
-#define GSI_V2_7_GSI_INST_RAM_n_MAXn 5119
-
-#define GSI_GSI_INST_RAM_n_INST_BYTE_3_BMSK 0xff000000
-#define GSI_GSI_INST_RAM_n_INST_BYTE_3_SHFT 0x18
-#define GSI_GSI_INST_RAM_n_INST_BYTE_2_BMSK 0xff0000
-#define GSI_GSI_INST_RAM_n_INST_BYTE_2_SHFT 0x10
-#define GSI_GSI_INST_RAM_n_INST_BYTE_1_BMSK 0xff00
-#define GSI_GSI_INST_RAM_n_INST_BYTE_1_SHFT 0x8
-#define GSI_GSI_INST_RAM_n_INST_BYTE_0_BMSK 0xff
-#define GSI_GSI_INST_RAM_n_INST_BYTE_0_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_k_CNTXT_0_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000f000 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_GSI_CH_k_CNTXT_0_ELEMENT_SIZE_BMSK 0xff000000
-#define GSI_EE_n_GSI_CH_k_CNTXT_0_ELEMENT_SIZE_SHFT 0x18
-#define GSI_EE_n_GSI_CH_k_CNTXT_0_CHSTATE_BMSK 0xf00000
-#define GSI_EE_n_GSI_CH_k_CNTXT_0_CHSTATE_SHFT 0x14
-#define GSI_EE_n_GSI_CH_k_CNTXT_0_ERINDEX_BMSK 0x7c000
-#define GSI_EE_n_GSI_CH_k_CNTXT_0_ERINDEX_SHFT 0xe
-#define GSI_V2_5_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_PROTOCOL_MSB_BMSK 0x2000
-#define GSI_V2_5_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_PROTOCOL_MSB_SHFT 0xd
-#define GSI_EE_n_GSI_CH_k_CNTXT_0_CHID_BMSK 0x1f00
-#define GSI_EE_n_GSI_CH_k_CNTXT_0_CHID_SHFT 0x8
-#define GSI_EE_n_GSI_CH_k_CNTXT_0_EE_BMSK 0xf0
-#define GSI_EE_n_GSI_CH_k_CNTXT_0_EE_SHFT 0x4
-#define GSI_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_DIR_BMSK 0x8
-#define GSI_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_DIR_SHFT 0x3
-#define GSI_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_PROTOCOL_BMSK 0x7
-#define GSI_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_PROTOCOL_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_k_CNTXT_1_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000f004 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_GSI_CH_k_CNTXT_1_R_LENGTH_BMSK 0xffff
-#define GSI_EE_n_GSI_CH_k_CNTXT_1_R_LENGTH_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_k_CNTXT_2_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000f008 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_GSI_CH_k_CNTXT_2_R_BASE_ADDR_LSBS_BMSK 0xffffffff
-#define GSI_EE_n_GSI_CH_k_CNTXT_2_R_BASE_ADDR_LSBS_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_k_CNTXT_3_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000f00c + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_GSI_CH_k_CNTXT_3_R_BASE_ADDR_MSBS_BMSK 0xffffffff
-#define GSI_EE_n_GSI_CH_k_CNTXT_3_R_BASE_ADDR_MSBS_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_k_CNTXT_4_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000f010 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_GSI_CH_k_CNTXT_4_READ_PTR_LSB_BMSK 0xffffffff
-#define GSI_EE_n_GSI_CH_k_CNTXT_4_READ_PTR_LSB_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_k_CNTXT_5_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000f014 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_GSI_CH_k_CNTXT_5_READ_PTR_MSB_BMSK 0xffffffff
-#define GSI_EE_n_GSI_CH_k_CNTXT_5_READ_PTR_MSB_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_k_CNTXT_6_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000f018 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_GSI_CH_k_CNTXT_6_WRITE_PTR_LSB_BMSK 0xffffffff
-#define GSI_EE_n_GSI_CH_k_CNTXT_6_WRITE_PTR_LSB_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_k_CNTXT_7_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000f01c + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_GSI_CH_k_CNTXT_7_WRITE_PTR_MSB_BMSK 0xffffffff
-#define GSI_EE_n_GSI_CH_k_CNTXT_7_WRITE_PTR_MSB_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000f054 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_RMSK 0xffff
-#define GSI_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_MAXk 30
-#define GSI_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_MAXn 3
-#define GSI_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_READ_PTR_BMSK 0xffff
-#define GSI_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_READ_PTR_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000f058 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_RMSK 0xffff
-#define GSI_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_MAXk 30
-#define GSI_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_MAXn 3
-#define GSI_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_RE_INTR_DB_BMSK 0xffff
-#define GSI_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_RE_INTR_DB_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_k_QOS_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001c05c + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_GSI_CH_k_QOS_RMSK 0x303
-#define GSI_EE_n_GSI_CH_k_QOS_MAXk 30
-#define GSI_EE_n_GSI_CH_k_QOS_MAXn 3
-#define GSI_EE_n_GSI_CH_k_QOS_USE_ESCAPE_BUF_ONLY_BMSK 0x400
-#define GSI_EE_n_GSI_CH_k_QOS_USE_ESCAPE_BUF_ONLY_SHFT 0xa
-#define GSI_EE_n_GSI_CH_k_QOS_USE_DB_ENG_BMSK 0x200
-#define GSI_EE_n_GSI_CH_k_QOS_USE_DB_ENG_SHFT 0x9
-#define GSI_EE_n_GSI_CH_k_QOS_MAX_PREFETCH_BMSK 0x100
-#define GSI_EE_n_GSI_CH_k_QOS_MAX_PREFETCH_SHFT 0x8
-#define GSI_EE_n_GSI_CH_k_QOS_WRR_WEIGHT_BMSK 0xf
-#define GSI_EE_n_GSI_CH_k_QOS_WRR_WEIGHT_SHFT 0x0
-
-#define GSI_V2_5_EE_n_GSI_CH_k_QOS_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000f05c + 0x4000 * (n) + 0x80 * (k))
-#define GSI_V2_5_EE_n_GSI_CH_k_QOS_EMPTY_LVL_THRSHOLD_BMSK 0xff0000
-#define GSI_V2_5_EE_n_GSI_CH_k_QOS_EMPTY_LVL_THRSHOLD_SHFT 0x10
-#define GSI_V2_5_EE_n_GSI_CH_k_QOS_PREFETCH_MODE_BMSK 0x3c00
-#define GSI_V2_5_EE_n_GSI_CH_k_QOS_PREFETCH_MODE_SHFT 0xa
-#define GSI_V2_5_EE_n_GSI_CH_k_QOS_USE_DB_ENG_BMSK 0x200
-#define GSI_V2_5_EE_n_GSI_CH_k_QOS_USE_DB_ENG_SHFT 0x9
-#define GSI_V2_5_EE_n_GSI_CH_k_QOS_MAX_PREFETCH_BMSK 0x100
-#define GSI_V2_5_EE_n_GSI_CH_k_QOS_MAX_PREFETCH_SHFT 0x8
-#define GSI_V2_5_EE_n_GSI_CH_k_QOS_WRR_WEIGHT_BMSK 0xf
-#define GSI_V2_5_EE_n_GSI_CH_k_QOS_WRR_WEIGHT_SHFT 0x0
-
-
-#define GSI_EE_n_GSI_CH_k_SCRATCH_0_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000f060 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_GSI_CH_k_SCRATCH_0_SCRATCH_BMSK 0xffffffff
-#define GSI_EE_n_GSI_CH_k_SCRATCH_0_SCRATCH_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_k_SCRATCH_1_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000f064 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_GSI_CH_k_SCRATCH_1_SCRATCH_BMSK 0xffffffff
-#define GSI_EE_n_GSI_CH_k_SCRATCH_1_SCRATCH_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_k_SCRATCH_2_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000f068 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_GSI_CH_k_SCRATCH_2_SCRATCH_BMSK 0xffffffff
-#define GSI_EE_n_GSI_CH_k_SCRATCH_2_SCRATCH_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_k_SCRATCH_3_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000f06c + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_GSI_CH_k_SCRATCH_3_SCRATCH_BMSK 0xffffffff
-#define GSI_EE_n_GSI_CH_k_SCRATCH_3_SCRATCH_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_CNTXT_0_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00010000 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_EV_CH_k_CNTXT_0_ELEMENT_SIZE_BMSK 0xff000000
-#define GSI_EE_n_EV_CH_k_CNTXT_0_ELEMENT_SIZE_SHFT 0x18
-#define GSI_EE_n_EV_CH_k_CNTXT_0_CHSTATE_BMSK 0xf00000
-#define GSI_EE_n_EV_CH_k_CNTXT_0_CHSTATE_SHFT 0x14
-#define GSI_EE_n_EV_CH_k_CNTXT_0_INTYPE_BMSK 0x10000
-#define GSI_EE_n_EV_CH_k_CNTXT_0_INTYPE_SHFT 0x10
-#define GSI_EE_n_EV_CH_k_CNTXT_0_EVCHID_BMSK 0xff00
-#define GSI_EE_n_EV_CH_k_CNTXT_0_EVCHID_SHFT 0x8
-#define GSI_EE_n_EV_CH_k_CNTXT_0_EE_BMSK 0xf0
-#define GSI_EE_n_EV_CH_k_CNTXT_0_EE_SHFT 0x4
-#define GSI_EE_n_EV_CH_k_CNTXT_0_CHTYPE_BMSK 0xf
-#define GSI_EE_n_EV_CH_k_CNTXT_0_CHTYPE_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_CNTXT_1_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00010004 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_EV_CH_k_CNTXT_1_R_LENGTH_BMSK 0xffff
-#define GSI_EE_n_EV_CH_k_CNTXT_1_R_LENGTH_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_CNTXT_2_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00010008 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_EV_CH_k_CNTXT_2_R_BASE_ADDR_LSBS_BMSK 0xffffffff
-#define GSI_EE_n_EV_CH_k_CNTXT_2_R_BASE_ADDR_LSBS_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_CNTXT_3_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001000c + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_EV_CH_k_CNTXT_3_R_BASE_ADDR_MSBS_BMSK 0xffffffff
-#define GSI_EE_n_EV_CH_k_CNTXT_3_R_BASE_ADDR_MSBS_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_CNTXT_4_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00010010 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_EV_CH_k_CNTXT_4_READ_PTR_LSB_BMSK 0xffffffff
-#define GSI_EE_n_EV_CH_k_CNTXT_4_READ_PTR_LSB_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_CNTXT_5_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00010014 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_EV_CH_k_CNTXT_5_READ_PTR_MSB_BMSK 0xffffffff
-#define GSI_EE_n_EV_CH_k_CNTXT_5_READ_PTR_MSB_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_CNTXT_6_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00010018 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_EV_CH_k_CNTXT_6_WRITE_PTR_LSB_BMSK 0xffffffff
-#define GSI_EE_n_EV_CH_k_CNTXT_6_WRITE_PTR_LSB_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_CNTXT_7_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001001c + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_EV_CH_k_CNTXT_7_WRITE_PTR_MSB_BMSK 0xffffffff
-#define GSI_EE_n_EV_CH_k_CNTXT_7_WRITE_PTR_MSB_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_CNTXT_8_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00010020 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_EV_CH_k_CNTXT_8_INT_MOD_CNT_BMSK 0xff000000
-#define GSI_EE_n_EV_CH_k_CNTXT_8_INT_MOD_CNT_SHFT 0x18
-#define GSI_EE_n_EV_CH_k_CNTXT_8_INT_MODC_BMSK 0xff0000
-#define GSI_EE_n_EV_CH_k_CNTXT_8_INT_MODC_SHFT 0x10
-#define GSI_EE_n_EV_CH_k_CNTXT_8_INT_MODT_BMSK 0xffff
-#define GSI_EE_n_EV_CH_k_CNTXT_8_INT_MODT_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_CNTXT_9_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00010024 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_EV_CH_k_CNTXT_9_INTVEC_BMSK 0xffffffff
-#define GSI_EE_n_EV_CH_k_CNTXT_9_INTVEC_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_CNTXT_10_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00010028 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_EV_CH_k_CNTXT_10_MSI_ADDR_LSB_BMSK 0xffffffff
-#define GSI_EE_n_EV_CH_k_CNTXT_10_MSI_ADDR_LSB_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_CNTXT_11_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001002c + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_EV_CH_k_CNTXT_11_MSI_ADDR_MSB_BMSK 0xffffffff
-#define GSI_EE_n_EV_CH_k_CNTXT_11_MSI_ADDR_MSB_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_CNTXT_12_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00010030 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_EV_CH_k_CNTXT_12_RP_UPDATE_ADDR_LSB_BMSK 0xffffffff
-#define GSI_EE_n_EV_CH_k_CNTXT_12_RP_UPDATE_ADDR_LSB_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_CNTXT_13_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00010034 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_EV_CH_k_CNTXT_13_RP_UPDATE_ADDR_MSB_BMSK 0xffffffff
-#define GSI_EE_n_EV_CH_k_CNTXT_13_RP_UPDATE_ADDR_MSB_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_SCRATCH_0_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00010048 + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_EV_CH_k_SCRATCH_0_SCRATCH_BMSK 0xffffffff
-#define GSI_EE_n_EV_CH_k_SCRATCH_0_SCRATCH_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_SCRATCH_1_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001004c + 0x4000 * (n) + 0x80 * (k))
-#define GSI_EE_n_EV_CH_k_SCRATCH_1_SCRATCH_BMSK 0xffffffff
-#define GSI_EE_n_EV_CH_k_SCRATCH_1_SCRATCH_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_k_DOORBELL_0_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00011000 + 0x4000 * (n) + 0x8 * (k))
-#define GSI_EE_n_GSI_CH_k_DOORBELL_0_WRITE_PTR_LSB_BMSK 0xffffffff
-#define GSI_EE_n_GSI_CH_k_DOORBELL_0_WRITE_PTR_LSB_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_k_DOORBELL_1_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00011004 + 0x4000 * (n) + 0x8 * (k))
-#define GSI_EE_n_GSI_CH_k_DOORBELL_1_WRITE_PTR_MSB_BMSK 0xffffffff
-#define GSI_EE_n_GSI_CH_k_DOORBELL_1_WRITE_PTR_MSB_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_DOORBELL_0_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00011100 + 0x4000 * (n) + 0x8 * (k))
-#define GSI_EE_n_EV_CH_k_DOORBELL_0_WRITE_PTR_LSB_BMSK 0xffffffff
-#define GSI_EE_n_EV_CH_k_DOORBELL_0_WRITE_PTR_LSB_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_k_DOORBELL_1_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00011104 + 0x4000 * (n) + 0x8 * (k))
-#define GSI_EE_n_EV_CH_k_DOORBELL_1_WRITE_PTR_MSB_BMSK 0xffffffff
-#define GSI_EE_n_EV_CH_k_DOORBELL_1_WRITE_PTR_MSB_SHFT 0x0
-
-#define GSI_EE_n_GSI_STATUS_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00012000 + 0x4000 * (n))
-#define GSI_EE_n_GSI_STATUS_ENABLED_BMSK 0x1
-#define GSI_EE_n_GSI_STATUS_ENABLED_SHFT 0x0
-
-#define GSI_EE_n_GSI_CH_CMD_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00012008 + 0x4000 * (n))
-#define GSI_EE_n_GSI_CH_CMD_OPCODE_BMSK 0xff000000
-#define GSI_EE_n_GSI_CH_CMD_OPCODE_SHFT 0x18
-#define GSI_EE_n_GSI_CH_CMD_CHID_BMSK 0xff
-#define GSI_EE_n_GSI_CH_CMD_CHID_SHFT 0x0
-
-#define GSI_EE_n_EV_CH_CMD_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00012010 + 0x4000 * (n))
-#define GSI_EE_n_EV_CH_CMD_OPCODE_BMSK 0xff000000
-#define GSI_EE_n_EV_CH_CMD_OPCODE_SHFT 0x18
-#define GSI_EE_n_EV_CH_CMD_CHID_BMSK 0xff
-#define GSI_EE_n_EV_CH_CMD_CHID_SHFT 0x0
-
-#define GSI_EE_n_GSI_EE_GENERIC_CMD_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00012018 + 0x4000 * (n))
-#define GSI_EE_n_GSI_EE_GENERIC_CMD_OPCODE_BMSK 0x1f
-#define GSI_EE_n_GSI_EE_GENERIC_CMD_OPCODE_SHFT 0x0
-#define GSI_EE_n_GSI_EE_GENERIC_CMD_VIRT_CHAN_IDX_BMSK 0x3e0
-#define GSI_EE_n_GSI_EE_GENERIC_CMD_VIRT_CHAN_IDX_SHFT 0x5
-#define GSI_EE_n_GSI_EE_GENERIC_CMD_EE_BMSK 0x3c00
-#define GSI_EE_n_GSI_EE_GENERIC_CMD_EE_SHFT 0xa
-
-#define GSI_V1_0_EE_n_GSI_HW_PARAM_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f040 + 0x4000 * (n))
-#define GSI_V1_0_EE_n_GSI_HW_PARAM_PERIPH_SEC_GRP_BMSK 0x7c000000
-#define GSI_V1_0_EE_n_GSI_HW_PARAM_PERIPH_SEC_GRP_SHFT 0x1a
-#define GSI_V1_0_EE_n_GSI_HW_PARAM_USE_AXI_M_BMSK 0x2000000
-#define GSI_V1_0_EE_n_GSI_HW_PARAM_USE_AXI_M_SHFT 0x19
-#define GSI_V1_0_EE_n_GSI_HW_PARAM_PERIPH_CONF_ADDR_BUS_W_BMSK 0x1f00000
-#define GSI_V1_0_EE_n_GSI_HW_PARAM_PERIPH_CONF_ADDR_BUS_W_SHFT 0x14
-#define GSI_V1_0_EE_n_GSI_HW_PARAM_NUM_EES_BMSK 0xf0000
-#define GSI_V1_0_EE_n_GSI_HW_PARAM_NUM_EES_SHFT 0x10
-#define GSI_V1_0_EE_n_GSI_HW_PARAM_GSI_CH_NUM_BMSK 0xff00
-#define GSI_V1_0_EE_n_GSI_HW_PARAM_GSI_CH_NUM_SHFT 0x8
-#define GSI_V1_0_EE_n_GSI_HW_PARAM_GSI_EV_CH_NUM_BMSK 0xff
-#define GSI_V1_0_EE_n_GSI_HW_PARAM_GSI_EV_CH_NUM_SHFT 0x0
-
-#define GSI_V1_2_EE_n_GSI_HW_PARAM_0_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f038 + 0x4000 * (n))
-#define GSI_V1_2_EE_n_GSI_HW_PARAM_0_USE_AXI_M_BMSK 0x80000000
-#define GSI_V1_2_EE_n_GSI_HW_PARAM_0_USE_AXI_M_SHFT 0x1f
-#define GSI_V1_2_EE_n_GSI_HW_PARAM_0_PERIPH_SEC_GRP_BMSK 0x7c000000
-#define GSI_V1_2_EE_n_GSI_HW_PARAM_0_PERIPH_SEC_GRP_SHFT 0x1a
-#define GSI_V1_2_EE_n_GSI_HW_PARAM_0_PERIPH_CONF_ADDR_BUS_W_BMSK 0x3e00000
-#define GSI_V1_2_EE_n_GSI_HW_PARAM_0_PERIPH_CONF_ADDR_BUS_W_SHFT 0x15
-#define GSI_V1_2_EE_n_GSI_HW_PARAM_0_NUM_EES_BMSK 0x1f0000
-#define GSI_V1_2_EE_n_GSI_HW_PARAM_0_NUM_EES_SHFT 0x10
-#define GSI_V1_2_EE_n_GSI_HW_PARAM_0_GSI_CH_NUM_BMSK 0xff00
-#define GSI_V1_2_EE_n_GSI_HW_PARAM_0_GSI_CH_NUM_SHFT 0x8
-#define GSI_V1_2_EE_n_GSI_HW_PARAM_0_GSI_EV_CH_NUM_BMSK 0xff
-#define GSI_V1_2_EE_n_GSI_HW_PARAM_0_GSI_EV_CH_NUM_SHFT 0x0
-
-#define GSI_V1_3_EE_n_GSI_HW_PARAM_2_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f040 + 0x4000 * (n))
-#define GSI_V1_3_EE_n_GSI_HW_PARAM_2_RMSK 0x7fff
-#define GSI_V1_3_EE_n_GSI_HW_PARAM_2_MAXn 2
-#define GSI_V1_3_EE_n_GSI_HW_PARAM_2_GSI_CH_FULL_LOGIC_BMSK 0x4000
-#define GSI_V1_3_EE_n_GSI_HW_PARAM_2_GSI_CH_FULL_LOGIC_SHFT 0xe
-#define GSI_V1_3_EE_n_GSI_HW_PARAM_2_GSI_CH_PEND_TRANSLATE_BMSK 0x2000
-#define GSI_V1_3_EE_n_GSI_HW_PARAM_2_GSI_CH_PEND_TRANSLATE_SHFT 0xd
-#define GSI_V1_3_EE_n_GSI_HW_PARAM_2_GSI_NUM_EV_PER_EE_BMSK 0x1f00
-#define GSI_V1_3_EE_n_GSI_HW_PARAM_2_GSI_NUM_EV_PER_EE_SHFT 0x8
-#define GSI_V1_3_EE_n_GSI_HW_PARAM_2_GSI_NUM_CH_PER_EE_BMSK 0xf8
-#define GSI_V1_3_EE_n_GSI_HW_PARAM_2_GSI_NUM_CH_PER_EE_SHFT 0x3
-#define GSI_V1_3_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_BMSK 0x7
-#define GSI_V1_3_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_SHFT 0x0
-#define GSI_V1_3_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_ONE_KB_FVAL 0x0
-#define GSI_V1_3_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_TWO_KB_FVAL 0x1
-
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001f040 + 0x4000 * (n))
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_RMSK 0x7fff
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_MAXn 2
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_SDMA_N_IOVEC_BMSK 0x38000000
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_SDMA_N_IOVEC_SHFT 0x1b
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_SDMA_MAX_BURST_BMSK 0x7F80000
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_SDMA_MAX_BURST_SHFT 0x13
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_SDMA_N_INT_BMSK 0x70000
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_SDMA_N_INT_SHFT 0x10
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_USE_SDMA_BMSK 0x8000
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_USE_SDMA_SHFT 0xf
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_CH_FULL_LOGIC_BMSK 0x4000
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_CH_FULL_LOGIC_SHFT 0xe
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_CH_PEND_TRANSLATE_BMSK 0x2000
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_CH_PEND_TRANSLATE_SHFT 0xd
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_NUM_EV_PER_EE_BMSK 0x1f00
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_NUM_EV_PER_EE_SHFT 0x8
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_NUM_CH_PER_EE_BMSK 0xf8
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_NUM_CH_PER_EE_SHFT 0x3
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_BMSK 0x7
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_SHFT 0x0
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_ONE_KB_FVAL 0x0
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_TWO_KB_FVAL 0x1
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_TWO_N_HALF_KB_FVAL 0x2
-#define GSI_V2_0_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_THREE_KB_FVAL 0x3
-
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_OFFS(n) \
-		(GSI_GSI_REG_BASE_OFFS + 0x0001f040 + 0x4000 * (n))
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_USE_INTER_EE_BMSK 0x8000
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_USE_INTER_EE_SHFT 0xf
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_USE_RD_WR_ENG_BMSK 0x4000
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_USE_RD_WR_ENG_SHFT 0xE
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_RMSK 0x7fff
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_MAXn 2
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_SDMA_N_IOVEC_BMSK 0x38000000
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_SDMA_N_IOVEC_SHFT 0x1b
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_SDMA_MAX_BURST_BMSK 0x7F80000
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_SDMA_MAX_BURST_SHFT 0x13
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_SDMA_N_INT_BMSK 0x70000
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_SDMA_N_INT_SHFT 0x10
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_USE_SDMA_BMSK 0x8000
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_USE_SDMA_SHFT 0xf
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_CH_FULL_LOGIC_BMSK 0x4000
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_CH_FULL_LOGIC_SHFT 0xe
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_CH_PEND_TRANSLATE_BMSK 0x2000
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_CH_PEND_TRANSLATE_SHFT 0xd
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_NUM_EV_PER_EE_BMSK 0x1f00
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_NUM_EV_PER_EE_SHFT 0x8
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_NUM_CH_PER_EE_BMSK 0xf8
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_NUM_CH_PER_EE_SHFT 0x3
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_BMSK 0x7
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_SHFT 0x0
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_ONE_KB_FVAL 0x0
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_TWO_KB_FVAL 0x1
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_TWO_N_HALF_KB_FVAL 0x2
-#define GSI_V2_2_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_THREE_KB_FVAL 0x3
-
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_OFFS(n) \
-		(GSI_GSI_REG_BASE_OFFS + 0x00012040 + 0x4000 * (n))
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_USE_INTER_EE_BMSK 0x8000
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_USE_INTER_EE_SHFT 0xf
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_USE_RD_WR_ENG_BMSK 0x4000
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_USE_RD_WR_ENG_SHFT 0xE
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_RMSK 0x7fff
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_MAXn 2
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_SDMA_N_IOVEC_BMSK 0x38000000
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_SDMA_N_IOVEC_SHFT 0x1b
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_SDMA_MAX_BURST_BMSK 0x7F80000
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_SDMA_MAX_BURST_SHFT 0x13
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_SDMA_N_INT_BMSK 0x70000
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_SDMA_N_INT_SHFT 0x10
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_USE_SDMA_BMSK 0x8000
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_USE_SDMA_SHFT 0xf
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_CH_FULL_LOGIC_BMSK 0x4000
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_CH_FULL_LOGIC_SHFT 0xe
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_CH_PEND_TRANSLATE_BMSK 0x2000
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_CH_PEND_TRANSLATE_SHFT 0xd
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_NUM_EV_PER_EE_BMSK 0x1f00
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_NUM_EV_PER_EE_SHFT 0x8
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_NUM_CH_PER_EE_BMSK 0xf8
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_NUM_CH_PER_EE_SHFT 0x3
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_BMSK 0x7
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_SHFT 0x0
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_ONE_KB_FVAL 0x0
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_TWO_KB_FVAL 0x1
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_TWO_N_HALF_KB_FVAL 0x2
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_THREE_KB_FVAL 0x3
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_THREE_N_HALF_KB_FVAL 0x4
-#define GSI_V2_5_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_FOUR_KB_FVAL 0x5
-
-
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_OFFS(n) \
-			(GSI_GSI_REG_BASE_OFFS + 0x00012040 + 0x4000 * (n))
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_USE_INTER_EE_BMSK 0x8000
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_USE_INTER_EE_SHFT 0xf
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_USE_RD_WR_ENG_BMSK 0x4000
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_USE_RD_WR_ENG_SHFT 0xE
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_RMSK 0x7fff
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_MAXn 2
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_SDMA_N_IOVEC_BMSK 0x38000000
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_SDMA_N_IOVEC_SHFT 0x1b
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_SDMA_MAX_BURST_BMSK 0x7F80000
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_SDMA_MAX_BURST_SHFT 0x13
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_SDMA_N_INT_BMSK 0x70000
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_SDMA_N_INT_SHFT 0x10
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_USE_SDMA_BMSK 0x8000
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_USE_SDMA_SHFT 0xf
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_CH_FULL_LOGIC_BMSK 0x4000
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_CH_FULL_LOGIC_SHFT 0xe
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_CH_PEND_TRANSLATE_BMSK 0x2000
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_CH_PEND_TRANSLATE_SHFT 0xd
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_NUM_EV_PER_EE_BMSK 0x1f00
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_NUM_EV_PER_EE_SHFT 0x8
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_NUM_CH_PER_EE_BMSK 0xf8
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_NUM_CH_PER_EE_SHFT 0x3
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_BMSK 0x7
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_SHFT 0x0
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_ONE_KB_FVAL 0x0
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_TWO_KB_FVAL 0x1
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_TWO_N_HALF_KB_FVAL 0x2
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_THREE_KB_FVAL 0x3
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_THREE_N_HALF_KB_FVAL 0x4
-#define GSI_V2_7_EE_n_GSI_HW_PARAM_2_GSI_IRAM_SIZE_FOUR_KB_FVAL 0x5
-
-#define GSI_EE_n_GSI_SW_VERSION_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00012044 + 0x4000 * (n))
-#define GSI_EE_n_GSI_SW_VERSION_MAJOR_BMSK 0xf0000000
-#define GSI_EE_n_GSI_SW_VERSION_MAJOR_SHFT 0x1c
-#define GSI_EE_n_GSI_SW_VERSION_MINOR_BMSK 0xfff0000
-#define GSI_EE_n_GSI_SW_VERSION_MINOR_SHFT 0x10
-#define GSI_EE_n_GSI_SW_VERSION_STEP_BMSK 0xffff
-#define GSI_EE_n_GSI_SW_VERSION_STEP_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_TYPE_IRQ_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00012080 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_TYPE_IRQ_GENERAL_BMSK 0x40
-#define GSI_EE_n_CNTXT_TYPE_IRQ_GENERAL_SHFT 0x6
-#define GSI_EE_n_CNTXT_TYPE_IRQ_INTER_EE_EV_CTRL_BMSK 0x20
-#define GSI_EE_n_CNTXT_TYPE_IRQ_INTER_EE_EV_CTRL_SHFT 0x5
-#define GSI_EE_n_CNTXT_TYPE_IRQ_INTER_EE_CH_CTRL_BMSK 0x10
-#define GSI_EE_n_CNTXT_TYPE_IRQ_INTER_EE_CH_CTRL_SHFT 0x4
-#define GSI_EE_n_CNTXT_TYPE_IRQ_IEOB_BMSK 0x8
-#define GSI_EE_n_CNTXT_TYPE_IRQ_IEOB_SHFT 0x3
-#define GSI_EE_n_CNTXT_TYPE_IRQ_GLOB_EE_BMSK 0x4
-#define GSI_EE_n_CNTXT_TYPE_IRQ_GLOB_EE_SHFT 0x2
-#define GSI_EE_n_CNTXT_TYPE_IRQ_EV_CTRL_BMSK 0x2
-#define GSI_EE_n_CNTXT_TYPE_IRQ_EV_CTRL_SHFT 0x1
-#define GSI_EE_n_CNTXT_TYPE_IRQ_CH_CTRL_BMSK 0x1
-#define GSI_EE_n_CNTXT_TYPE_IRQ_CH_CTRL_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_TYPE_IRQ_MSK_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00012088 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_TYPE_IRQ_MSK_GENERAL_BMSK 0x40
-#define GSI_EE_n_CNTXT_TYPE_IRQ_MSK_GENERAL_SHFT 0x6
-#define GSI_EE_n_CNTXT_TYPE_IRQ_MSK_INTER_EE_EV_CTRL_BMSK 0x20
-#define GSI_EE_n_CNTXT_TYPE_IRQ_MSK_INTER_EE_EV_CTRL_SHFT 0x5
-#define GSI_EE_n_CNTXT_TYPE_IRQ_MSK_INTER_EE_CH_CTRL_BMSK 0x10
-#define GSI_EE_n_CNTXT_TYPE_IRQ_MSK_INTER_EE_CH_CTRL_SHFT 0x4
-#define GSI_EE_n_CNTXT_TYPE_IRQ_MSK_IEOB_BMSK 0x8
-#define GSI_EE_n_CNTXT_TYPE_IRQ_MSK_IEOB_SHFT 0x3
-#define GSI_EE_n_CNTXT_TYPE_IRQ_MSK_GLOB_EE_BMSK 0x4
-#define GSI_EE_n_CNTXT_TYPE_IRQ_MSK_GLOB_EE_SHFT 0x2
-#define GSI_EE_n_CNTXT_TYPE_IRQ_MSK_EV_CTRL_BMSK 0x2
-#define GSI_EE_n_CNTXT_TYPE_IRQ_MSK_EV_CTRL_SHFT 0x1
-#define GSI_EE_n_CNTXT_TYPE_IRQ_MSK_CH_CTRL_BMSK 0x1
-#define GSI_EE_n_CNTXT_TYPE_IRQ_MSK_CH_CTRL_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_SRC_GSI_CH_IRQ_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00012090 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_SRC_GSI_CH_IRQ_GSI_CH_BIT_MAP_BMSK 0xffffffff
-#define GSI_EE_n_CNTXT_SRC_GSI_CH_IRQ_GSI_CH_BIT_MAP_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_SRC_EV_CH_IRQ_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00012094 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_SRC_EV_CH_IRQ_EV_CH_BIT_MAP_BMSK 0xffffffff
-#define GSI_EE_n_CNTXT_SRC_EV_CH_IRQ_EV_CH_BIT_MAP_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00012098 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_GSI_CH_BIT_MAP_MSK_BMSK 0x1ffff
-#define GSI_V2_5_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_GSI_CH_BIT_MAP_MSK_BMSK 0x7fffff
-#define GSI_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_GSI_CH_BIT_MAP_MSK_SHFT 0x0
-#define GSI_V2_5_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_GSI_CH_BIT_MAP_MSK_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001209c + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_EV_CH_BIT_MAP_MSK_BMSK 0xfff
-#define GSI_V2_5_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_EV_CH_BIT_MAP_MSK_BMSK 0xfffff
-#define GSI_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_EV_CH_BIT_MAP_MSK_SHFT 0x0
-#define GSI_V2_5_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_EV_CH_BIT_MAP_MSK_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_SRC_GSI_CH_IRQ_CLR_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x000120a0 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_SRC_GSI_CH_IRQ_CLR_GSI_CH_BIT_MAP_BMSK 0xffffffff
-#define GSI_EE_n_CNTXT_SRC_GSI_CH_IRQ_CLR_GSI_CH_BIT_MAP_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_SRC_EV_CH_IRQ_CLR_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x000120a4 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_SRC_EV_CH_IRQ_CLR_EV_CH_BIT_MAP_BMSK 0xffffffff
-#define GSI_EE_n_CNTXT_SRC_EV_CH_IRQ_CLR_EV_CH_BIT_MAP_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_SRC_IEOB_IRQ_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x000120b0 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_SRC_IEOB_IRQ_EV_CH_BIT_MAP_BMSK 0xffffffff
-#define GSI_EE_n_CNTXT_SRC_IEOB_IRQ_EV_CH_BIT_MAP_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x000120b8 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_EV_CH_BIT_MAP_MSK_BMSK 0xfff
-#define GSI_V2_5_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_EV_CH_BIT_MAP_MSK_BMSK 0xfffff
-#define GSI_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_EV_CH_BIT_MAP_MSK_SHFT 0x0
-#define GSI_V2_5_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_EV_CH_BIT_MAP_MSK_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_SRC_IEOB_IRQ_CLR_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x000120c0 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_SRC_IEOB_IRQ_CLR_EV_CH_BIT_MAP_BMSK 0xffffffff
-#define GSI_EE_n_CNTXT_SRC_IEOB_IRQ_CLR_EV_CH_BIT_MAP_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_GLOB_IRQ_STTS_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00012100 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_GLOB_IRQ_STTS_GP_INT3_BMSK 0x8
-#define GSI_EE_n_CNTXT_GLOB_IRQ_STTS_GP_INT3_SHFT 0x3
-#define GSI_EE_n_CNTXT_GLOB_IRQ_STTS_GP_INT2_BMSK 0x4
-#define GSI_EE_n_CNTXT_GLOB_IRQ_STTS_GP_INT2_SHFT 0x2
-#define GSI_EE_n_CNTXT_GLOB_IRQ_STTS_GP_INT1_BMSK 0x2
-#define GSI_EE_n_CNTXT_GLOB_IRQ_STTS_GP_INT1_SHFT 0x1
-#define GSI_EE_n_CNTXT_GLOB_IRQ_STTS_ERROR_INT_BMSK 0x1
-#define GSI_EE_n_CNTXT_GLOB_IRQ_STTS_ERROR_INT_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_GLOB_IRQ_EN_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00012108 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_GLOB_IRQ_EN_GP_INT3_BMSK 0x8
-#define GSI_EE_n_CNTXT_GLOB_IRQ_EN_GP_INT3_SHFT 0x3
-#define GSI_EE_n_CNTXT_GLOB_IRQ_EN_GP_INT2_BMSK 0x4
-#define GSI_EE_n_CNTXT_GLOB_IRQ_EN_GP_INT2_SHFT 0x2
-#define GSI_EE_n_CNTXT_GLOB_IRQ_EN_GP_INT1_BMSK 0x2
-#define GSI_EE_n_CNTXT_GLOB_IRQ_EN_GP_INT1_SHFT 0x1
-#define GSI_EE_n_CNTXT_GLOB_IRQ_EN_ERROR_INT_BMSK 0x1
-#define GSI_EE_n_CNTXT_GLOB_IRQ_EN_ERROR_INT_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_GLOB_IRQ_CLR_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00012110 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_GLOB_IRQ_CLR_GP_INT3_BMSK 0x8
-#define GSI_EE_n_CNTXT_GLOB_IRQ_CLR_GP_INT3_SHFT 0x3
-#define GSI_EE_n_CNTXT_GLOB_IRQ_CLR_GP_INT2_BMSK 0x4
-#define GSI_EE_n_CNTXT_GLOB_IRQ_CLR_GP_INT2_SHFT 0x2
-#define GSI_EE_n_CNTXT_GLOB_IRQ_CLR_GP_INT1_BMSK 0x2
-#define GSI_EE_n_CNTXT_GLOB_IRQ_CLR_GP_INT1_SHFT 0x1
-#define GSI_EE_n_CNTXT_GLOB_IRQ_CLR_ERROR_INT_BMSK 0x1
-#define GSI_EE_n_CNTXT_GLOB_IRQ_CLR_ERROR_INT_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_GSI_IRQ_STTS_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00012118 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_GSI_IRQ_STTS_GSI_MCS_STACK_OVRFLOW_BMSK 0x8
-#define GSI_EE_n_CNTXT_GSI_IRQ_STTS_GSI_MCS_STACK_OVRFLOW_SHFT 0x3
-#define GSI_EE_n_CNTXT_GSI_IRQ_STTS_GSI_CMD_FIFO_OVRFLOW_BMSK 0x4
-#define GSI_EE_n_CNTXT_GSI_IRQ_STTS_GSI_CMD_FIFO_OVRFLOW_SHFT 0x2
-#define GSI_EE_n_CNTXT_GSI_IRQ_STTS_GSI_BUS_ERROR_BMSK 0x2
-#define GSI_EE_n_CNTXT_GSI_IRQ_STTS_GSI_BUS_ERROR_SHFT 0x1
-#define GSI_EE_n_CNTXT_GSI_IRQ_STTS_GSI_BREAK_POINT_BMSK 0x1
-#define GSI_EE_n_CNTXT_GSI_IRQ_STTS_GSI_BREAK_POINT_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_GSI_IRQ_EN_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00012120 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_GSI_IRQ_EN_GSI_MCS_STACK_OVRFLOW_BMSK 0x8
-#define GSI_EE_n_CNTXT_GSI_IRQ_EN_GSI_MCS_STACK_OVRFLOW_SHFT 0x3
-#define GSI_EE_n_CNTXT_GSI_IRQ_EN_GSI_CMD_FIFO_OVRFLOW_BMSK 0x4
-#define GSI_EE_n_CNTXT_GSI_IRQ_EN_GSI_CMD_FIFO_OVRFLOW_SHFT 0x2
-#define GSI_EE_n_CNTXT_GSI_IRQ_EN_GSI_BUS_ERROR_BMSK 0x2
-#define GSI_EE_n_CNTXT_GSI_IRQ_EN_GSI_BUS_ERROR_SHFT 0x1
-#define GSI_EE_n_CNTXT_GSI_IRQ_EN_GSI_BREAK_POINT_BMSK 0x1
-#define GSI_EE_n_CNTXT_GSI_IRQ_EN_GSI_BREAK_POINT_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_GSI_IRQ_CLR_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00012128 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_GSI_IRQ_CLR_GSI_MCS_STACK_OVRFLOW_BMSK 0x8
-#define GSI_EE_n_CNTXT_GSI_IRQ_CLR_GSI_MCS_STACK_OVRFLOW_SHFT 0x3
-#define GSI_EE_n_CNTXT_GSI_IRQ_CLR_GSI_CMD_FIFO_OVRFLOW_BMSK 0x4
-#define GSI_EE_n_CNTXT_GSI_IRQ_CLR_GSI_CMD_FIFO_OVRFLOW_SHFT 0x2
-#define GSI_EE_n_CNTXT_GSI_IRQ_CLR_GSI_BUS_ERROR_BMSK 0x2
-#define GSI_EE_n_CNTXT_GSI_IRQ_CLR_GSI_BUS_ERROR_SHFT 0x1
-#define GSI_EE_n_CNTXT_GSI_IRQ_CLR_GSI_BREAK_POINT_BMSK 0x1
-#define GSI_EE_n_CNTXT_GSI_IRQ_CLR_GSI_BREAK_POINT_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_MSI_BASE_LSB(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00012188 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_MSI_BASE_MSB(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0001218c + 0x4000 * (n))
-
-#define GSI_EE_n_CNTXT_INTSET_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00012180 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_INTSET_INTYPE_BMSK 0x1
-#define GSI_EE_n_CNTXT_INTSET_INTYPE_SHFT 0x0
-
-#define GSI_EE_n_ERROR_LOG_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00012200 + 0x4000 * (n))
-#define GSI_EE_n_ERROR_LOG_TODO_BMSK 0xffffffff
-#define GSI_EE_n_ERROR_LOG_TODO_SHFT 0x0
-
-#define GSI_EE_n_ERROR_LOG_CLR_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00012210 + 0x4000 * (n))
-#define GSI_EE_n_ERROR_LOG_CLR_TODO_BMSK 0xffffffff
-#define GSI_EE_n_ERROR_LOG_CLR_TODO_SHFT 0x0
-
-#define GSI_EE_n_CNTXT_SCRATCH_0_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00012400 + 0x4000 * (n))
-#define GSI_EE_n_CNTXT_SCRATCH_0_SCRATCH_BMSK 0xffffffff
-#define GSI_EE_n_CNTXT_SCRATCH_0_SCRATCH_SHFT 0x0
-
-#define GSI_INTER_EE_n_SRC_GSI_CH_IRQ_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000c018 + 0x1000 * (n))
-#define GSI_INTER_EE_n_SRC_GSI_CH_IRQ_GSI_CH_BIT_MAP_BMSK 0xffffffff
-#define GSI_INTER_EE_n_SRC_GSI_CH_IRQ_GSI_CH_BIT_MAP_SHFT 0x0
-
-#define GSI_INTER_EE_n_SRC_EV_CH_IRQ_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000c01c + 0x1000 * (n))
-#define GSI_INTER_EE_n_SRC_EV_CH_IRQ_EV_CH_BIT_MAP_BMSK 0xffffffff
-#define GSI_INTER_EE_n_SRC_EV_CH_IRQ_EV_CH_BIT_MAP_SHFT 0x0
-
-#define GSI_INTER_EE_n_SRC_GSI_CH_IRQ_CLR_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000c028 + 0x1000 * (n))
-#define GSI_INTER_EE_n_SRC_GSI_CH_IRQ_CLR_GSI_CH_BIT_MAP_BMSK 0xffffffff
-#define GSI_INTER_EE_n_SRC_GSI_CH_IRQ_CLR_GSI_CH_BIT_MAP_SHFT 0x0
-
-#define GSI_INTER_EE_n_SRC_EV_CH_IRQ_CLR_OFFS(n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x0000c02c + 0x1000 * (n))
-#define GSI_INTER_EE_n_SRC_EV_CH_IRQ_CLR_EV_CH_BIT_MAP_BMSK 0xffffffff
-#define GSI_INTER_EE_n_SRC_EV_CH_IRQ_CLR_EV_CH_BIT_MAP_SHFT 0x0
-
-#define GSI_V2_5_GSI_MAP_EE_n_CH_k_VP_TABLE_OFFS(k, n) \
-	(GSI_GSI_REG_BASE_OFFS + 0x00003800 + 0x80 * (n) + 0x4 * (k))
-#define GSI_V2_5_GSI_MAP_EE_n_CH_k_VP_TABLE_VALID_BMSK 0x20
-#define GSI_V2_5_GSI_MAP_EE_n_CH_k_VP_TABLE_VALID_SHFT 0x5
-#define GSI_V2_5_GSI_MAP_EE_n_CH_k_VP_TABLE_PHY_CH_BMSK 0x1f
-#define GSI_V2_5_GSI_MAP_EE_n_CH_k_VP_TABLE_PHY_CH_SHFT 0x0
-
-#endif /* __GSI_REG_V2_H__ */

ipa.c

@@ -0,0 +1,5 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+static void _dataipa_techpack_stub(void)
+{
+}

ipa/Makefile

@@ -1,6 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-
-obj-$(CONFIG_IPA3) += ipa_v3/ ipa_clients/ ipa_common
-obj-$(CONFIG_IPA_UT) += test/
-
-ipa_common += ipa_api.o ipa_rm.o ipa_rm_dependency_graph.o ipa_rm_peers_list.o ipa_rm_resource.o ipa_rm_inactivity_timer.o

ipa/ipa_api.c

@@ -1,3749 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2015-2019, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/ipa.h>
-#include <linux/device.h>
-#include <linux/init.h>
-#include <linux/kernel.h>
-#include <linux/mm.h>
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/platform_device.h>
-#include <linux/ipa_uc_offload.h>
-#include <linux/pci.h>
-#include "ipa_api.h"
-#include "ipa_v3/ipa_i.h"
-
-/*
- * The following for adding code (ie. for EMULATION) not found on x86.
- */
-#if defined(CONFIG_IPA_EMULATION)
-# include "ipa_v3/ipa_emulation_stubs.h"
-#endif
-
-#define DRV_NAME "ipa"
-
-#define IPA_API_DISPATCH_RETURN(api, p...) \
-	do { \
-		if (!ipa_api_ctrl) { \
-			pr_err("%s:%d IPA HW is not supported\n", \
-				__func__, __LINE__); \
-			ret = -EPERM; \
-		} \
-		else { \
-			if (ipa_api_ctrl->api) { \
-				ret = ipa_api_ctrl->api(p); \
-			} else { \
-				WARN(1, \
-					"%s not implemented for IPA ver %d\n", \
-						__func__, ipa_api_hw_type); \
-				ret = -EPERM; \
-			} \
-		} \
-	} while (0)
-
-#define IPA_API_DISPATCH(api, p...) \
-	do { \
-		if (!ipa_api_ctrl) \
-			pr_err("%s:%d IPA HW is not supported\n", \
-				__func__, __LINE__); \
-		else { \
-			if (ipa_api_ctrl->api) { \
-				ipa_api_ctrl->api(p); \
-			} else { \
-				WARN(1, \
-					"%s not implemented for IPA ver %d\n",\
-						__func__, ipa_api_hw_type); \
-			} \
-		} \
-	} while (0)
-
-#define IPA_API_DISPATCH_RETURN_PTR(api, p...) \
-	do { \
-		if (!ipa_api_ctrl) { \
-			pr_err("%s:%d IPA HW is not supported\n", \
-				__func__, __LINE__); \
-			ret = NULL; \
-		} \
-		else { \
-			if (ipa_api_ctrl->api) { \
-				ret = ipa_api_ctrl->api(p); \
-			} else { \
-				WARN(1, "%s not implemented for IPA ver %d\n",\
-						__func__, ipa_api_hw_type); \
-				ret = NULL; \
-			} \
-		} \
-	} while (0)
-
-#define IPA_API_DISPATCH_RETURN_BOOL(api, p...) \
-	do { \
-		if (!ipa_api_ctrl) { \
-			pr_err("%s:%d IPA HW is not supported\n", \
-				__func__, __LINE__); \
-			ret = false; \
-		} \
-		else { \
-			if (ipa_api_ctrl->api) { \
-				ret = ipa_api_ctrl->api(p); \
-			} else { \
-				WARN(1, "%s not implemented for IPA ver %d\n",\
-						__func__, ipa_api_hw_type); \
-				ret = false; \
-			} \
-		} \
-	} while (0)
-
-#if defined(CONFIG_IPA_EMULATION)
-static bool running_emulation = true;
-#else
-static bool running_emulation;
-#endif
-
-static enum ipa_hw_type ipa_api_hw_type;
-static struct ipa_api_controller *ipa_api_ctrl;
-
-const char *ipa_clients_strings[IPA_CLIENT_MAX] = {
-	__stringify(IPA_CLIENT_HSIC1_PROD),
-	__stringify(IPA_CLIENT_HSIC1_CONS),
-	__stringify(IPA_CLIENT_HSIC2_PROD),
-	__stringify(IPA_CLIENT_HSIC2_CONS),
-	__stringify(IPA_CLIENT_HSIC3_PROD),
-	__stringify(IPA_CLIENT_HSIC3_CONS),
-	__stringify(IPA_CLIENT_HSIC4_PROD),
-	__stringify(IPA_CLIENT_HSIC4_CONS),
-	__stringify(IPA_CLIENT_HSIC5_PROD),
-	__stringify(IPA_CLIENT_HSIC5_CONS),
-	__stringify(IPA_CLIENT_WLAN1_PROD),
-	__stringify(IPA_CLIENT_WLAN1_CONS),
-	__stringify(IPA_CLIENT_WLAN2_PROD),
-	__stringify(IPA_CLIENT_WLAN2_CONS),
-	__stringify(RESERVED_PROD_14),
-	__stringify(IPA_CLIENT_WLAN3_CONS),
-	__stringify(RESERVED_PROD_16),
-	__stringify(IPA_CLIENT_WLAN4_CONS),
-	__stringify(IPA_CLIENT_USB_PROD),
-	__stringify(IPA_CLIENT_USB_CONS),
-	__stringify(IPA_CLIENT_USB2_PROD),
-	__stringify(IPA_CLIENT_USB2_CONS),
-	__stringify(IPA_CLIENT_USB3_PROD),
-	__stringify(IPA_CLIENT_USB3_CONS),
-	__stringify(IPA_CLIENT_USB4_PROD),
-	__stringify(IPA_CLIENT_USB4_CONS),
-	__stringify(IPA_CLIENT_UC_USB_PROD),
-	__stringify(IPA_CLIENT_USB_DPL_CONS),
-	__stringify(IPA_CLIENT_A2_EMBEDDED_PROD),
-	__stringify(IPA_CLIENT_A2_EMBEDDED_CONS),
-	__stringify(IPA_CLIENT_A2_TETHERED_PROD),
-	__stringify(IPA_CLIENT_A2_TETHERED_CONS),
-	__stringify(IPA_CLIENT_APPS_LAN_PROD),
-	__stringify(IPA_CLIENT_APPS_LAN_CONS),
-	__stringify(IPA_CLIENT_APPS_WAN_PROD),
-	__stringify(IPA_CLIENT_APPS_WAN_CONS),
-	__stringify(IPA_CLIENT_APPS_CMD_PROD),
-	__stringify(IPA_CLIENT_A5_LAN_WAN_CONS),
-	__stringify(IPA_CLIENT_ODU_PROD),
-	__stringify(IPA_CLIENT_ODU_EMB_CONS),
-	__stringify(RESERVED_PROD_40),
-	__stringify(IPA_CLIENT_ODU_TETH_CONS),
-	__stringify(IPA_CLIENT_MHI_PROD),
-	__stringify(IPA_CLIENT_MHI_CONS),
-	__stringify(IPA_CLIENT_MEMCPY_DMA_SYNC_PROD),
-	__stringify(IPA_CLIENT_MEMCPY_DMA_SYNC_CONS),
-	__stringify(IPA_CLIENT_MEMCPY_DMA_ASYNC_PROD),
-	__stringify(IPA_CLIENT_MEMCPY_DMA_ASYNC_CONS),
-	__stringify(IPA_CLIENT_ETHERNET_PROD),
-	__stringify(IPA_CLIENT_ETHERNET_CONS),
-	__stringify(IPA_CLIENT_Q6_LAN_PROD),
-	__stringify(IPA_CLIENT_Q6_LAN_CONS),
-	__stringify(IPA_CLIENT_Q6_WAN_PROD),
-	__stringify(IPA_CLIENT_Q6_WAN_CONS),
-	__stringify(IPA_CLIENT_Q6_CMD_PROD),
-	__stringify(IPA_CLIENT_Q6_DUN_CONS),
-	__stringify(IPA_CLIENT_Q6_DECOMP_PROD),
-	__stringify(IPA_CLIENT_Q6_DECOMP_CONS),
-	__stringify(IPA_CLIENT_Q6_DECOMP2_PROD),
-	__stringify(IPA_CLIENT_Q6_DECOMP2_CONS),
-	__stringify(RESERVED_PROD_60),
-	__stringify(IPA_CLIENT_Q6_LTE_WIFI_AGGR_CONS),
-	__stringify(IPA_CLIENT_TEST_PROD),
-	__stringify(IPA_CLIENT_TEST_CONS),
-	__stringify(IPA_CLIENT_TEST1_PROD),
-	__stringify(IPA_CLIENT_TEST1_CONS),
-	__stringify(IPA_CLIENT_TEST2_PROD),
-	__stringify(IPA_CLIENT_TEST2_CONS),
-	__stringify(IPA_CLIENT_TEST3_PROD),
-	__stringify(IPA_CLIENT_TEST3_CONS),
-	__stringify(IPA_CLIENT_TEST4_PROD),
-	__stringify(IPA_CLIENT_TEST4_CONS),
-	__stringify(RESERVED_PROD_72),
-	__stringify(IPA_CLIENT_DUMMY_CONS),
-	__stringify(IPA_CLIENT_Q6_DL_NLO_DATA_PROD),
-	__stringify(IPA_CLIENT_Q6_UL_NLO_DATA_CONS),
-	__stringify(RESERVED_PROD_76),
-	__stringify(IPA_CLIENT_Q6_UL_NLO_ACK_CONS),
-	__stringify(RESERVED_PROD_78),
-	__stringify(IPA_CLIENT_Q6_QBAP_STATUS_CONS),
-	__stringify(RESERVED_PROD_80),
-	__stringify(IPA_CLIENT_MHI_DPL_CONS),
-	__stringify(RESERVED_PROD_82),
-	__stringify(IPA_CLIENT_ODL_DPL_CONS),
-	__stringify(IPA_CLIENT_Q6_AUDIO_DMA_MHI_PROD),
-	__stringify(IPA_CLIENT_Q6_AUDIO_DMA_MHI_CONS),
-	__stringify(IPA_CLIENT_WIGIG_PROD),
-	__stringify(IPA_CLIENT_WIGIG1_CONS),
-	__stringify(RESERVERD_PROD_88),
-	__stringify(IPA_CLIENT_WIGIG2_CONS),
-	__stringify(RESERVERD_PROD_90),
-	__stringify(IPA_CLIENT_WIGIG3_CONS),
-	__stringify(RESERVERD_PROD_92),
-	__stringify(IPA_CLIENT_WIGIG4_CONS),
-	__stringify(RESERVERD_PROD_94),
-	__stringify(IPA_CLIENT_APPS_WAN_COAL_CONS),
-	__stringify(IPA_CLIENT_MHI_PRIME_RMNET_PROD),
-	__stringify(IPA_CLIENT_MHI_PRIME_RMNET_CONS),
-	__stringify(IPA_CLIENT_MHI_PRIME_TETH_PROD),
-	__stringify(IPA_CLIENT_MHI_PRIME_TETH_CONS),
-	__stringify(IPA_CLIENT_MHI_PRIME_DPL_PROD),
-};
-
-/**
- * ipa_write_64() - convert 64 bit value to byte array
- * @w: 64 bit integer
- * @dest: byte array
- *
- * Return value: converted value
- */
-u8 *ipa_write_64(u64 w, u8 *dest)
-{
-	if (unlikely(dest == NULL)) {
-		pr_err("%s: NULL address\n", __func__);
-		return dest;
-	}
-	*dest++ = (u8)((w) & 0xFF);
-	*dest++ = (u8)((w >> 8) & 0xFF);
-	*dest++ = (u8)((w >> 16) & 0xFF);
-	*dest++ = (u8)((w >> 24) & 0xFF);
-	*dest++ = (u8)((w >> 32) & 0xFF);
-	*dest++ = (u8)((w >> 40) & 0xFF);
-	*dest++ = (u8)((w >> 48) & 0xFF);
-	*dest++ = (u8)((w >> 56) & 0xFF);
-
-	return dest;
-}
-
-/**
- * ipa_write_32() - convert 32 bit value to byte array
- * @w: 32 bit integer
- * @dest: byte array
- *
- * Return value: converted value
- */
-u8 *ipa_write_32(u32 w, u8 *dest)
-{
-	if (unlikely(dest == NULL)) {
-		pr_err("%s: NULL address\n", __func__);
-		return dest;
-	}
-	*dest++ = (u8)((w) & 0xFF);
-	*dest++ = (u8)((w >> 8) & 0xFF);
-	*dest++ = (u8)((w >> 16) & 0xFF);
-	*dest++ = (u8)((w >> 24) & 0xFF);
-
-	return dest;
-}
-
-/**
- * ipa_write_16() - convert 16 bit value to byte array
- * @hw: 16 bit integer
- * @dest: byte array
- *
- * Return value: converted value
- */
-u8 *ipa_write_16(u16 hw, u8 *dest)
-{
-	if (unlikely(dest == NULL)) {
-		pr_err("%s: NULL address\n", __func__);
-		return dest;
-	}
-	*dest++ = (u8)((hw) & 0xFF);
-	*dest++ = (u8)((hw >> 8) & 0xFF);
-
-	return dest;
-}
-
-/**
- * ipa_write_8() - convert 8 bit value to byte array
- * @hw: 8 bit integer
- * @dest: byte array
- *
- * Return value: converted value
- */
-u8 *ipa_write_8(u8 b, u8 *dest)
-{
-	if (unlikely(dest == NULL)) {
-		WARN(1, "%s: NULL address\n", __func__);
-		return dest;
-	}
-	*dest++ = (b) & 0xFF;
-
-	return dest;
-}
-
-/**
- * ipa_pad_to_64() - pad byte array to 64 bit value
- * @dest: byte array
- *
- * Return value: padded value
- */
-u8 *ipa_pad_to_64(u8 *dest)
-{
-	int i;
-	int j;
-
-	if (unlikely(dest == NULL)) {
-		WARN(1, "%s: NULL address\n", __func__);
-		return dest;
-	}
-
-	i = (long)dest & 0x7;
-
-	if (i)
-		for (j = 0; j < (8 - i); j++)
-			*dest++ = 0;
-
-	return dest;
-}
-
-/**
- * ipa_pad_to_32() - pad byte array to 32 bit value
- * @dest: byte array
- *
- * Return value: padded value
- */
-u8 *ipa_pad_to_32(u8 *dest)
-{
-	int i;
-	int j;
-
-	if (unlikely(dest == NULL)) {
-		WARN(1, "%s: NULL address\n", __func__);
-		return dest;
-	}
-
-	i = (long)dest & 0x7;
-
-	if (i)
-		for (j = 0; j < (4 - i); j++)
-			*dest++ = 0;
-
-	return dest;
-}
-
-int ipa_smmu_store_sgt(struct sg_table **out_ch_ptr,
-	struct sg_table *in_sgt_ptr)
-{
-	unsigned int nents;
-
-	if (in_sgt_ptr != NULL) {
-		*out_ch_ptr = kzalloc(sizeof(struct sg_table), GFP_KERNEL);
-		if (*out_ch_ptr == NULL)
-			return -ENOMEM;
-
-		nents = in_sgt_ptr->nents;
-
-		(*out_ch_ptr)->sgl =
-			kcalloc(nents, sizeof(struct scatterlist),
-				GFP_KERNEL);
-		if ((*out_ch_ptr)->sgl == NULL) {
-			kfree(*out_ch_ptr);
-			*out_ch_ptr = NULL;
-			return -ENOMEM;
-		}
-
-		memcpy((*out_ch_ptr)->sgl, in_sgt_ptr->sgl,
-			nents*sizeof((*out_ch_ptr)->sgl));
-		(*out_ch_ptr)->nents = nents;
-		(*out_ch_ptr)->orig_nents = in_sgt_ptr->orig_nents;
-	}
-	return 0;
-}
-
-int ipa_smmu_free_sgt(struct sg_table **out_sgt_ptr)
-{
-	if (*out_sgt_ptr != NULL) {
-		kfree((*out_sgt_ptr)->sgl);
-		(*out_sgt_ptr)->sgl = NULL;
-		kfree(*out_sgt_ptr);
-		*out_sgt_ptr = NULL;
-	}
-	return 0;
-}
-
-/**
- * ipa_clear_endpoint_delay() - Clear ep_delay.
- * @clnt_hdl:	[in] IPA client handle
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:		Should not be called from atomic context
- */
-int ipa_clear_endpoint_delay(u32 clnt_hdl)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_clear_endpoint_delay, clnt_hdl);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_clear_endpoint_delay);
-
-/**
- * ipa_reset_endpoint() - reset an endpoint from BAM perspective
- * @clnt_hdl:	[in] IPA client handle
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:		Should not be called from atomic context
- */
-int ipa_reset_endpoint(u32 clnt_hdl)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_reset_endpoint, clnt_hdl);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_reset_endpoint);
-
-/**
- * ipa_disable_endpoint() - Disable an endpoint from IPA perspective
- * @clnt_hdl:	[in] IPA client handle
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:		Should not be called from atomic context
- */
-int ipa_disable_endpoint(u32 clnt_hdl)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_disable_endpoint, clnt_hdl);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_disable_endpoint);
-
-
-/**
- * ipa_cfg_ep - IPA end-point configuration
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- * @ipa_ep_cfg:	[in] IPA end-point configuration params
- *
- * This includes nat, header, mode, aggregation and route settings and is a one
- * shot API to configure the IPA end-point fully
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_cfg_ep(u32 clnt_hdl, const struct ipa_ep_cfg *ipa_ep_cfg)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_cfg_ep, clnt_hdl, ipa_ep_cfg);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_cfg_ep);
-
-/**
- * ipa_cfg_ep_nat() - IPA end-point NAT configuration
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- * @ep_nat:	[in] IPA NAT end-point configuration params
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_cfg_ep_nat(u32 clnt_hdl, const struct ipa_ep_cfg_nat *ep_nat)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_cfg_ep_nat, clnt_hdl, ep_nat);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_cfg_ep_nat);
-
-/**
- * ipa_cfg_ep_conn_track() - IPA end-point IPv6CT configuration
- * @clnt_hdl:		[in] opaque client handle assigned by IPA to client
- * @ep_conn_track:	[in] IPA IPv6CT end-point configuration params
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_cfg_ep_conn_track(u32 clnt_hdl,
-	const struct ipa_ep_cfg_conn_track *ep_conn_track)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_cfg_ep_conn_track, clnt_hdl,
-		ep_conn_track);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_cfg_ep_conn_track);
-
-/**
- * ipa_cfg_ep_hdr() -  IPA end-point header configuration
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- * @ipa_ep_cfg:	[in] IPA end-point configuration params
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_cfg_ep_hdr(u32 clnt_hdl, const struct ipa_ep_cfg_hdr *ep_hdr)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_cfg_ep_hdr, clnt_hdl, ep_hdr);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_cfg_ep_hdr);
-
-/**
- * ipa_cfg_ep_hdr_ext() -  IPA end-point extended header configuration
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- * @ep_hdr_ext:	[in] IPA end-point configuration params
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_cfg_ep_hdr_ext(u32 clnt_hdl,
-		       const struct ipa_ep_cfg_hdr_ext *ep_hdr_ext)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_cfg_ep_hdr_ext, clnt_hdl, ep_hdr_ext);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_cfg_ep_hdr_ext);
-
-/**
- * ipa_cfg_ep_mode() - IPA end-point mode configuration
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- * @ipa_ep_cfg:	[in] IPA end-point configuration params
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_cfg_ep_mode(u32 clnt_hdl, const struct ipa_ep_cfg_mode *ep_mode)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_cfg_ep_mode, clnt_hdl, ep_mode);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_cfg_ep_mode);
-
-/**
- * ipa_cfg_ep_aggr() - IPA end-point aggregation configuration
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- * @ipa_ep_cfg:	[in] IPA end-point configuration params
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_cfg_ep_aggr(u32 clnt_hdl, const struct ipa_ep_cfg_aggr *ep_aggr)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_cfg_ep_aggr, clnt_hdl, ep_aggr);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_cfg_ep_aggr);
-
-/**
- * ipa_cfg_ep_deaggr() -  IPA end-point deaggregation configuration
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- * @ep_deaggr:	[in] IPA end-point configuration params
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_cfg_ep_deaggr(u32 clnt_hdl,
-			const struct ipa_ep_cfg_deaggr *ep_deaggr)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_cfg_ep_deaggr, clnt_hdl, ep_deaggr);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_cfg_ep_deaggr);
-
-/**
- * ipa_cfg_ep_route() - IPA end-point routing configuration
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- * @ipa_ep_cfg:	[in] IPA end-point configuration params
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_cfg_ep_route(u32 clnt_hdl, const struct ipa_ep_cfg_route *ep_route)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_cfg_ep_route, clnt_hdl, ep_route);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_cfg_ep_route);
-
-/**
- * ipa_cfg_ep_holb() - IPA end-point holb configuration
- *
- * If an IPA producer pipe is full, IPA HW by default will block
- * indefinitely till space opens up. During this time no packets
- * including those from unrelated pipes will be processed. Enabling
- * HOLB means IPA HW will be allowed to drop packets as/when needed
- * and indefinite blocking is avoided.
- *
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- * @ipa_ep_cfg:	[in] IPA end-point configuration params
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa_cfg_ep_holb(u32 clnt_hdl, const struct ipa_ep_cfg_holb *ep_holb)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_cfg_ep_holb, clnt_hdl, ep_holb);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_cfg_ep_holb);
-
-
-/**
- * ipa_cfg_ep_cfg() - IPA end-point cfg configuration
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- * @ipa_ep_cfg:	[in] IPA end-point configuration params
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_cfg_ep_cfg(u32 clnt_hdl, const struct ipa_ep_cfg_cfg *cfg)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_cfg_ep_cfg, clnt_hdl, cfg);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_cfg_ep_cfg);
-
-/**
- * ipa_cfg_ep_metadata_mask() - IPA end-point meta-data mask configuration
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- * @ipa_ep_cfg:	[in] IPA end-point configuration params
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_cfg_ep_metadata_mask(u32 clnt_hdl, const struct ipa_ep_cfg_metadata_mask
-		*metadata_mask)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_cfg_ep_metadata_mask, clnt_hdl,
-			metadata_mask);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_cfg_ep_metadata_mask);
-
-/**
- * ipa_cfg_ep_holb_by_client() - IPA end-point holb configuration
- *
- * Wrapper function for ipa_cfg_ep_holb() with client name instead of
- * client handle. This function is used for clients that does not have
- * client handle.
- *
- * @client:	[in] client name
- * @ipa_ep_cfg:	[in] IPA end-point configuration params
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa_cfg_ep_holb_by_client(enum ipa_client_type client,
-				const struct ipa_ep_cfg_holb *ep_holb)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_cfg_ep_holb_by_client, client, ep_holb);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_cfg_ep_holb_by_client);
-
-/**
- * ipa_cfg_ep_ctrl() -  IPA end-point Control configuration
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- * @ipa_ep_cfg_ctrl:	[in] IPA end-point configuration params
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa_cfg_ep_ctrl(u32 clnt_hdl, const struct ipa_ep_cfg_ctrl *ep_ctrl)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_cfg_ep_ctrl, clnt_hdl, ep_ctrl);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_cfg_ep_ctrl);
-
-/**
- * ipa_add_hdr() - add the specified headers to SW and optionally commit them to
- * IPA HW
- * @hdrs:	[inout] set of headers to add
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_add_hdr(struct ipa_ioc_add_hdr *hdrs)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_add_hdr, hdrs);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_add_hdr);
-
-/**
- * ipa_add_hdr_usr() - add the specified headers to SW and optionally
- * commit them to IPA HW
- * @hdrs:		[inout] set of headers to add
- * @user_only:	[in] indicate rules installed by userspace
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_add_hdr_usr(struct ipa_ioc_add_hdr *hdrs, bool user_only)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_add_hdr_usr, hdrs, user_only);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_add_hdr_usr);
-
-/**
- * ipa_del_hdr() - Remove the specified headers from SW and optionally
- * commit them to IPA HW
- * @hdls:	[inout] set of headers to delete
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_del_hdr(struct ipa_ioc_del_hdr *hdls)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_del_hdr, hdls);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_del_hdr);
-
-/**
- * ipa_commit_hdr() - commit to IPA HW the current header table in SW
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_commit_hdr(void)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_commit_hdr);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_commit_hdr);
-
-/**
- * ipa_reset_hdr() - reset the current header table in SW (does not commit to
- * HW)
- *
- * @user_only:	[in] indicate delete rules installed by userspace
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_reset_hdr(bool user_only)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_reset_hdr, user_only);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_reset_hdr);
-
-/**
- * ipa_get_hdr() - Lookup the specified header resource
- * @lookup:	[inout] header to lookup and its handle
- *
- * lookup the specified header resource and return handle if it exists
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- *		Caller should call ipa_put_hdr later if this function succeeds
- */
-int ipa_get_hdr(struct ipa_ioc_get_hdr *lookup)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_get_hdr, lookup);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_get_hdr);
-
-/**
- * ipa_put_hdr() - Release the specified header handle
- * @hdr_hdl:	[in] the header handle to release
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_put_hdr(u32 hdr_hdl)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_put_hdr, hdr_hdl);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_put_hdr);
-
-/**
- * ipa_copy_hdr() - Lookup the specified header resource and return a copy of it
- * @copy:	[inout] header to lookup and its copy
- *
- * lookup the specified header resource and return a copy of it (along with its
- * attributes) if it exists, this would be called for partial headers
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_copy_hdr(struct ipa_ioc_copy_hdr *copy)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_copy_hdr, copy);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_copy_hdr);
-
-/**
- * ipa_add_hdr_proc_ctx() - add the specified headers to SW
- * and optionally commit them to IPA HW
- * @proc_ctxs:	[inout] set of processing context headers to add
- * @user_only:	[in] indicate rules installed by userspace
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_add_hdr_proc_ctx(struct ipa_ioc_add_hdr_proc_ctx *proc_ctxs,
-							bool user_only)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_add_hdr_proc_ctx, proc_ctxs, user_only);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_add_hdr_proc_ctx);
-
-/**
- * ipa_del_hdr_proc_ctx() -
- * Remove the specified processing context headers from SW and
- * optionally commit them to IPA HW.
- * @hdls:	[inout] set of processing context headers to delete
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_del_hdr_proc_ctx(struct ipa_ioc_del_hdr_proc_ctx *hdls)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_del_hdr_proc_ctx, hdls);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_del_hdr_proc_ctx);
-
-/**
- * ipa_add_rt_rule() - Add the specified routing rules to SW and optionally
- * commit to IPA HW
- * @rules:	[inout] set of routing rules to add
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_add_rt_rule(struct ipa_ioc_add_rt_rule *rules)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_add_rt_rule, rules);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_add_rt_rule);
-
-/**
- * ipa_add_rt_rule_v2() - Add the specified routing rules to SW
- * and optionally commit to IPA HW
- * @rules:	[inout] set of routing rules to add
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_add_rt_rule_v2(struct ipa_ioc_add_rt_rule_v2 *rules)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_add_rt_rule_v2, rules);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_add_rt_rule_v2);
-
-/**
- * ipa_add_rt_rule_usr() - Add the specified routing rules to SW and optionally
- * commit to IPA HW
- * @rules:	[inout] set of routing rules to add
- * @user_only:	[in] indicate rules installed by userspace
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_add_rt_rule_usr(struct ipa_ioc_add_rt_rule *rules, bool user_only)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_add_rt_rule_usr, rules, user_only);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_add_rt_rule_usr);
-
-/**
- * ipa_add_rt_rule_usr_v2() - Add the specified routing rules to
- * SW and optionally commit to IPA HW
- * @rules:	[inout] set of routing rules to add
- * @user_only:	[in] indicate rules installed by userspace
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_add_rt_rule_usr_v2(struct ipa_ioc_add_rt_rule_v2 *rules, bool user_only)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_add_rt_rule_usr_v2, rules, user_only);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_add_rt_rule_usr_v2);
-
-/**
- * ipa_del_rt_rule() - Remove the specified routing rules to SW and optionally
- * commit to IPA HW
- * @hdls:	[inout] set of routing rules to delete
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_del_rt_rule(struct ipa_ioc_del_rt_rule *hdls)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_del_rt_rule, hdls);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_del_rt_rule);
-
-/**
- * ipa_commit_rt_rule() - Commit the current SW routing table of specified type
- * to IPA HW
- * @ip:	The family of routing tables
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_commit_rt(enum ipa_ip_type ip)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_commit_rt, ip);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_commit_rt);
-
-/**
- * ipa_reset_rt() - reset the current SW routing table of specified type
- * (does not commit to HW)
- * @ip:	The family of routing tables
- * @user_only:	[in] indicate delete rules installed by userspace
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_reset_rt(enum ipa_ip_type ip, bool user_only)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_reset_rt, ip, user_only);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_reset_rt);
-
-/**
- * ipa_get_rt_tbl() - lookup the specified routing table and return handle if it
- * exists, if lookup succeeds the routing table ref cnt is increased
- * @lookup:	[inout] routing table to lookup and its handle
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- *	Caller should call ipa_put_rt_tbl later if this function succeeds
- */
-int ipa_get_rt_tbl(struct ipa_ioc_get_rt_tbl *lookup)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_get_rt_tbl, lookup);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_get_rt_tbl);
-
-/**
- * ipa_put_rt_tbl() - Release the specified routing table handle
- * @rt_tbl_hdl:	[in] the routing table handle to release
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_put_rt_tbl(u32 rt_tbl_hdl)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_put_rt_tbl, rt_tbl_hdl);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_put_rt_tbl);
-
-/**
- * ipa_query_rt_index() - find the routing table index
- *			which name and ip type are given as parameters
- * @in:	[out] the index of the wanted routing table
- *
- * Returns: the routing table which name is given as parameter, or NULL if it
- * doesn't exist
- */
-int ipa_query_rt_index(struct ipa_ioc_get_rt_tbl_indx *in)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_query_rt_index, in);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_query_rt_index);
-
-/**
- * ipa_mdfy_rt_rule() - Modify the specified routing rules in SW and optionally
- * commit to IPA HW
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_mdfy_rt_rule(struct ipa_ioc_mdfy_rt_rule *hdls)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_mdfy_rt_rule, hdls);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_mdfy_rt_rule);
-
-/**
- * ipa_mdfy_rt_rule_v2() - Modify the specified routing rules in
- * SW and optionally commit to IPA HW
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_mdfy_rt_rule_v2(struct ipa_ioc_mdfy_rt_rule_v2 *hdls)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_mdfy_rt_rule_v2, hdls);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_mdfy_rt_rule_v2);
-
-/**
- * ipa_add_flt_rule() - Add the specified filtering rules to SW and optionally
- * commit to IPA HW
- * @rules:	[inout] set of filtering rules to add
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_add_flt_rule(struct ipa_ioc_add_flt_rule *rules)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_add_flt_rule, rules);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_add_flt_rule);
-
-/**
- * ipa_add_flt_rule_v2() - Add the specified filtering rules to
- * SW and optionally commit to IPA HW
- * @rules:	[inout] set of filtering rules to add
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_add_flt_rule_v2(struct ipa_ioc_add_flt_rule_v2 *rules)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_add_flt_rule_v2, rules);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_add_flt_rule_v2);
-
-/**
- * ipa_add_flt_rule_usr() - Add the specified filtering rules to
- * SW and optionally commit to IPA HW
- * @rules:		[inout] set of filtering rules to add
- * @user_only:	[in] indicate rules installed by userspace
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_add_flt_rule_usr(struct ipa_ioc_add_flt_rule *rules, bool user_only)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_add_flt_rule_usr, rules, user_only);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_add_flt_rule_usr);
-
-/**
- * ipa_add_flt_rule_usr_v2() - Add the specified filtering rules
- * to SW and optionally commit to IPA HW
- * @rules:		[inout] set of filtering rules to add
- * @user_only:	[in] indicate rules installed by userspace
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_add_flt_rule_usr_v2(struct ipa_ioc_add_flt_rule_v2 *rules,
-	bool user_only)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_add_flt_rule_usr_v2,
-		rules, user_only);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_add_flt_rule_usr_v2);
-
-/**
- * ipa_del_flt_rule() - Remove the specified filtering rules from SW and
- * optionally commit to IPA HW
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_del_flt_rule(struct ipa_ioc_del_flt_rule *hdls)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_del_flt_rule, hdls);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_del_flt_rule);
-
-/**
- * ipa_mdfy_flt_rule() - Modify the specified filtering rules in SW and
- * optionally commit to IPA HW
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_mdfy_flt_rule(struct ipa_ioc_mdfy_flt_rule *hdls)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_mdfy_flt_rule, hdls);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_mdfy_flt_rule);
-
-/**
- * ipa_mdfy_flt_rule_v2() - Modify the specified filtering rules
- * in SW and optionally commit to IPA HW
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_mdfy_flt_rule_v2(struct ipa_ioc_mdfy_flt_rule_v2 *hdls)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_mdfy_flt_rule_v2, hdls);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_mdfy_flt_rule_v2);
-
-/**
- * ipa_commit_flt() - Commit the current SW filtering table of specified type to
- * IPA HW
- * @ip:	[in] the family of routing tables
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_commit_flt(enum ipa_ip_type ip)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_commit_flt, ip);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_commit_flt);
-
-/**
- * ipa_reset_flt() - Reset the current SW filtering table of specified type
- * (does not commit to HW)
- * @ip:			[in] the family of routing tables
- * @user_only:	[in] indicate delete rules installed by userspace
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_reset_flt(enum ipa_ip_type ip, bool user_only)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_reset_flt, ip, user_only);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_reset_flt);
-
-/**
- * ipa_allocate_nat_device() - Allocates memory for the NAT device
- * @mem:	[in/out] memory parameters
- *
- * Called by NAT client driver to allocate memory for the NAT entries. Based on
- * the request size either shared or system memory will be used.
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa_allocate_nat_device(struct ipa_ioc_nat_alloc_mem *mem)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_allocate_nat_device, mem);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_allocate_nat_device);
-
-/**
- * ipa_allocate_nat_table() - Allocates memory for the NAT table
- * @table_alloc: [in/out] memory parameters
- *
- * Called by NAT client to allocate memory for the table entries.
- * Based on the request size either shared or system memory will be used.
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa_allocate_nat_table(struct ipa_ioc_nat_ipv6ct_table_alloc *table_alloc)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_allocate_nat_table, table_alloc);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_allocate_nat_table);
-
-
-/**
- * ipa_allocate_ipv6ct_table() - Allocates memory for the IPv6CT table
- * @table_alloc: [in/out] memory parameters
- *
- * Called by IPv6CT client to allocate memory for the table entries.
- * Based on the request size either shared or system memory will be used.
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa_allocate_ipv6ct_table(
-	struct ipa_ioc_nat_ipv6ct_table_alloc *table_alloc)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_allocate_ipv6ct_table, table_alloc);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_allocate_ipv6ct_table);
-
-/**
- * ipa_nat_init_cmd() - Post IP_V4_NAT_INIT command to IPA HW
- * @init:	[in] initialization command attributes
- *
- * Called by NAT client driver to post IP_V4_NAT_INIT command to IPA HW
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa_nat_init_cmd(struct ipa_ioc_v4_nat_init *init)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_nat_init_cmd, init);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_nat_init_cmd);
-
-/**
- * ipa_ipv6ct_init_cmd() - Post IP_V6_CONN_TRACK_INIT command to IPA HW
- * @init:	[in] initialization command attributes
- *
- * Called by IPv6CT client driver to post IP_V6_CONN_TRACK_INIT command
- * to IPA HW.
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa_ipv6ct_init_cmd(struct ipa_ioc_ipv6ct_init *init)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_ipv6ct_init_cmd, init);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_ipv6ct_init_cmd);
-
-/**
- * ipa_nat_dma_cmd() - Post NAT_DMA command to IPA HW
- * @dma:	[in] initialization command attributes
- *
- * Called by NAT client driver to post NAT_DMA command to IPA HW
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa_nat_dma_cmd(struct ipa_ioc_nat_dma_cmd *dma)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_nat_dma_cmd, dma);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_nat_dma_cmd);
-
-/**
- * ipa_table_dma_cmd() - Post TABLE_DMA command to IPA HW
- * @dma:	[in] initialization command attributes
- *
- * Called by NAT/IPv6CT client to post TABLE_DMA command to IPA HW
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa_table_dma_cmd(struct ipa_ioc_nat_dma_cmd *dma)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_table_dma_cmd, dma);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_table_dma_cmd);
-
-/**
- * ipa_nat_del_cmd() - Delete the NAT table
- * @del:	[in] delete NAT table parameters
- *
- * Called by NAT client driver to delete the nat table
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa_nat_del_cmd(struct ipa_ioc_v4_nat_del *del)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_nat_del_cmd, del);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_nat_del_cmd);
-
-/**
- * ipa_del_nat_table() - Delete the NAT table
- * @del:	[in] delete table parameters
- *
- * Called by NAT client to delete the table
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa_del_nat_table(struct ipa_ioc_nat_ipv6ct_table_del *del)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_del_nat_table, del);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_del_nat_table);
-
-/**
- * ipa_del_ipv6ct_table() - Delete the IPv6CT table
- * @del:	[in] delete table parameters
- *
- * Called by IPv6CT client to delete the table
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa_del_ipv6ct_table(struct ipa_ioc_nat_ipv6ct_table_del *del)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_del_ipv6ct_table, del);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_del_ipv6ct_table);
-
-/**
- * ipa3_nat_mdfy_pdn() - Modify a PDN entry in PDN config table in IPA SRAM
- * @mdfy_pdn:	[in] PDN info to be written to SRAM
- *
- * Called by NAT client driver to modify an entry in the PDN config table
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa_nat_mdfy_pdn(struct ipa_ioc_nat_pdn_entry *mdfy_pdn)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_nat_mdfy_pdn, mdfy_pdn);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_nat_mdfy_pdn);
-
-/**
- * ipa_send_msg() - Send "message" from kernel client to IPA driver
- * @meta: [in] message meta-data
- * @buff: [in] the payload for message
- * @callback: [in] free callback
- *
- * Client supplies the message meta-data and payload which IPA driver buffers
- * till read by user-space. After read from user space IPA driver invokes the
- * callback supplied to free the message payload. Client must not touch/free
- * the message payload after calling this API.
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_send_msg(struct ipa_msg_meta *meta, void *buff,
-		  ipa_msg_free_fn callback)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_send_msg, meta, buff, callback);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_send_msg);
-
-/**
- * ipa_register_pull_msg() - register pull message type
- * @meta: [in] message meta-data
- * @callback: [in] pull callback
- *
- * Register message callback by kernel client with IPA driver for IPA driver to
- * pull message on-demand.
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_register_pull_msg(struct ipa_msg_meta *meta, ipa_msg_pull_fn callback)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_register_pull_msg, meta, callback);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_register_pull_msg);
-
-/**
- * ipa_deregister_pull_msg() - De-register pull message type
- * @meta: [in] message meta-data
- *
- * De-register "message" by kernel client from IPA driver
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_deregister_pull_msg(struct ipa_msg_meta *meta)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_deregister_pull_msg, meta);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_deregister_pull_msg);
-
-/**
- * ipa_register_intf() - register "logical" interface
- * @name: [in] interface name
- * @tx:	[in] TX properties of the interface
- * @rx:	[in] RX properties of the interface
- *
- * Register an interface and its tx and rx properties, this allows
- * configuration of rules from user-space
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_register_intf(const char *name, const struct ipa_tx_intf *tx,
-		       const struct ipa_rx_intf *rx)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_register_intf, name, tx, rx);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_register_intf);
-
-/**
- * ipa_register_intf_ext() - register "logical" interface which has only
- * extended properties
- * @name: [in] interface name
- * @tx:	[in] TX properties of the interface
- * @rx:	[in] RX properties of the interface
- * @ext: [in] EXT properties of the interface
- *
- * Register an interface and its tx, rx and ext properties, this allows
- * configuration of rules from user-space
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_register_intf_ext(const char *name, const struct ipa_tx_intf *tx,
-	const struct ipa_rx_intf *rx,
-	const struct ipa_ext_intf *ext)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_register_intf_ext, name, tx, rx, ext);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_register_intf_ext);
-
-/**
- * ipa_deregister_intf() - de-register previously registered logical interface
- * @name: [in] interface name
- *
- * De-register a previously registered interface
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_deregister_intf(const char *name)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_deregister_intf, name);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_deregister_intf);
-
-/**
- * ipa_set_aggr_mode() - Set the aggregation mode which is a global setting
- * @mode:	[in] the desired aggregation mode for e.g. straight MBIM, QCNCM,
- * etc
- *
- * Returns:	0 on success
- */
-int ipa_set_aggr_mode(enum ipa_aggr_mode mode)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_set_aggr_mode, mode);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_set_aggr_mode);
-
-
-/**
- * ipa_set_qcncm_ndp_sig() - Set the NDP signature used for QCNCM aggregation
- * mode
- * @sig:	[in] the first 3 bytes of QCNCM NDP signature (expected to be
- * "QND")
- *
- * Set the NDP signature used for QCNCM aggregation mode. The fourth byte
- * (expected to be 'P') needs to be set using the header addition mechanism
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa_set_qcncm_ndp_sig(char sig[3])
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_set_qcncm_ndp_sig, sig);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_set_qcncm_ndp_sig);
-
-/**
- * ipa_set_single_ndp_per_mbim() - Enable/disable single NDP per MBIM frame
- * configuration
- * @enable:	[in] true for single NDP/MBIM; false otherwise
- *
- * Returns:	0 on success
- */
-int ipa_set_single_ndp_per_mbim(bool enable)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_set_single_ndp_per_mbim, enable);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_set_single_ndp_per_mbim);
-
-/**
- * ipa_tx_dp() - Data-path tx handler
- * @dst:	[in] which IPA destination to route tx packets to
- * @skb:	[in] the packet to send
- * @metadata:	[in] TX packet meta-data
- *
- * Data-path tx handler, this is used for both SW data-path which by-passes most
- * IPA HW blocks AND the regular HW data-path for WLAN AMPDU traffic only. If
- * dst is a "valid" CONS type, then SW data-path is used. If dst is the
- * WLAN_AMPDU PROD type, then HW data-path for WLAN AMPDU is used. Anything else
- * is an error. For errors, client needs to free the skb as needed. For success,
- * IPA driver will later invoke client callback if one was supplied. That
- * callback should free the skb. If no callback supplied, IPA driver will free
- * the skb internally
- *
- * The function will use two descriptors for this send command
- * (for A5_WLAN_AMPDU_PROD only one desciprtor will be sent),
- * the first descriptor will be used to inform the IPA hardware that
- * apps need to push data into the IPA (IP_PACKET_INIT immediate command).
- * Once this send was done from SPS point-of-view the IPA driver will
- * get notified by the supplied callback - ipa_sps_irq_tx_comp()
- *
- * ipa_sps_irq_tx_comp will call to the user supplied
- * callback (from ipa_connect)
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa_tx_dp(enum ipa_client_type dst, struct sk_buff *skb,
-		struct ipa_tx_meta *meta)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_tx_dp, dst, skb, meta);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_tx_dp);
-
-/**
- * ipa_tx_dp_mul() - Data-path tx handler for multiple packets
- * @src: [in] - Client that is sending data
- * @ipa_tx_data_desc:	[in] data descriptors from wlan
- *
- * this is used for to transfer data descriptors that received
- * from WLAN1_PROD pipe to IPA HW
- *
- * The function will send data descriptors from WLAN1_PROD (one
- * at a time) using sps_transfer_one. Will set EOT flag for last
- * descriptor Once this send was done from SPS point-of-view the
- * IPA driver will get notified by the supplied callback -
- * ipa_sps_irq_tx_no_aggr_notify()
- *
- * ipa_sps_irq_tx_no_aggr_notify will call to the user supplied
- * callback (from ipa_connect)
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa_tx_dp_mul(enum ipa_client_type src,
-			struct ipa_tx_data_desc *data_desc)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_tx_dp_mul, src, data_desc);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_tx_dp_mul);
-
-void ipa_free_skb(struct ipa_rx_data *data)
-{
-	IPA_API_DISPATCH(ipa_free_skb, data);
-}
-EXPORT_SYMBOL(ipa_free_skb);
-
-/**
- * ipa_setup_sys_pipe() - Setup an IPA end-point in system-BAM mode and perform
- * IPA EP configuration
- * @sys_in:	[in] input needed to setup BAM pipe and configure EP
- * @clnt_hdl:	[out] client handle
- *
- *  - configure the end-point registers with the supplied
- *    parameters from the user.
- *  - call SPS APIs to create a system-to-bam connection with IPA.
- *  - allocate descriptor FIFO
- *  - register callback function(ipa_sps_irq_rx_notify or
- *    ipa_sps_irq_tx_notify - depends on client type) in case the driver is
- *    not configured to pulling mode
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_setup_sys_pipe, sys_in, clnt_hdl);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_setup_sys_pipe);
-
-/**
- * ipa_teardown_sys_pipe() - Teardown the system-BAM pipe and cleanup IPA EP
- * @clnt_hdl:	[in] the handle obtained from ipa_setup_sys_pipe
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa_teardown_sys_pipe(u32 clnt_hdl)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_teardown_sys_pipe, clnt_hdl);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_teardown_sys_pipe);
-
-int ipa_sys_setup(struct ipa_sys_connect_params *sys_in,
-	unsigned long *ipa_bam_or_gsi_hdl,
-	u32 *ipa_pipe_num, u32 *clnt_hdl, bool en_status)
-
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_sys_setup, sys_in, ipa_bam_or_gsi_hdl,
-			ipa_pipe_num, clnt_hdl, en_status);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_sys_setup);
-
-int ipa_sys_teardown(u32 clnt_hdl)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_sys_teardown, clnt_hdl);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_sys_teardown);
-
-int ipa_sys_update_gsi_hdls(u32 clnt_hdl, unsigned long gsi_ch_hdl,
-	unsigned long gsi_ev_hdl)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_sys_update_gsi_hdls, clnt_hdl,
-		gsi_ch_hdl, gsi_ev_hdl);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_sys_update_gsi_hdls);
-
-/**
- * ipa_connect_wdi_pipe() - WDI client connect
- * @in:	[in] input parameters from client
- * @out: [out] output params to client
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_connect_wdi_pipe(struct ipa_wdi_in_params *in,
-		struct ipa_wdi_out_params *out)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_connect_wdi_pipe, in, out);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_connect_wdi_pipe);
-
-/**
- * ipa_disconnect_wdi_pipe() - WDI client disconnect
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_disconnect_wdi_pipe(u32 clnt_hdl)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_disconnect_wdi_pipe, clnt_hdl);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_disconnect_wdi_pipe);
-
-/**
- * ipa_enable_wdi_pipe() - WDI client enable
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_enable_wdi_pipe(u32 clnt_hdl)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_enable_wdi_pipe, clnt_hdl);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_enable_wdi_pipe);
-
-/**
- * ipa_disable_wdi_pipe() - WDI client disable
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_disable_wdi_pipe(u32 clnt_hdl)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_disable_wdi_pipe, clnt_hdl);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_disable_wdi_pipe);
-
-/**
- * ipa_resume_wdi_pipe() - WDI client resume
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_resume_wdi_pipe(u32 clnt_hdl)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_resume_wdi_pipe, clnt_hdl);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_resume_wdi_pipe);
-
-/**
- * ipa_suspend_wdi_pipe() - WDI client suspend
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa_suspend_wdi_pipe(u32 clnt_hdl)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_suspend_wdi_pipe, clnt_hdl);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_suspend_wdi_pipe);
-
-/**
- * ipa_get_wdi_stats() - Query WDI statistics from uc
- * @stats:	[inout] stats blob from client populated by driver
- *
- * Returns:	0 on success, negative on failure
- *
- * @note Cannot be called from atomic context
- *
- */
-int ipa_get_wdi_stats(struct IpaHwStatsWDIInfoData_t *stats)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_get_wdi_stats, stats);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_get_wdi_stats);
-
-/**
- * ipa_get_smem_restr_bytes()- Return IPA smem restricted bytes
- *
- * Return value: u16 - number of IPA smem restricted bytes
- */
-u16 ipa_get_smem_restr_bytes(void)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_get_smem_restr_bytes);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_get_smem_restr_bytes);
-
-/**
- * ipa_broadcast_wdi_quota_reach_ind() - quota reach
- * @uint32_t fid: [in] input netdev ID
- * @uint64_t num_bytes: [in] used bytes
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa_broadcast_wdi_quota_reach_ind(uint32_t fid,
-		uint64_t num_bytes)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_broadcast_wdi_quota_reach_ind,
-		fid, num_bytes);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_broadcast_wdi_quota_reach_ind);
-
-/**
- * ipa_uc_wdi_get_dbpa() - To retrieve
- * doorbell physical address of wlan pipes
- * @param:  [in/out] input/output parameters
- *          from/to client
- *
- * Returns:	0 on success, negative on failure
- *
- */
-int ipa_uc_wdi_get_dbpa(
-	struct ipa_wdi_db_params *param)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_uc_wdi_get_dbpa, param);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_uc_wdi_get_dbpa);
-
-/**
- * ipa_uc_reg_rdyCB() - To register uC
- * ready CB if uC not ready, wdi only.
- * @inout:	[in/out] input/output parameters
- * from/to client
- *
- * Returns:	0 on success, negative on failure
- *
- */
-int ipa_uc_reg_rdyCB(
-	struct ipa_wdi_uc_ready_params *inout)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_uc_reg_rdyCB, inout);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_uc_reg_rdyCB);
-
-/**
-* ipa_wigig_internal_init() - get uc db and register uC
-* ready CB if uC not ready, wigig only.
-* @inout:	[in/out] uc ready input/output parameters
-* from/to client
-* @int_notify: [in] wigig misc interrupt handler function
-* @uc_db_pa: [out] uC db physical address
-*
-* Returns:	0 on success, negative on failure
-*
-*/
-int ipa_wigig_internal_init(
-	struct ipa_wdi_uc_ready_params *inout,
-	ipa_wigig_misc_int_cb int_notify,
-	phys_addr_t *uc_db_pa)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_wigig_internal_init, inout,
-		int_notify, uc_db_pa);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_wigig_internal_init);
-
-/**
- * ipa_uc_dereg_rdyCB() - To de-register uC ready CB
- *
- * Returns:	0 on success, negative on failure
- *
- */
-int ipa_uc_dereg_rdyCB(void)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_uc_dereg_rdyCB);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_uc_dereg_rdyCB);
-
-/**
- * teth_bridge_init() - Initialize the Tethering bridge driver
- * @params - in/out params for USB initialization API (please look at struct
- *  definition for more info)
- *
- * USB driver gets a pointer to a callback function (usb_notify_cb) and an
- * associated data. USB driver installs this callback function in the call to
- * ipa_connect().
- *
- * Builds IPA resource manager dependency graph.
- *
- * Return codes: 0: success,
- *		-EINVAL - Bad parameter
- *		Other negative value - Failure
- */
-int teth_bridge_init(struct teth_bridge_init_params *params)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(teth_bridge_init, params);
-
-	return ret;
-}
-EXPORT_SYMBOL(teth_bridge_init);
-
-/**
- * teth_bridge_disconnect() - Disconnect tethering bridge module
- */
-int teth_bridge_disconnect(enum ipa_client_type client)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(teth_bridge_disconnect, client);
-
-	return ret;
-}
-EXPORT_SYMBOL(teth_bridge_disconnect);
-
-/**
- * teth_bridge_connect() - Connect bridge for a tethered Rmnet / MBIM call
- * @connect_params:	Connection info
- *
- * Return codes: 0: success
- *		-EINVAL: invalid parameters
- *		-EPERM: Operation not permitted as the bridge is already
- *		connected
- */
-int teth_bridge_connect(struct teth_bridge_connect_params *connect_params)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(teth_bridge_connect, connect_params);
-
-	return ret;
-}
-EXPORT_SYMBOL(teth_bridge_connect);
-
-/* ipa_set_client() - provide client mapping
- * @client: client type
- *
- * Return value: none
- */
-
-void ipa_set_client(int index, enum ipacm_client_enum client, bool uplink)
-{
-	IPA_API_DISPATCH(ipa_set_client, index, client, uplink);
-}
-EXPORT_SYMBOL(ipa_set_client);
-
-/**
- * ipa_get_client() - provide client mapping
- * @client: client type
- *
- * Return value: none
- */
-enum ipacm_client_enum ipa_get_client(int pipe_idx)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_get_client, pipe_idx);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_get_client);
-
-/**
- * ipa_get_client_uplink() - provide client mapping
- * @client: client type
- *
- * Return value: none
- */
-bool ipa_get_client_uplink(int pipe_idx)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_get_client_uplink, pipe_idx);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_get_client_uplink);
-
-/**
- * ipa_dma_init() -Initialize IPADMA.
- *
- * This function initialize all IPADMA internal data and connect in dma:
- *	MEMCPY_DMA_SYNC_PROD ->MEMCPY_DMA_SYNC_CONS
- *	MEMCPY_DMA_ASYNC_PROD->MEMCPY_DMA_SYNC_CONS
- *
- * Return codes: 0: success
- *		-EFAULT: IPADMA is already initialized
- *		-ENOMEM: allocating memory error
- *		-EPERM: pipe connection failed
- */
-int ipa_dma_init(void)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_dma_init);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_dma_init);
-
-/**
- * ipa_dma_enable() -Vote for IPA clocks.
- *
- *Return codes: 0: success
- *		-EINVAL: IPADMA is not initialized
- *		-EPERM: Operation not permitted as ipa_dma is already
- *		 enabled
- */
-int ipa_dma_enable(void)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_dma_enable);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_dma_enable);
-
-/**
- * ipa_dma_disable()- Unvote for IPA clocks.
- *
- * enter to power save mode.
- *
- * Return codes: 0: success
- *		-EINVAL: IPADMA is not initialized
- *		-EPERM: Operation not permitted as ipa_dma is already
- *			diabled
- *		-EFAULT: can not disable ipa_dma as there are pending
- *			memcopy works
- */
-int ipa_dma_disable(void)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_dma_disable);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_dma_disable);
-
-/**
- * ipa_dma_sync_memcpy()- Perform synchronous memcpy using IPA.
- *
- * @dest: physical address to store the copied data.
- * @src: physical address of the source data to copy.
- * @len: number of bytes to copy.
- *
- * Return codes: 0: success
- *		-EINVAL: invalid params
- *		-EPERM: operation not permitted as ipa_dma isn't enable or
- *			initialized
- *		-SPS_ERROR: on sps faliures
- *		-EFAULT: other
- */
-int ipa_dma_sync_memcpy(u64 dest, u64 src, int len)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_dma_sync_memcpy, dest, src, len);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_dma_sync_memcpy);
-
-/**
- * ipa_dma_async_memcpy()- Perform asynchronous memcpy using IPA.
- *
- * @dest: physical address to store the copied data.
- * @src: physical address of the source data to copy.
- * @len: number of bytes to copy.
- * @user_cb: callback function to notify the client when the copy was done.
- * @user_param: cookie for user_cb.
- *
- * Return codes: 0: success
- *		-EINVAL: invalid params
- *		-EPERM: operation not permitted as ipa_dma isn't enable or
- *			initialized
- *		-SPS_ERROR: on sps faliures
- *		-EFAULT: descr fifo is full.
- */
-int ipa_dma_async_memcpy(u64 dest, u64 src, int len,
-		void (*user_cb)(void *user1), void *user_param)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_dma_async_memcpy, dest, src, len, user_cb,
-		user_param);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_dma_async_memcpy);
-
-/**
- * ipa_dma_uc_memcpy() - Perform a memcpy action using IPA uC
- * @dest: physical address to store the copied data.
- * @src: physical address of the source data to copy.
- * @len: number of bytes to copy.
- *
- * Return codes: 0: success
- *		-EINVAL: invalid params
- *		-EPERM: operation not permitted as ipa_dma isn't enable or
- *			initialized
- *		-EBADF: IPA uC is not loaded
- */
-int ipa_dma_uc_memcpy(phys_addr_t dest, phys_addr_t src, int len)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_dma_uc_memcpy, dest, src, len);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_dma_uc_memcpy);
-
-/**
- * ipa_dma_destroy() -teardown IPADMA pipes and release ipadma.
- *
- * this is a blocking function, returns just after destroying IPADMA.
- */
-void ipa_dma_destroy(void)
-{
-	IPA_API_DISPATCH(ipa_dma_destroy);
-}
-EXPORT_SYMBOL(ipa_dma_destroy);
-
-int ipa_mhi_init_engine(struct ipa_mhi_init_engine *params)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_mhi_init_engine, params);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_mhi_init_engine);
-
-/**
- * ipa_connect_mhi_pipe() - Connect pipe to IPA and start corresponding
- * MHI channel
- * @in: connect parameters
- * @clnt_hdl: [out] client handle for this pipe
- *
- * This function is called by IPA MHI client driver on MHI channel start.
- * This function is called after MHI engine was started.
- * This function is doing the following:
- *	- Send command to uC to start corresponding MHI channel
- *	- Configure IPA EP control
- *
- * Return codes: 0	  : success
- *		 negative : error
- */
-int ipa_connect_mhi_pipe(struct ipa_mhi_connect_params_internal *in,
-		u32 *clnt_hdl)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_connect_mhi_pipe, in, clnt_hdl);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_connect_mhi_pipe);
-
-/**
- * ipa_disconnect_mhi_pipe() - Disconnect pipe from IPA and reset corresponding
- * MHI channel
- * @in: connect parameters
- * @clnt_hdl: [out] client handle for this pipe
- *
- * This function is called by IPA MHI client driver on MHI channel reset.
- * This function is called after MHI channel was started.
- * This function is doing the following:
- *	- Send command to uC to reset corresponding MHI channel
- *	- Configure IPA EP control
- *
- * Return codes: 0	  : success
- *		 negative : error
- */
-int ipa_disconnect_mhi_pipe(u32 clnt_hdl)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_disconnect_mhi_pipe, clnt_hdl);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_disconnect_mhi_pipe);
-
-bool ipa_mhi_stop_gsi_channel(enum ipa_client_type client)
-{
-	bool ret;
-
-	IPA_API_DISPATCH_RETURN_BOOL(ipa_mhi_stop_gsi_channel, client);
-
-	return ret;
-}
-
-int ipa_uc_mhi_reset_channel(int channelHandle)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_uc_mhi_reset_channel, channelHandle);
-
-	return ret;
-}
-
-bool ipa_mhi_sps_channel_empty(enum ipa_client_type client)
-{
-	bool ret;
-
-	IPA_API_DISPATCH_RETURN_BOOL(ipa_mhi_sps_channel_empty, client);
-
-	return ret;
-}
-
-int ipa_qmi_enable_force_clear_datapath_send(
-	struct ipa_enable_force_clear_datapath_req_msg_v01 *req)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_qmi_enable_force_clear_datapath_send, req);
-
-	return ret;
-}
-
-int ipa_qmi_disable_force_clear_datapath_send(
-	struct ipa_disable_force_clear_datapath_req_msg_v01 *req)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_qmi_disable_force_clear_datapath_send, req);
-
-	return ret;
-}
-
-int ipa_generate_tag_process(void)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_generate_tag_process);
-
-	return ret;
-}
-
-int ipa_disable_sps_pipe(enum ipa_client_type client)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_disable_sps_pipe, client);
-
-	return ret;
-}
-
-int ipa_mhi_reset_channel_internal(enum ipa_client_type client)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_mhi_reset_channel_internal, client);
-
-	return ret;
-}
-
-int ipa_mhi_start_channel_internal(enum ipa_client_type client)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_mhi_start_channel_internal, client);
-
-	return ret;
-}
-
-void ipa_get_holb(int ep_idx, struct ipa_ep_cfg_holb *holb)
-{
-	IPA_API_DISPATCH(ipa_get_holb, ep_idx, holb);
-}
-
-void ipa_set_tag_process_before_gating(bool val)
-{
-	IPA_API_DISPATCH(ipa_set_tag_process_before_gating, val);
-}
-
-int ipa_mhi_query_ch_info(enum ipa_client_type client,
-		struct gsi_chan_info *ch_info)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_mhi_query_ch_info, client, ch_info);
-
-	return ret;
-}
-
-int ipa_uc_mhi_suspend_channel(int channelHandle)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_uc_mhi_suspend_channel, channelHandle);
-
-	return ret;
-}
-
-int ipa_uc_mhi_stop_event_update_channel(int channelHandle)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_uc_mhi_stop_event_update_channel,
-			channelHandle);
-
-	return ret;
-}
-
-bool ipa_has_open_aggr_frame(enum ipa_client_type client)
-{
-	bool ret;
-
-	IPA_API_DISPATCH_RETURN_BOOL(ipa_has_open_aggr_frame, client);
-
-	return ret;
-}
-
-int ipa_mhi_resume_channels_internal(enum ipa_client_type client,
-		bool LPTransitionRejected, bool brstmode_enabled,
-		union __packed gsi_channel_scratch ch_scratch, u8 index)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_mhi_resume_channels_internal, client,
-			LPTransitionRejected, brstmode_enabled, ch_scratch,
-			index);
-
-	return ret;
-}
-
-int ipa_uc_mhi_send_dl_ul_sync_info(union IpaHwMhiDlUlSyncCmdData_t *cmd)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_uc_mhi_send_dl_ul_sync_info,
-			cmd);
-
-	return ret;
-}
-
-int ipa_mhi_destroy_channel(enum ipa_client_type client)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_mhi_destroy_channel, client);
-
-	return ret;
-}
-
-int ipa_uc_mhi_init(void (*ready_cb)(void),
-		void (*wakeup_request_cb)(void))
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_uc_mhi_init, ready_cb, wakeup_request_cb);
-
-	return ret;
-}
-
-void ipa_uc_mhi_cleanup(void)
-{
-	IPA_API_DISPATCH(ipa_uc_mhi_cleanup);
-}
-
-int ipa_uc_mhi_print_stats(char *dbg_buff, int size)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_uc_mhi_print_stats, dbg_buff, size);
-
-	return ret;
-}
-
-/**
- * ipa_uc_state_check() - Check the status of the uC interface
- *
- * Return value: 0 if the uC is loaded, interface is initialized
- *               and there was no recent failure in one of the commands.
- *               A negative value is returned otherwise.
- */
-int ipa_uc_state_check(void)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_uc_state_check);
-
-	return ret;
-}
-
-int ipa_write_qmap_id(struct ipa_ioc_write_qmapid *param_in)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_write_qmap_id, param_in);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_write_qmap_id);
-
-/**
- * ipa_add_interrupt_handler() - Adds handler to an interrupt type
- * @interrupt:		Interrupt type
- * @handler:		The handler to be added
- * @deferred_flag:	whether the handler processing should be deferred in
- *			a workqueue
- * @private_data:	the client's private data
- *
- * Adds handler to an interrupt type and enable the specific bit
- * in IRQ_EN register, associated interrupt in IRQ_STTS register will be enabled
- */
-int ipa_add_interrupt_handler(enum ipa_irq_type interrupt,
-	ipa_irq_handler_t handler,
-	bool deferred_flag,
-	void *private_data)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_add_interrupt_handler, interrupt, handler,
-		deferred_flag, private_data);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_add_interrupt_handler);
-
-/**
- * ipa_remove_interrupt_handler() - Removes handler to an interrupt type
- * @interrupt:		Interrupt type
- *
- * Removes the handler and disable the specific bit in IRQ_EN register
- */
-int ipa_remove_interrupt_handler(enum ipa_irq_type interrupt)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_remove_interrupt_handler, interrupt);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_remove_interrupt_handler);
-
-/**
- * ipa_restore_suspend_handler() - restores the original suspend IRQ handler
- * as it was registered in the IPA init sequence.
- * Return codes:
- * 0: success
- * -EPERM: failed to remove current handler or failed to add original handler
- */
-int ipa_restore_suspend_handler(void)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_restore_suspend_handler);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_restore_suspend_handler);
-
-/**
- * ipa_bam_reg_dump() - Dump selected BAM registers for IPA and DMA-BAM
- *
- * Function is rate limited to avoid flooding kernel log buffer
- */
-void ipa_bam_reg_dump(void)
-{
-	IPA_API_DISPATCH(ipa_bam_reg_dump);
-}
-EXPORT_SYMBOL(ipa_bam_reg_dump);
-
-/**
- * ipa_get_ep_mapping() - provide endpoint mapping
- * @client: client type
- *
- * Return value: endpoint mapping
- */
-int ipa_get_ep_mapping(enum ipa_client_type client)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_get_ep_mapping, client);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_get_ep_mapping);
-
-/**
- * ipa_is_ready() - check if IPA module was initialized
- * successfully
- *
- * Return value: true for yes; false for no
- */
-bool ipa_is_ready(void)
-{
-	if (!ipa_api_ctrl || !ipa_api_ctrl->ipa_is_ready)
-		return false;
-	return ipa_api_ctrl->ipa_is_ready();
-}
-EXPORT_SYMBOL(ipa_is_ready);
-
-/**
- * ipa_proxy_clk_vote() - called to add IPA clock proxy vote
- *
- * Return value: none
- */
-void ipa_proxy_clk_vote(void)
-{
-	IPA_API_DISPATCH(ipa_proxy_clk_vote);
-}
-EXPORT_SYMBOL(ipa_proxy_clk_vote);
-
-/**
- * ipa_proxy_clk_unvote() - called to remove IPA clock proxy vote
- *
- * Return value: none
- */
-void ipa_proxy_clk_unvote(void)
-{
-	IPA_API_DISPATCH(ipa_proxy_clk_unvote);
-}
-EXPORT_SYMBOL(ipa_proxy_clk_unvote);
-
-/**
- * ipa_get_hw_type() - Return IPA HW version
- *
- * Return value: enum ipa_hw_type
- */
-enum ipa_hw_type ipa_get_hw_type(void)
-{
-	return ipa_api_hw_type;
-}
-EXPORT_SYMBOL(ipa_get_hw_type);
-
-/**
- * ipa_is_client_handle_valid() - check if IPA client handle is valid handle
- *
- * Return value: true for yes; false for no
- */
-bool ipa_is_client_handle_valid(u32 clnt_hdl)
-{
-	if (!ipa_api_ctrl || !ipa_api_ctrl->ipa_is_client_handle_valid)
-		return false;
-	return ipa_api_ctrl->ipa_is_client_handle_valid(clnt_hdl);
-}
-EXPORT_SYMBOL(ipa_is_client_handle_valid);
-
-/**
- * ipa_get_client_mapping() - provide client mapping
- * @pipe_idx: IPA end-point number
- *
- * Return value: client mapping
- */
-enum ipa_client_type ipa_get_client_mapping(int pipe_idx)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_get_client_mapping, pipe_idx);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_get_client_mapping);
-
-/**
- * ipa_get_rm_resource_from_ep() - this function is part of the deprecated
- * RM mechanism but is still used by some drivers so we kept the definition.
- */
-
-enum ipa_rm_resource_name ipa_get_rm_resource_from_ep(int pipe_idx)
-{
-	IPAERR("IPA RM is not supported idx=%d\n", pipe_idx);
-	return -EFAULT;
-}
-EXPORT_SYMBOL(ipa_get_rm_resource_from_ep);
-
-
-
-/**
- * ipa_get_modem_cfg_emb_pipe_flt()- Return ipa_ctx->modem_cfg_emb_pipe_flt
- *
- * Return value: true if modem configures embedded pipe flt, false otherwise
- */
-bool ipa_get_modem_cfg_emb_pipe_flt(void)
-{
-	if (!ipa_api_ctrl || !ipa_api_ctrl->ipa_get_modem_cfg_emb_pipe_flt)
-		return false;
-	return ipa_api_ctrl->ipa_get_modem_cfg_emb_pipe_flt();
-}
-EXPORT_SYMBOL(ipa_get_modem_cfg_emb_pipe_flt);
-
-/**
- * ipa_get_transport_type()- Return ipa_ctx->transport_prototype
- *
- * Return value: enum ipa_transport_type
- */
-enum ipa_transport_type ipa_get_transport_type(void)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_get_transport_type);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_get_transport_type);
-
-/**
- * ipa_get_smmu_domain()- Return the smmu domain
- *
- * Return value: pointer to iommu domain if smmu_cb valid, NULL otherwise
- */
-struct iommu_domain *ipa_get_smmu_domain(void)
-{
-	struct iommu_domain *ret;
-
-	IPA_API_DISPATCH_RETURN_PTR(ipa_get_smmu_domain);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_get_smmu_domain);
-
-/**
- * ipa_disable_apps_wan_cons_deaggr()- set
- * ipa_ctx->ipa_client_apps_wan_cons_agg_gro
- *
- * Return value: 0 or negative in case of failure
- */
-int ipa_disable_apps_wan_cons_deaggr(uint32_t agg_size, uint32_t agg_count)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_disable_apps_wan_cons_deaggr, agg_size,
-		agg_count);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_disable_apps_wan_cons_deaggr);
-
-/**
- * ipa_get_dma_dev()- Returns ipa_ctx dma dev pointer
- *
- * Return value: pointer to ipa_ctx dma dev pointer
- */
-struct device *ipa_get_dma_dev(void)
-{
-	struct device *ret;
-
-	IPA_API_DISPATCH_RETURN_PTR(ipa_get_dma_dev);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_get_dma_dev);
-
-/**
- * ipa_release_wdi_mapping() - release iommu mapping
- *
- *
- * @num_buffers: number of buffers to be released
- *
- * @info: pointer to wdi buffers info array
- *
- * Return codes: 0	  : success
- *		 negative : error
- */
-int ipa_release_wdi_mapping(u32 num_buffers, struct ipa_wdi_buffer_info *info)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_release_wdi_mapping, num_buffers, info);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_release_wdi_mapping);
-
-/**
- * ipa_create_wdi_mapping() - Perform iommu mapping
- *
- *
- * @num_buffers: number of buffers to be mapped
- *
- * @info: pointer to wdi buffers info array
- *
- * Return codes: 0	  : success
- *		 negative : error
- */
-int ipa_create_wdi_mapping(u32 num_buffers, struct ipa_wdi_buffer_info *info)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_create_wdi_mapping, num_buffers, info);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_create_wdi_mapping);
-
-/**
- * ipa_get_gsi_ep_info() - provide gsi ep information
- * @client: IPA client type
- *
- * Return value: pointer to ipa_gsi_ep_info
- */
-const struct ipa_gsi_ep_config *ipa_get_gsi_ep_info(enum ipa_client_type client)
-{
-	if (!ipa_api_ctrl || !ipa_api_ctrl->ipa_get_gsi_ep_info)
-		return NULL;
-	return ipa_api_ctrl->ipa_get_gsi_ep_info(client);
-}
-EXPORT_SYMBOL(ipa_get_gsi_ep_info);
-
-/**
- * ipa_stop_gsi_channel()- Stops a GSI channel in IPA
- *
- * Return value: 0 on success, negative otherwise
- */
-int ipa_stop_gsi_channel(u32 clnt_hdl)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_stop_gsi_channel, clnt_hdl);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_stop_gsi_channel);
-
-/**
- * ipa_start_gsi_channel()- Startsa GSI channel in IPA
- *
- * Return value: 0 on success, negative otherwise
- */
-int ipa_start_gsi_channel(u32 clnt_hdl)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_start_gsi_channel, clnt_hdl);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_start_gsi_channel);
-
-/**
- * ipa_is_vlan_mode - check if a LAN driver should load in VLAN mode
- * @iface - type of vlan capable device
- * @res - query result: true for vlan mode, false for non vlan mode
- *
- * API must be called after ipa_is_ready() returns true, otherwise it will fail
- *
- * Returns: 0 on success, negative on failure
- */
-int ipa_is_vlan_mode(enum ipa_vlan_ifaces iface, bool *res)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_is_vlan_mode, iface, res);
-
-	return ret;
-
-}
-EXPORT_SYMBOL(ipa_is_vlan_mode);
-
-/**
- * ipa_get_version_string() - Get string representation of IPA version
- * @ver: IPA version
- *
- * Return: Constant string representation
- */
-const char *ipa_get_version_string(enum ipa_hw_type ver)
-{
-	const char *str;
-
-	switch (ver) {
-	case IPA_HW_v1_0:
-		str = "1.0";
-		break;
-	case IPA_HW_v1_1:
-		str = "1.1";
-		break;
-	case IPA_HW_v2_0:
-		str = "2.0";
-		break;
-	case IPA_HW_v2_1:
-		str = "2.1";
-		break;
-	case IPA_HW_v2_5:
-		str = "2.5/2.6";
-		break;
-	case IPA_HW_v2_6L:
-		str = "2.6L";
-		break;
-	case IPA_HW_v3_0:
-		str = "3.0";
-		break;
-	case IPA_HW_v3_1:
-		str = "3.1";
-		break;
-	case IPA_HW_v3_5:
-		str = "3.5";
-		break;
-	case IPA_HW_v3_5_1:
-		str = "3.5.1";
-		break;
-	case IPA_HW_v4_0:
-		str = "4.0";
-		break;
-	case IPA_HW_v4_1:
-		str = "4.1";
-		break;
-	case IPA_HW_v4_2:
-		str = "4.2";
-		break;
-	case IPA_HW_v4_5:
-		str = "4.5";
-		break;
-	case IPA_HW_v4_7:
-		str = "4.7";
-		break;
-	default:
-		str = "Invalid version";
-		break;
-	}
-
-	return str;
-}
-EXPORT_SYMBOL(ipa_get_version_string);
-
-static const struct of_device_id ipa_plat_drv_match[] = {
-	{ .compatible = "qcom,ipa", },
-	{ .compatible = "qcom,ipa-smmu-ap-cb", },
-	{ .compatible = "qcom,ipa-smmu-wlan-cb", },
-	{ .compatible = "qcom,ipa-smmu-uc-cb", },
-	{ .compatible = "qcom,ipa-smmu-11ad-cb", },
-	{ .compatible = "qcom,smp2p-map-ipa-1-in", },
-	{ .compatible = "qcom,smp2p-map-ipa-1-out", },
-	{}
-};
-
-/*********************************************************/
-/*                PCIe Version                           */
-/*********************************************************/
-
-static const struct of_device_id ipa_pci_drv_match[] = {
-	{ .compatible = "qcom,ipa", },
-	{}
-};
-
-/*
- * Forward declarations of static functions required for PCI
- * registraion
- *
- * VENDOR and DEVICE should be defined in pci_ids.h
- */
-static int ipa_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
-static void ipa_pci_remove(struct pci_dev *pdev);
-static void ipa_pci_shutdown(struct pci_dev *pdev);
-static pci_ers_result_t ipa_pci_io_error_detected(struct pci_dev *dev,
-	pci_channel_state_t state);
-static pci_ers_result_t ipa_pci_io_slot_reset(struct pci_dev *dev);
-static void ipa_pci_io_resume(struct pci_dev *dev);
-
-#define LOCAL_VENDOR 0x17CB
-#define LOCAL_DEVICE 0x00ff
-
-static const char ipa_pci_driver_name[] = "qcipav3";
-
-static const struct pci_device_id ipa_pci_tbl[] = {
-	{ PCI_DEVICE(LOCAL_VENDOR, LOCAL_DEVICE) },
-	{ 0, 0, 0, 0, 0, 0, 0 }
-};
-
-MODULE_DEVICE_TABLE(pci, ipa_pci_tbl);
-
-/* PCI Error Recovery */
-static const struct pci_error_handlers ipa_pci_err_handler = {
-	.error_detected = ipa_pci_io_error_detected,
-	.slot_reset = ipa_pci_io_slot_reset,
-	.resume = ipa_pci_io_resume,
-};
-
-static struct pci_driver ipa_pci_driver = {
-	.name     = ipa_pci_driver_name,
-	.id_table = ipa_pci_tbl,
-	.probe    = ipa_pci_probe,
-	.remove   = ipa_pci_remove,
-	.shutdown = ipa_pci_shutdown,
-	.err_handler = &ipa_pci_err_handler
-};
-
-static int ipa_generic_plat_drv_probe(struct platform_device *pdev_p)
-{
-	int result;
-
-	/*
-	 * IPA probe function can be called for multiple times as the same probe
-	 * function handles multiple compatibilities
-	 */
-	pr_debug("ipa: IPA driver probing started for %s\n",
-		pdev_p->dev.of_node->name);
-
-	if (!ipa_api_ctrl) {
-		ipa_api_ctrl = kzalloc(sizeof(*ipa_api_ctrl), GFP_KERNEL);
-		if (!ipa_api_ctrl)
-			return -ENOMEM;
-
-		/* Get IPA HW Version */
-		result = of_property_read_u32(pdev_p->dev.of_node,
-			"qcom,ipa-hw-ver", &ipa_api_hw_type);
-		if ((result) || (ipa_api_hw_type == 0)) {
-			pr_err("ipa: get resource failed for ipa-hw-ver!\n");
-			kfree(ipa_api_ctrl);
-			ipa_api_ctrl = 0;
-			return -ENODEV;
-		}
-		pr_debug("ipa: ipa_api_hw_type = %d\n", ipa_api_hw_type);
-	}
-
-	/* call probe based on IPA HW version */
-	switch (ipa_api_hw_type) {
-	case IPA_HW_v3_0:
-	case IPA_HW_v3_1:
-	case IPA_HW_v3_5:
-	case IPA_HW_v3_5_1:
-	case IPA_HW_v4_0:
-	case IPA_HW_v4_1:
-	case IPA_HW_v4_2:
-	case IPA_HW_v4_5:
-	case IPA_HW_v4_7:
-		result = ipa3_plat_drv_probe(pdev_p, ipa_api_ctrl,
-			ipa_plat_drv_match);
-		break;
-	default:
-		pr_err("ipa: unsupported version %d\n", ipa_api_hw_type);
-		return -EPERM;
-	}
-
-	if (result && result != -EPROBE_DEFER)
-		pr_err("ipa: ipa_plat_drv_probe failed\n");
-
-	return result;
-}
-
-static int ipa_ap_suspend(struct device *dev)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_ap_suspend, dev);
-
-	return ret;
-}
-
-static int ipa_ap_resume(struct device *dev)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_ap_resume, dev);
-
-	return ret;
-}
-
-int ipa_register_ipa_ready_cb(void (*ipa_ready_cb)(void *user_data),
-			      void *user_data)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_register_ipa_ready_cb,
-				ipa_ready_cb, user_data);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_register_ipa_ready_cb);
-
-/**
- * ipa_inc_client_enable_clks() - Increase active clients counter, and
- * enable ipa clocks if necessary
- *
- * Please do not use this API, use the wrapper macros instead (ipa_i.h)
- * IPA_ACTIVE_CLIENTS_INC_XXX();
- *
- * Return codes:
- * None
- */
-void ipa_inc_client_enable_clks(struct ipa_active_client_logging_info *id)
-{
-	IPA_API_DISPATCH(ipa_inc_client_enable_clks, id);
-}
-EXPORT_SYMBOL(ipa_inc_client_enable_clks);
-
-/**
- * ipa_dec_client_disable_clks() - Increase active clients counter, and
- * enable ipa clocks if necessary
- *
- * Please do not use this API, use the wrapper macros instead (ipa_i.h)
- * IPA_ACTIVE_CLIENTS_DEC_XXX();
- *
- * Return codes:
- * None
- */
-void ipa_dec_client_disable_clks(struct ipa_active_client_logging_info *id)
-{
-	IPA_API_DISPATCH(ipa_dec_client_disable_clks, id);
-}
-EXPORT_SYMBOL(ipa_dec_client_disable_clks);
-
-/**
- * ipa_inc_client_enable_clks_no_block() - Only increment the number of active
- * clients if no asynchronous actions should be done.Asynchronous actions are
- * locking a mutex and waking up IPA HW.
- *
- * Please do not use this API, use the wrapper macros instead(ipa_i.h)
- *
- *
- * Return codes : 0 for success
- *		-EPERM if an asynchronous action should have been done
- */
-int ipa_inc_client_enable_clks_no_block(
-	struct ipa_active_client_logging_info *id)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_inc_client_enable_clks_no_block, id);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_inc_client_enable_clks_no_block);
-
-/**
- * ipa_suspend_resource_no_block() - suspend client endpoints related to the
- * IPA_RM resource and decrement active clients counter. This function is
- * guaranteed to avoid sleeping.
- *
- * @resource: [IN] IPA Resource Manager resource
- *
- * Return codes: 0 on success, negative on failure.
- */
-int ipa_suspend_resource_no_block(enum ipa_rm_resource_name resource)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_suspend_resource_no_block, resource);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_suspend_resource_no_block);
-/**
- * ipa_resume_resource() - resume client endpoints related to the IPA_RM
- * resource.
- *
- * @resource: [IN] IPA Resource Manager resource
- *
- * Return codes: 0 on success, negative on failure.
- */
-int ipa_resume_resource(enum ipa_rm_resource_name resource)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_resume_resource, resource);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_resume_resource);
-
-/**
- * ipa_suspend_resource_sync() - suspend client endpoints related to the IPA_RM
- * resource and decrement active clients counter, which may result in clock
- * gating of IPA clocks.
- *
- * @resource: [IN] IPA Resource Manager resource
- *
- * Return codes: 0 on success, negative on failure.
- */
-int ipa_suspend_resource_sync(enum ipa_rm_resource_name resource)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_suspend_resource_sync, resource);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_suspend_resource_sync);
-
-/**
- * ipa_set_required_perf_profile() - set IPA to the specified performance
- *	profile based on the bandwidth, unless minimum voltage required is
- *	higher. In this case the floor_voltage specified will be used.
- * @floor_voltage: minimum voltage to operate
- * @bandwidth_mbps: needed bandwidth from IPA
- *
- * Return codes: 0 on success, negative on failure.
- */
-int ipa_set_required_perf_profile(enum ipa_voltage_level floor_voltage,
-	u32 bandwidth_mbps)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_set_required_perf_profile, floor_voltage,
-		bandwidth_mbps);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_set_required_perf_profile);
-
-/**
- * ipa_get_ipc_logbuf() - return a pointer to IPA driver IPC log
- */
-void *ipa_get_ipc_logbuf(void)
-{
-	void *ret;
-
-	IPA_API_DISPATCH_RETURN_PTR(ipa_get_ipc_logbuf);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_get_ipc_logbuf);
-
-/**
- * ipa_get_ipc_logbuf_low() - return a pointer to IPA driver IPC low prio log
- */
-void *ipa_get_ipc_logbuf_low(void)
-{
-	void *ret;
-
-	IPA_API_DISPATCH_RETURN_PTR(ipa_get_ipc_logbuf_low);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_get_ipc_logbuf_low);
-
-/**
- * ipa_assert() - general function for assertion
- */
-void ipa_assert(void)
-{
-	pr_err("IPA: unrecoverable error has occurred, asserting\n");
-	BUG();
-}
-
-/**
- * ipa_rx_poll() - Poll the rx packets from IPA HW in the
- * softirq context
- *
- * @budget: number of packets to be polled in single iteration
- *
- * Return codes: >= 0  : Actual number of packets polled
- *
- */
-int ipa_rx_poll(u32 clnt_hdl, int budget)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_rx_poll, clnt_hdl, budget);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_rx_poll);
-
-/**
- * ipa_recycle_wan_skb() - Recycle the Wan skb
- *
- * @skb: skb that needs to recycle
- *
- */
-void ipa_recycle_wan_skb(struct sk_buff *skb)
-{
-	IPA_API_DISPATCH(ipa_recycle_wan_skb, skb);
-}
-EXPORT_SYMBOL(ipa_recycle_wan_skb);
-
-/**
- * ipa_setup_uc_ntn_pipes() - setup uc offload pipes
- */
-int ipa_setup_uc_ntn_pipes(struct ipa_ntn_conn_in_params *inp,
-		ipa_notify_cb notify, void *priv, u8 hdr_len,
-		struct ipa_ntn_conn_out_params *outp)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_setup_uc_ntn_pipes, inp,
-		notify, priv, hdr_len, outp);
-
-	return ret;
-}
-
-/**
- * ipa_tear_down_uc_offload_pipes() - tear down uc offload pipes
- */
-int ipa_tear_down_uc_offload_pipes(int ipa_ep_idx_ul,
-		int ipa_ep_idx_dl, struct ipa_ntn_conn_in_params *params)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_tear_down_uc_offload_pipes, ipa_ep_idx_ul,
-		ipa_ep_idx_dl, params);
-
-	return ret;
-}
-
-/**
- * ipa_get_pdev() - return a pointer to IPA dev struct
- *
- * Return value: a pointer to IPA dev struct
- *
- */
-struct device *ipa_get_pdev(void)
-{
-	struct device *ret;
-
-	IPA_API_DISPATCH_RETURN_PTR(ipa_get_pdev);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_get_pdev);
-
-int ipa_ntn_uc_reg_rdyCB(void (*ipauc_ready_cb)(void *user_data),
-			      void *user_data)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_ntn_uc_reg_rdyCB,
-				ipauc_ready_cb, user_data);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_ntn_uc_reg_rdyCB);
-
-void ipa_ntn_uc_dereg_rdyCB(void)
-{
-	IPA_API_DISPATCH(ipa_ntn_uc_dereg_rdyCB);
-}
-EXPORT_SYMBOL(ipa_ntn_uc_dereg_rdyCB);
-
-int ipa_get_smmu_params(struct ipa_smmu_in_params *in,
-	struct ipa_smmu_out_params *out)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_get_smmu_params, in, out);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_get_smmu_params);
-
-/**
- * ipa_conn_wdi_pipes() - connect wdi pipes
- */
-int ipa_conn_wdi_pipes(struct ipa_wdi_conn_in_params *in,
-	struct ipa_wdi_conn_out_params *out,
-	ipa_wdi_meter_notifier_cb wdi_notify)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_conn_wdi_pipes, in, out, wdi_notify);
-
-	return ret;
-}
-
-/**
- * ipa_disconn_wdi_pipes() - disconnect wdi pipes
- */
-int ipa_disconn_wdi_pipes(int ipa_ep_idx_tx, int ipa_ep_idx_rx)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_disconn_wdi_pipes, ipa_ep_idx_tx,
-		ipa_ep_idx_rx);
-
-	return ret;
-}
-
-/**
- * ipa_enable_wdi_pipes() - enable wdi pipes
- */
-int ipa_enable_wdi_pipes(int ipa_ep_idx_tx, int ipa_ep_idx_rx)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_enable_wdi_pipes, ipa_ep_idx_tx,
-		ipa_ep_idx_rx);
-
-	return ret;
-}
-
-/**
- * ipa_disable_wdi_pipes() - disable wdi pipes
- */
-int ipa_disable_wdi_pipes(int ipa_ep_idx_tx, int ipa_ep_idx_rx)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_disable_wdi_pipes, ipa_ep_idx_tx,
-		ipa_ep_idx_rx);
-
-	return ret;
-}
-
-/**
- * ipa_wigig_uc_msi_init() - smmu map\unmap msi related wigig HW registers
- *	and init\deinit uC msi config
- */
-int ipa_wigig_uc_msi_init(bool init,
-	phys_addr_t periph_baddr_pa,
-	phys_addr_t pseudo_cause_pa,
-	phys_addr_t int_gen_tx_pa,
-	phys_addr_t int_gen_rx_pa,
-	phys_addr_t dma_ep_misc_pa)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_wigig_uc_msi_init, init,
-		periph_baddr_pa,
-		pseudo_cause_pa,
-		int_gen_tx_pa,
-		int_gen_rx_pa,
-		dma_ep_misc_pa);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_wigig_uc_msi_init);
-
-/**
- * ipa_conn_wigig_rx_pipe_i() - connect wigig rx pipe
- */
-int ipa_conn_wigig_rx_pipe_i(void *in, struct ipa_wigig_conn_out_params *out,
-	struct dentry **parent)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_conn_wigig_rx_pipe_i, in, out, parent);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_conn_wigig_rx_pipe_i);
-
-/**
- * ipa_conn_wigig_client_i() - connect a wigig client
- */
-int ipa_conn_wigig_client_i(void *in,
-	struct ipa_wigig_conn_out_params *out,
-	ipa_notify_cb tx_notify,
-	void *priv)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_conn_wigig_client_i, in, out,
-		tx_notify, priv);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_conn_wigig_client_i);
-
-/**
- * ipa_disconn_wigig_pipe_i() - disconnect a wigig pipe
- */
-int ipa_disconn_wigig_pipe_i(enum ipa_client_type client,
-	struct ipa_wigig_pipe_setup_info_smmu *pipe_smmu,
-	void *dbuff)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_disconn_wigig_pipe_i, client,
-		pipe_smmu, dbuff);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_disconn_wigig_pipe_i);
-
-/**
- * ipa_enable_wigig_pipe() - enable a wigig pipe
- */
-int ipa_enable_wigig_pipe_i(enum ipa_client_type client)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_enable_wigig_pipe_i, client);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_enable_wigig_pipe_i);
-
-/**
- * ipa_disable_wigig_pipe_i() - disable a wigig pipe
- */
-int ipa_disable_wigig_pipe_i(enum ipa_client_type client)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_disable_wigig_pipe_i, client);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_disable_wigig_pipe_i);
-
-/**
- * ipa_tz_unlock_reg() - Allow AP access to memory regions controlled by TZ
- */
-int ipa_tz_unlock_reg(struct ipa_tz_unlock_reg_info *reg_info, u16 num_regs)
-{
-	int ret;
-
-	IPA_API_DISPATCH_RETURN(ipa_tz_unlock_reg, reg_info, num_regs);
-
-	return ret;
-}
-
-void ipa_register_client_callback(int (*client_cb)(bool is_lock),
-				bool (*teth_port_state)(void),
-					enum ipa_client_type client)
-{
-	IPA_API_DISPATCH(ipa_register_client_callback,
-		client_cb, teth_port_state, client);
-}
-
-void ipa_deregister_client_callback(enum ipa_client_type client)
-{
-	IPA_API_DISPATCH(ipa_deregister_client_callback,
-		client);
-}
-
-
-static const struct dev_pm_ops ipa_pm_ops = {
-	.suspend_noirq = ipa_ap_suspend,
-	.resume_noirq = ipa_ap_resume,
-};
-
-static struct platform_driver ipa_plat_drv = {
-	.probe = ipa_generic_plat_drv_probe,
-	.driver = {
-		.name = DRV_NAME,
-		.pm = &ipa_pm_ops,
-		.of_match_table = ipa_plat_drv_match,
-	},
-};
-
-/*********************************************************/
-/*                PCIe Version                           */
-/*********************************************************/
-
-static int ipa_pci_probe(
-	struct pci_dev             *pci_dev,
-	const struct pci_device_id *ent)
-{
-	int result;
-
-	if (!pci_dev || !ent) {
-		pr_err(
-		    "Bad arg: pci_dev (%pK) and/or ent (%pK)\n",
-		    pci_dev, ent);
-		return -EOPNOTSUPP;
-	}
-
-	if (!ipa_api_ctrl) {
-		ipa_api_ctrl = kzalloc(sizeof(*ipa_api_ctrl), GFP_KERNEL);
-		if (ipa_api_ctrl == NULL)
-			return -ENOMEM;
-		/* Get IPA HW Version */
-		result = of_property_read_u32(NULL,
-			"qcom,ipa-hw-ver", &ipa_api_hw_type);
-		if (result || ipa_api_hw_type == 0) {
-			pr_err("ipa: get resource failed for ipa-hw-ver!\n");
-			kfree(ipa_api_ctrl);
-			ipa_api_ctrl = NULL;
-			return -ENODEV;
-		}
-		pr_debug("ipa: ipa_api_hw_type = %d\n", ipa_api_hw_type);
-	}
-
-	/*
-	 * Call a reduced version of platform_probe appropriate for PCIe
-	 */
-	result = ipa3_pci_drv_probe(pci_dev, ipa_api_ctrl, ipa_pci_drv_match);
-
-	if (result && result != -EPROBE_DEFER)
-		pr_err("ipa: ipa3_pci_drv_probe failed\n");
-
-	if (running_emulation)
-		ipa_ut_module_init();
-
-	return result;
-}
-
-static void ipa_pci_remove(struct pci_dev *pci_dev)
-{
-	if (running_emulation)
-		ipa_ut_module_exit();
-}
-
-static void ipa_pci_shutdown(struct pci_dev *pci_dev)
-{
-}
-
-static pci_ers_result_t ipa_pci_io_error_detected(struct pci_dev *pci_dev,
-	pci_channel_state_t state)
-{
-	return 0;
-}
-
-static pci_ers_result_t ipa_pci_io_slot_reset(struct pci_dev *pci_dev)
-{
-	return 0;
-}
-
-static void ipa_pci_io_resume(struct pci_dev *pci_dev)
-{
-}
-
-static int __init ipa_module_init(void)
-{
-	pr_debug("IPA module init\n");
-
-	if (running_emulation) {
-		/* Register as a PCI device driver */
-		return pci_register_driver(&ipa_pci_driver);
-	}
-	/* Register as a platform device driver */
-	return platform_driver_register(&ipa_plat_drv);
-}
-subsys_initcall(ipa_module_init);
-
-MODULE_LICENSE("GPL v2");
-MODULE_DESCRIPTION("IPA HW device driver");

ipa/ipa_api.h

@@ -1,495 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2015-2019, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/ipa_mhi.h>
-#include <linux/ipa_uc_offload.h>
-#include <linux/ipa_wdi3.h>
-#include "ipa_common_i.h"
-
-#ifndef _IPA_API_H_
-#define _IPA_API_H_
-
-struct ipa_api_controller {
-	int (*ipa_reset_endpoint)(u32 clnt_hdl);
-
-	int (*ipa_clear_endpoint_delay)(u32 clnt_hdl);
-
-	int (*ipa_disable_endpoint)(u32 clnt_hdl);
-
-	int (*ipa_cfg_ep)(u32 clnt_hdl, const struct ipa_ep_cfg *ipa_ep_cfg);
-
-	int (*ipa_cfg_ep_nat)(u32 clnt_hdl,
-		const struct ipa_ep_cfg_nat *ipa_ep_cfg);
-
-	int (*ipa_cfg_ep_conn_track)(u32 clnt_hdl,
-		const struct ipa_ep_cfg_conn_track *ipa_ep_cfg);
-
-	int (*ipa_cfg_ep_hdr)(u32 clnt_hdl,
-		const struct ipa_ep_cfg_hdr *ipa_ep_cfg);
-
-	int (*ipa_cfg_ep_hdr_ext)(u32 clnt_hdl,
-			const struct ipa_ep_cfg_hdr_ext *ipa_ep_cfg);
-
-	int (*ipa_cfg_ep_mode)(u32 clnt_hdl,
-		const struct ipa_ep_cfg_mode *ipa_ep_cfg);
-
-	int (*ipa_cfg_ep_aggr)(u32 clnt_hdl,
-		const struct ipa_ep_cfg_aggr *ipa_ep_cfg);
-
-	int (*ipa_cfg_ep_deaggr)(u32 clnt_hdl,
-		const struct ipa_ep_cfg_deaggr *ipa_ep_cfg);
-
-	int (*ipa_cfg_ep_route)(u32 clnt_hdl,
-		const struct ipa_ep_cfg_route *ipa_ep_cfg);
-
-	int (*ipa_cfg_ep_holb)(u32 clnt_hdl,
-		const struct ipa_ep_cfg_holb *ipa_ep_cfg);
-
-	int (*ipa_cfg_ep_cfg)(u32 clnt_hdl,
-		const struct ipa_ep_cfg_cfg *ipa_ep_cfg);
-
-	int (*ipa_cfg_ep_metadata_mask)(u32 clnt_hdl,
-		const struct ipa_ep_cfg_metadata_mask *ipa_ep_cfg);
-
-	int (*ipa_cfg_ep_holb_by_client)(enum ipa_client_type client,
-		const struct ipa_ep_cfg_holb *ipa_ep_cfg);
-
-	int (*ipa_cfg_ep_ctrl)(u32 clnt_hdl,
-		const struct ipa_ep_cfg_ctrl *ep_ctrl);
-
-	int (*ipa_add_hdr)(struct ipa_ioc_add_hdr *hdrs);
-
-	int (*ipa_add_hdr_usr)(struct ipa_ioc_add_hdr *hdrs, bool user_only);
-
-	int (*ipa_del_hdr)(struct ipa_ioc_del_hdr *hdls);
-
-	int (*ipa_commit_hdr)(void);
-
-	int (*ipa_reset_hdr)(bool user_only);
-
-	int (*ipa_get_hdr)(struct ipa_ioc_get_hdr *lookup);
-
-	int (*ipa_put_hdr)(u32 hdr_hdl);
-
-	int (*ipa_copy_hdr)(struct ipa_ioc_copy_hdr *copy);
-
-	int (*ipa_add_hdr_proc_ctx)(struct ipa_ioc_add_hdr_proc_ctx *proc_ctxs,
-								bool user_only);
-
-	int (*ipa_del_hdr_proc_ctx)(struct ipa_ioc_del_hdr_proc_ctx *hdls);
-
-	int (*ipa_add_rt_rule)(struct ipa_ioc_add_rt_rule *rules);
-
-	int (*ipa_add_rt_rule_v2)(struct ipa_ioc_add_rt_rule_v2 *rules);
-
-	int (*ipa_add_rt_rule_usr)(struct ipa_ioc_add_rt_rule *rules,
-							bool user_only);
-
-	int (*ipa_add_rt_rule_usr_v2)(struct ipa_ioc_add_rt_rule_v2 *rules,
-							bool user_only);
-
-	int (*ipa_del_rt_rule)(struct ipa_ioc_del_rt_rule *hdls);
-
-	int (*ipa_commit_rt)(enum ipa_ip_type ip);
-
-	int (*ipa_reset_rt)(enum ipa_ip_type ip, bool user_only);
-
-	int (*ipa_get_rt_tbl)(struct ipa_ioc_get_rt_tbl *lookup);
-
-	int (*ipa_put_rt_tbl)(u32 rt_tbl_hdl);
-
-	int (*ipa_query_rt_index)(struct ipa_ioc_get_rt_tbl_indx *in);
-
-	int (*ipa_mdfy_rt_rule)(struct ipa_ioc_mdfy_rt_rule *rules);
-
-	int (*ipa_mdfy_rt_rule_v2)(struct ipa_ioc_mdfy_rt_rule_v2 *rules);
-
-	int (*ipa_add_flt_rule)(struct ipa_ioc_add_flt_rule *rules);
-
-	int (*ipa_add_flt_rule_v2)(struct ipa_ioc_add_flt_rule_v2 *rules);
-
-	int (*ipa_add_flt_rule_usr)(struct ipa_ioc_add_flt_rule *rules,
-								bool user_only);
-
-	int (*ipa_add_flt_rule_usr_v2)
-		(struct ipa_ioc_add_flt_rule_v2 *rules, bool user_only);
-
-	int (*ipa_del_flt_rule)(struct ipa_ioc_del_flt_rule *hdls);
-
-	int (*ipa_mdfy_flt_rule)(struct ipa_ioc_mdfy_flt_rule *rules);
-
-	int (*ipa_mdfy_flt_rule_v2)(struct ipa_ioc_mdfy_flt_rule_v2 *rules);
-
-	int (*ipa_commit_flt)(enum ipa_ip_type ip);
-
-	int (*ipa_reset_flt)(enum ipa_ip_type ip, bool user_only);
-
-	int (*ipa_allocate_nat_device)(struct ipa_ioc_nat_alloc_mem *mem);
-
-	int (*ipa_allocate_nat_table)(
-		struct ipa_ioc_nat_ipv6ct_table_alloc *table_alloc);
-
-	int (*ipa_allocate_ipv6ct_table)(
-		struct ipa_ioc_nat_ipv6ct_table_alloc *table_alloc);
-
-	int (*ipa_nat_init_cmd)(struct ipa_ioc_v4_nat_init *init);
-
-	int (*ipa_ipv6ct_init_cmd)(struct ipa_ioc_ipv6ct_init *init);
-
-	int (*ipa_nat_dma_cmd)(struct ipa_ioc_nat_dma_cmd *dma);
-
-	int (*ipa_table_dma_cmd)(struct ipa_ioc_nat_dma_cmd *dma);
-
-	int (*ipa_nat_del_cmd)(struct ipa_ioc_v4_nat_del *del);
-
-	int (*ipa_del_nat_table)(struct ipa_ioc_nat_ipv6ct_table_del *del);
-
-	int (*ipa_del_ipv6ct_table)(struct ipa_ioc_nat_ipv6ct_table_del *del);
-
-	int (*ipa_nat_mdfy_pdn)(struct ipa_ioc_nat_pdn_entry *mdfy_pdn);
-
-	int (*ipa_send_msg)(struct ipa_msg_meta *meta, void *buff,
-		ipa_msg_free_fn callback);
-
-	int (*ipa_register_pull_msg)(struct ipa_msg_meta *meta,
-		ipa_msg_pull_fn callback);
-
-	int (*ipa_deregister_pull_msg)(struct ipa_msg_meta *meta);
-
-	int (*ipa_register_intf)(const char *name,
-		const struct ipa_tx_intf *tx,
-		const struct ipa_rx_intf *rx);
-
-	int (*ipa_register_intf_ext)(const char *name,
-		const struct ipa_tx_intf *tx,
-		const struct ipa_rx_intf *rx,
-		const struct ipa_ext_intf *ext);
-
-	int (*ipa_deregister_intf)(const char *name);
-
-	int (*ipa_set_aggr_mode)(enum ipa_aggr_mode mode);
-
-	int (*ipa_set_qcncm_ndp_sig)(char sig[3]);
-
-	int (*ipa_set_single_ndp_per_mbim)(bool enable);
-
-	int (*ipa_tx_dp)(enum ipa_client_type dst, struct sk_buff *skb,
-		struct ipa_tx_meta *metadata);
-
-	int (*ipa_tx_dp_mul)(enum ipa_client_type dst,
-			struct ipa_tx_data_desc *data_desc);
-
-	void (*ipa_free_skb)(struct ipa_rx_data *data);
-
-	int (*ipa_setup_sys_pipe)(struct ipa_sys_connect_params *sys_in,
-		u32 *clnt_hdl);
-
-	int (*ipa_teardown_sys_pipe)(u32 clnt_hdl);
-
-	int (*ipa_sys_setup)(struct ipa_sys_connect_params *sys_in,
-		unsigned long *ipa_bam_hdl,
-		u32 *ipa_pipe_num, u32 *clnt_hdl, bool en_status);
-
-	int (*ipa_sys_teardown)(u32 clnt_hdl);
-
-	int (*ipa_sys_update_gsi_hdls)(u32 clnt_hdl, unsigned long gsi_ch_hdl,
-		unsigned long gsi_ev_hdl);
-
-	int (*ipa_connect_wdi_pipe)(struct ipa_wdi_in_params *in,
-		struct ipa_wdi_out_params *out);
-
-	int (*ipa_disconnect_wdi_pipe)(u32 clnt_hdl);
-
-	int (*ipa_enable_wdi_pipe)(u32 clnt_hdl);
-
-	int (*ipa_disable_wdi_pipe)(u32 clnt_hdl);
-
-	int (*ipa_resume_wdi_pipe)(u32 clnt_hdl);
-
-	int (*ipa_suspend_wdi_pipe)(u32 clnt_hdl);
-
-	int (*ipa_get_wdi_stats)(struct IpaHwStatsWDIInfoData_t *stats);
-
-	u16 (*ipa_get_smem_restr_bytes)(void);
-
-	int (*ipa_broadcast_wdi_quota_reach_ind)(uint32_t fid,
-		uint64_t num_bytes);
-
-	int (*ipa_uc_wdi_get_dbpa)(struct ipa_wdi_db_params *out);
-
-	int (*ipa_uc_reg_rdyCB)(struct ipa_wdi_uc_ready_params *param);
-
-	int (*ipa_uc_dereg_rdyCB)(void);
-
-	int (*teth_bridge_init)(struct teth_bridge_init_params *params);
-
-	int (*teth_bridge_disconnect)(enum ipa_client_type client);
-
-	int (*teth_bridge_connect)(
-		struct teth_bridge_connect_params *connect_params);
-
-	void (*ipa_set_client)(
-		int index, enum ipacm_client_enum client, bool uplink);
-
-	enum ipacm_client_enum (*ipa_get_client)(int pipe_idx);
-
-	bool (*ipa_get_client_uplink)(int pipe_idx);
-
-	int (*ipa_dma_init)(void);
-
-	int (*ipa_dma_enable)(void);
-
-	int (*ipa_dma_disable)(void);
-
-	int (*ipa_dma_sync_memcpy)(u64 dest, u64 src, int len);
-
-	int (*ipa_dma_async_memcpy)(u64 dest, u64 src, int len,
-		void (*user_cb)(void *user1), void *user_param);
-
-	int (*ipa_dma_uc_memcpy)(phys_addr_t dest, phys_addr_t src, int len);
-
-	void (*ipa_dma_destroy)(void);
-
-	bool (*ipa_has_open_aggr_frame)(enum ipa_client_type client);
-
-	int (*ipa_generate_tag_process)(void);
-
-	int (*ipa_disable_sps_pipe)(enum ipa_client_type client);
-
-	void (*ipa_set_tag_process_before_gating)(bool val);
-
-	int (*ipa_mhi_init_engine)(struct ipa_mhi_init_engine *params);
-
-	int (*ipa_connect_mhi_pipe)(struct ipa_mhi_connect_params_internal *in,
-		u32 *clnt_hdl);
-
-	int (*ipa_disconnect_mhi_pipe)(u32 clnt_hdl);
-
-	bool (*ipa_mhi_stop_gsi_channel)(enum ipa_client_type client);
-
-	int (*ipa_qmi_disable_force_clear)(u32 request_id);
-
-	int (*ipa_qmi_enable_force_clear_datapath_send)(
-		struct ipa_enable_force_clear_datapath_req_msg_v01 *req);
-
-	int (*ipa_qmi_disable_force_clear_datapath_send)(
-		struct ipa_disable_force_clear_datapath_req_msg_v01 *req);
-
-	bool (*ipa_mhi_sps_channel_empty)(enum ipa_client_type client);
-
-	int (*ipa_mhi_reset_channel_internal)(enum ipa_client_type client);
-
-	int (*ipa_mhi_start_channel_internal)(enum ipa_client_type client);
-
-	void (*ipa_get_holb)(int ep_idx, struct ipa_ep_cfg_holb *holb);
-
-	int (*ipa_mhi_query_ch_info)(enum ipa_client_type client,
-			struct gsi_chan_info *ch_info);
-
-	int (*ipa_mhi_resume_channels_internal)(
-			enum ipa_client_type client,
-			bool LPTransitionRejected,
-			bool brstmode_enabled,
-			union __packed gsi_channel_scratch ch_scratch,
-			u8 index);
-
-	int  (*ipa_mhi_destroy_channel)(enum ipa_client_type client);
-
-	int (*ipa_uc_mhi_send_dl_ul_sync_info)
-		(union IpaHwMhiDlUlSyncCmdData_t *cmd);
-
-	int (*ipa_uc_mhi_init)
-		(void (*ready_cb)(void), void (*wakeup_request_cb)(void));
-
-	void (*ipa_uc_mhi_cleanup)(void);
-
-	int (*ipa_uc_mhi_print_stats)(char *dbg_buff, int size);
-
-	int (*ipa_uc_mhi_reset_channel)(int channelHandle);
-
-	int (*ipa_uc_mhi_suspend_channel)(int channelHandle);
-
-	int (*ipa_uc_mhi_stop_event_update_channel)(int channelHandle);
-
-	int (*ipa_uc_state_check)(void);
-
-	int (*ipa_write_qmap_id)(struct ipa_ioc_write_qmapid *param_in);
-
-	int (*ipa_add_interrupt_handler)(enum ipa_irq_type interrupt,
-		ipa_irq_handler_t handler,
-		bool deferred_flag,
-		void *private_data);
-
-	int (*ipa_remove_interrupt_handler)(enum ipa_irq_type interrupt);
-
-	int (*ipa_restore_suspend_handler)(void);
-
-	void (*ipa_bam_reg_dump)(void);
-
-	int (*ipa_get_ep_mapping)(enum ipa_client_type client);
-
-	bool (*ipa_is_ready)(void);
-
-	void (*ipa_proxy_clk_vote)(void);
-
-	void (*ipa_proxy_clk_unvote)(void);
-
-	bool (*ipa_is_client_handle_valid)(u32 clnt_hdl);
-
-	enum ipa_client_type (*ipa_get_client_mapping)(int pipe_idx);
-
-	bool (*ipa_get_modem_cfg_emb_pipe_flt)(void);
-
-	enum ipa_transport_type (*ipa_get_transport_type)(void);
-
-	int (*ipa_ap_suspend)(struct device *dev);
-
-	int (*ipa_ap_resume)(struct device *dev);
-
-	int (*ipa_stop_gsi_channel)(u32 clnt_hdl);
-
-	int (*ipa_start_gsi_channel)(u32 clnt_hdl);
-
-	struct iommu_domain *(*ipa_get_smmu_domain)(void);
-
-	int (*ipa_disable_apps_wan_cons_deaggr)(uint32_t agg_size,
-						uint32_t agg_count);
-
-	struct device *(*ipa_get_dma_dev)(void);
-
-	int (*ipa_release_wdi_mapping)(u32 num_buffers,
-		struct ipa_wdi_buffer_info *info);
-
-	int (*ipa_create_wdi_mapping)(u32 num_buffers,
-		struct ipa_wdi_buffer_info *info);
-
-	const struct ipa_gsi_ep_config *(*ipa_get_gsi_ep_info)
-		(enum ipa_client_type client);
-
-	int (*ipa_register_ipa_ready_cb)(void (*ipa_ready_cb)(void *user_data),
-		void *user_data);
-
-	void (*ipa_inc_client_enable_clks)(
-		struct ipa_active_client_logging_info *id);
-
-	void (*ipa_dec_client_disable_clks)(
-		struct ipa_active_client_logging_info *id);
-
-	int (*ipa_inc_client_enable_clks_no_block)(
-		struct ipa_active_client_logging_info *id);
-
-	int (*ipa_suspend_resource_no_block)(
-		enum ipa_rm_resource_name resource);
-
-	int (*ipa_resume_resource)(enum ipa_rm_resource_name name);
-
-	int (*ipa_suspend_resource_sync)(enum ipa_rm_resource_name resource);
-
-	int (*ipa_set_required_perf_profile)(
-		enum ipa_voltage_level floor_voltage, u32 bandwidth_mbps);
-
-	void *(*ipa_get_ipc_logbuf)(void);
-
-	void *(*ipa_get_ipc_logbuf_low)(void);
-
-	int (*ipa_rx_poll)(u32 clnt_hdl, int budget);
-
-	void (*ipa_recycle_wan_skb)(struct sk_buff *skb);
-
-	int (*ipa_setup_uc_ntn_pipes)(struct ipa_ntn_conn_in_params *in,
-		ipa_notify_cb notify, void *priv, u8 hdr_len,
-		struct ipa_ntn_conn_out_params *outp);
-
-	int (*ipa_tear_down_uc_offload_pipes)(int ipa_ep_idx_ul,
-		int ipa_ep_idx_dl, struct ipa_ntn_conn_in_params *params);
-
-	struct device *(*ipa_get_pdev)(void);
-
-	int (*ipa_ntn_uc_reg_rdyCB)(void (*ipauc_ready_cb)(void *user_data),
-		void *user_data);
-
-	void (*ipa_ntn_uc_dereg_rdyCB)(void);
-
-	int (*ipa_conn_wdi_pipes)(struct ipa_wdi_conn_in_params *in,
-		struct ipa_wdi_conn_out_params *out,
-		ipa_wdi_meter_notifier_cb wdi_notify);
-
-	int (*ipa_disconn_wdi_pipes)(int ipa_ep_idx_tx,
-		int ipa_ep_idx_rx);
-
-	int (*ipa_enable_wdi_pipes)(int ipa_ep_idx_tx,
-		int ipa_ep_idx_rx);
-
-	int (*ipa_disable_wdi_pipes)(int ipa_ep_idx_tx,
-		int ipa_ep_idx_rx);
-
-	int (*ipa_tz_unlock_reg)(struct ipa_tz_unlock_reg_info *reg_info,
-		u16 num_regs);
-
-	int (*ipa_get_smmu_params)(struct ipa_smmu_in_params *in,
-		struct ipa_smmu_out_params *out);
-	int (*ipa_is_vlan_mode)(enum ipa_vlan_ifaces iface, bool *res);
-
-	int (*ipa_wigig_internal_init)(
-		struct ipa_wdi_uc_ready_params *inout,
-		ipa_wigig_misc_int_cb int_notify,
-		phys_addr_t *uc_db_pa);
-
-	int (*ipa_conn_wigig_rx_pipe_i)(void *in,
-		struct ipa_wigig_conn_out_params *out,
-		struct dentry **parent);
-
-	int (*ipa_conn_wigig_client_i)(void *in,
-		struct ipa_wigig_conn_out_params *out,
-		ipa_notify_cb tx_notify,
-		void *priv);
-
-	int (*ipa_disconn_wigig_pipe_i)(enum ipa_client_type client,
-		struct ipa_wigig_pipe_setup_info_smmu *pipe_smmu,
-		void *dbuff);
-
-	int (*ipa_wigig_uc_msi_init)(bool init,
-		phys_addr_t periph_baddr_pa,
-		phys_addr_t pseudo_cause_pa,
-		phys_addr_t int_gen_tx_pa,
-		phys_addr_t int_gen_rx_pa,
-		phys_addr_t dma_ep_misc_pa);
-
-	int (*ipa_enable_wigig_pipe_i)(enum ipa_client_type client);
-
-	int (*ipa_disable_wigig_pipe_i)(enum ipa_client_type client);
-
-	void (*ipa_register_client_callback)(
-		int (*client_cb)(bool is_lock),
-		bool (*teth_port_state)(void), enum ipa_client_type client);
-
-	void (*ipa_deregister_client_callback)(enum ipa_client_type client);
-};
-
-#ifdef CONFIG_IPA3
-int ipa3_plat_drv_probe(struct platform_device *pdev_p,
-	struct ipa_api_controller *api_ctrl,
-	const struct of_device_id *pdrv_match);
-int ipa3_pci_drv_probe(
-	struct pci_dev            *pci_dev,
-	struct ipa_api_controller *api_ctrl,
-	const struct of_device_id *pdrv_match);
-#else
-static inline int ipa3_plat_drv_probe(struct platform_device *pdev_p,
-	struct ipa_api_controller *api_ctrl,
-	const struct of_device_id *pdrv_match)
-{
-	return -ENODEV;
-}
-static inline int ipa3_pci_drv_probe(
-	struct pci_dev            *pci_dev,
-	struct ipa_api_controller *api_ctrl,
-	const struct of_device_id *pdrv_match)
-{
-	return -ENODEV;
-}
-#endif /* (CONFIG_IPA3) */
-
-#endif /* _IPA_API_H_ */

ipa/ipa_clients/Makefile

@@ -1,6 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-
-obj-$(CONFIG_IPA3) += ipa_usb.o odu_bridge.o ipa_mhi_client.o ipa_uc_offload.o ipa_wdi3.o ipa_gsb.o ipa_wigig.o
-obj-$(CONFIG_IPA) += odu_bridge.o ipa_mhi_client.o ipa_uc_offload.o ipa_wdi3.o
-obj-$(CONFIG_ECM_IPA) += ecm_ipa.o
-obj-$(CONFIG_RNDIS_IPA) += rndis_ipa.o

ipa/ipa_clients/ecm_ipa.c

@@ -1,1477 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2013-2019, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/debugfs.h>
-#include <linux/errno.h>
-#include <linux/etherdevice.h>
-#include <linux/if_vlan.h>
-#include <linux/fs.h>
-#include <linux/module.h>
-#include <linux/netdevice.h>
-#include <linux/skbuff.h>
-#include <linux/sched.h>
-#include <linux/atomic.h>
-#include <linux/ecm_ipa.h>
-#include "../ipa_common_i.h"
-#include "../ipa_v3/ipa_pm.h"
-
-#define DRIVER_NAME "ecm_ipa"
-#define ECM_IPA_IPV4_HDR_NAME "ecm_eth_ipv4"
-#define ECM_IPA_IPV6_HDR_NAME "ecm_eth_ipv6"
-#define INACTIVITY_MSEC_DELAY 100
-#define DEFAULT_OUTSTANDING_HIGH 64
-#define DEFAULT_OUTSTANDING_LOW 32
-#define DEBUGFS_TEMP_BUF_SIZE 4
-#define TX_TIMEOUT (5 * HZ)
-
-#define IPA_ECM_IPC_LOG_PAGES 50
-
-#define IPA_ECM_IPC_LOGGING(buf, fmt, args...) \
-	do { \
-		if (buf) \
-			ipc_log_string((buf), fmt, __func__, __LINE__, \
-				## args); \
-	} while (0)
-
-static void *ipa_ecm_logbuf;
-
-#define ECM_IPA_DEBUG(fmt, args...) \
-	do { \
-		pr_debug(DRIVER_NAME " %s:%d "\
-			fmt, __func__, __LINE__, ## args);\
-		if (ipa_ecm_logbuf) { \
-			IPA_ECM_IPC_LOGGING(ipa_ecm_logbuf, \
-				DRIVER_NAME " %s:%d " fmt, ## args); \
-		} \
-	} while (0)
-
-#define ECM_IPA_DEBUG_XMIT(fmt, args...) \
-	pr_debug(DRIVER_NAME " %s:%d " fmt, __func__, __LINE__, ## args)
-
-#define ECM_IPA_INFO(fmt, args...) \
-	do { \
-		pr_info(DRIVER_NAME "@%s@%d@ctx:%s: "\
-			fmt, __func__, __LINE__, current->comm, ## args);\
-		if (ipa_ecm_logbuf) { \
-			IPA_ECM_IPC_LOGGING(ipa_ecm_logbuf, \
-				DRIVER_NAME " %s:%d " fmt, ## args); \
-		} \
-	} while (0)
-
-#define ECM_IPA_ERROR(fmt, args...) \
-	do { \
-		pr_err(DRIVER_NAME "@%s@%d@ctx:%s: "\
-			fmt, __func__, __LINE__, current->comm, ## args);\
-		if (ipa_ecm_logbuf) { \
-			IPA_ECM_IPC_LOGGING(ipa_ecm_logbuf, \
-				DRIVER_NAME " %s:%d " fmt, ## args); \
-		} \
-	} while (0)
-
-#define NULL_CHECK(ptr) \
-	do { \
-		if (!(ptr)) { \
-			ECM_IPA_ERROR("null pointer #ptr\n"); \
-			ret = -EINVAL; \
-		} \
-	} \
-	while (0)
-
-#define ECM_IPA_LOG_ENTRY() ECM_IPA_DEBUG("begin\n")
-#define ECM_IPA_LOG_EXIT() ECM_IPA_DEBUG("end\n")
-
-/**
- * enum ecm_ipa_state - specify the current driver internal state
- *  which is guarded by a state machine.
- *
- * The driver internal state changes due to its external API usage.
- * The driver saves its internal state to guard from caller illegal
- * call sequence.
- * states:
- * UNLOADED is the first state which is the default one and is also the state
- *  after the driver gets unloaded(cleanup).
- * INITIALIZED is the driver state once it finished registering
- *  the network device and all internal data struct were initialized
- * CONNECTED is the driver state once the USB pipes were connected to IPA
- * UP is the driver state after the interface mode was set to UP but the
- *  pipes are not connected yet - this state is meta-stable state.
- * CONNECTED_AND_UP is the driver state when the pipe were connected and
- *  the interface got UP request from the network stack. this is the driver
- *   idle operation state which allows it to transmit/receive data.
- * INVALID is a state which is not allowed.
- */
-enum ecm_ipa_state {
-	ECM_IPA_UNLOADED = 0,
-	ECM_IPA_INITIALIZED,
-	ECM_IPA_CONNECTED,
-	ECM_IPA_UP,
-	ECM_IPA_CONNECTED_AND_UP,
-	ECM_IPA_INVALID,
-};
-
-/**
- * enum ecm_ipa_operation - enumerations used to describe the API operation
- *
- * Those enums are used as input for the driver state machine.
- */
-enum ecm_ipa_operation {
-	ECM_IPA_INITIALIZE,
-	ECM_IPA_CONNECT,
-	ECM_IPA_OPEN,
-	ECM_IPA_STOP,
-	ECM_IPA_DISCONNECT,
-	ECM_IPA_CLEANUP,
-};
-
-#define ECM_IPA_STATE_DEBUG(ecm_ipa_ctx) \
-	ECM_IPA_DEBUG("Driver state - %s\n",\
-	ecm_ipa_state_string((ecm_ipa_ctx)->state))
-
-/**
- * struct ecm_ipa_dev - main driver context parameters
- * @net: network interface struct implemented by this driver
- * @directory: debugfs directory for various debuging switches
- * @eth_ipv4_hdr_hdl: saved handle for ipv4 header-insertion table
- * @eth_ipv6_hdr_hdl: saved handle for ipv6 header-insertion table
- * @usb_to_ipa_hdl: save handle for IPA pipe operations
- * @ipa_to_usb_hdl: save handle for IPA pipe operations
- * @outstanding_pkts: number of packets sent to IPA without TX complete ACKed
- * @outstanding_high: number of outstanding packets allowed
- * @outstanding_low: number of outstanding packets which shall cause
- *  to netdev queue start (after stopped due to outstanding_high reached)
- * @state: current state of ecm_ipa driver
- * @device_ready_notify: callback supplied by USB core driver
- * This callback shall be called by the Netdev once the Netdev internal
- * state is changed to RNDIS_IPA_CONNECTED_AND_UP
- * @ipa_to_usb_client: consumer client
- * @usb_to_ipa_client: producer client
- * @pm_hdl: handle for IPA PM
- * @is_vlan_mode: does the driver need to work in VLAN mode?
- */
-struct ecm_ipa_dev {
-	struct net_device *net;
-	struct dentry *directory;
-	u32 eth_ipv4_hdr_hdl;
-	u32 eth_ipv6_hdr_hdl;
-	u32 usb_to_ipa_hdl;
-	u32 ipa_to_usb_hdl;
-	atomic_t outstanding_pkts;
-	u8 outstanding_high;
-	u8 outstanding_low;
-	enum ecm_ipa_state state;
-	void (*device_ready_notify)(void);
-	enum ipa_client_type ipa_to_usb_client;
-	enum ipa_client_type usb_to_ipa_client;
-	u32 pm_hdl;
-	bool is_vlan_mode;
-};
-
-static int ecm_ipa_open(struct net_device *net);
-static void ecm_ipa_packet_receive_notify
-	(void *priv, enum ipa_dp_evt_type evt, unsigned long data);
-static void ecm_ipa_tx_complete_notify
-	(void *priv, enum ipa_dp_evt_type evt, unsigned long data);
-static void ecm_ipa_tx_timeout(struct net_device *net);
-static int ecm_ipa_stop(struct net_device *net);
-static void ecm_ipa_enable_data_path(struct ecm_ipa_dev *ecm_ipa_ctx);
-static int ecm_ipa_rules_cfg
-	(struct ecm_ipa_dev *ecm_ipa_ctx, const void *dst_mac,
-		const void *src_mac);
-static void ecm_ipa_rules_destroy(struct ecm_ipa_dev *ecm_ipa_ctx);
-static int ecm_ipa_register_properties(struct ecm_ipa_dev *ecm_ipa_ctx);
-static void ecm_ipa_deregister_properties(void);
-static struct net_device_stats *ecm_ipa_get_stats(struct net_device *net);
-static int ecm_ipa_register_pm_client(struct ecm_ipa_dev *ecm_ipa_ctx);
-static void ecm_ipa_deregister_pm_client(struct ecm_ipa_dev *ecm_ipa_ctx);
-static netdev_tx_t ecm_ipa_start_xmit
-	(struct sk_buff *skb, struct net_device *net);
-static int ecm_ipa_debugfs_atomic_open(struct inode *inode, struct file *file);
-static ssize_t ecm_ipa_debugfs_atomic_read
-	(struct file *file, char __user *ubuf, size_t count, loff_t *ppos);
-static void ecm_ipa_debugfs_init(struct ecm_ipa_dev *ecm_ipa_ctx);
-static void ecm_ipa_debugfs_destroy(struct ecm_ipa_dev *ecm_ipa_ctx);
-static int ecm_ipa_ep_registers_cfg(u32 usb_to_ipa_hdl, u32 ipa_to_usb_hdl,
-	bool is_vlan_mode);
-static int ecm_ipa_set_device_ethernet_addr
-	(u8 *dev_ethaddr, u8 device_ethaddr[]);
-static enum ecm_ipa_state ecm_ipa_next_state
-	(enum ecm_ipa_state current_state, enum ecm_ipa_operation operation);
-static const char *ecm_ipa_state_string(enum ecm_ipa_state state);
-static int ecm_ipa_init_module(void);
-static void ecm_ipa_cleanup_module(void);
-
-static const struct net_device_ops ecm_ipa_netdev_ops = {
-	.ndo_open		= ecm_ipa_open,
-	.ndo_stop		= ecm_ipa_stop,
-	.ndo_start_xmit = ecm_ipa_start_xmit,
-	.ndo_set_mac_address = eth_mac_addr,
-	.ndo_tx_timeout = ecm_ipa_tx_timeout,
-	.ndo_get_stats = ecm_ipa_get_stats,
-};
-
-const struct file_operations ecm_ipa_debugfs_atomic_ops = {
-	.open = ecm_ipa_debugfs_atomic_open,
-	.read = ecm_ipa_debugfs_atomic_read,
-};
-
-static void ecm_ipa_msg_free_cb(void *buff, u32 len, u32 type)
-{
-	kfree(buff);
-}
-
-/**
- * ecm_ipa_init() - create network device and initializes internal
- *  data structures
- * @params: in/out parameters required for ecm_ipa initialization
- *
- * Shall be called prior to pipe connection.
- * The out parameters (the callbacks) shall be supplied to ipa_connect.
- * Detailed description:
- *  - allocate the network device
- *  - set default values for driver internals
- *  - create debugfs folder and files
- *  - add header insertion rules for IPA driver (based on host/device
- *    Ethernet addresses given in input params)
- *  - register tx/rx properties to IPA driver (will be later used
- *    by IPA configuration manager to configure reset of the IPA rules)
- *  - set the carrier state to "off" (until ecm_ipa_connect is called)
- *  - register the network device
- *  - set the out parameters
- *
- * Returns negative errno, or zero on success
- */
-int ecm_ipa_init(struct ecm_ipa_params *params)
-{
-	int result = 0;
-	struct net_device *net;
-	struct ecm_ipa_dev *ecm_ipa_ctx;
-	int ret;
-
-	ECM_IPA_LOG_ENTRY();
-
-	ECM_IPA_DEBUG("%s initializing\n", DRIVER_NAME);
-	ret = 0;
-	NULL_CHECK(params);
-	if (ret)
-		return ret;
-
-	ECM_IPA_DEBUG
-		("host_ethaddr=%pM, device_ethaddr=%pM\n",
-		params->host_ethaddr,
-		params->device_ethaddr);
-
-	net = alloc_etherdev(sizeof(struct ecm_ipa_dev));
-	if (!net) {
-		result = -ENOMEM;
-		ECM_IPA_ERROR("fail to allocate etherdev\n");
-		goto fail_alloc_etherdev;
-	}
-	ECM_IPA_DEBUG("network device was successfully allocated\n");
-
-	ecm_ipa_ctx = netdev_priv(net);
-	if (!ecm_ipa_ctx) {
-		ECM_IPA_ERROR("fail to extract netdev priv\n");
-		result = -ENOMEM;
-		goto fail_netdev_priv;
-	}
-	memset(ecm_ipa_ctx, 0, sizeof(*ecm_ipa_ctx));
-	ECM_IPA_DEBUG("ecm_ipa_ctx (private) = %pK\n", ecm_ipa_ctx);
-
-	ecm_ipa_ctx->net = net;
-	ecm_ipa_ctx->outstanding_high = DEFAULT_OUTSTANDING_HIGH;
-	ecm_ipa_ctx->outstanding_low = DEFAULT_OUTSTANDING_LOW;
-	atomic_set(&ecm_ipa_ctx->outstanding_pkts, 0);
-	snprintf(net->name, sizeof(net->name), "%s%%d", "ecm");
-	net->netdev_ops = &ecm_ipa_netdev_ops;
-	net->watchdog_timeo = TX_TIMEOUT;
-	ECM_IPA_DEBUG("internal data structures were initialized\n");
-
-	if (!params->device_ready_notify)
-		ECM_IPA_DEBUG("device_ready_notify() was not supplied");
-	ecm_ipa_ctx->device_ready_notify = params->device_ready_notify;
-
-	ecm_ipa_debugfs_init(ecm_ipa_ctx);
-
-	result = ecm_ipa_set_device_ethernet_addr
-		(net->dev_addr, params->device_ethaddr);
-	if (result) {
-		ECM_IPA_ERROR("set device MAC failed\n");
-		goto fail_set_device_ethernet;
-	}
-	ECM_IPA_DEBUG("Device Ethernet address set %pM\n", net->dev_addr);
-
-	if (ipa_is_vlan_mode(IPA_VLAN_IF_ECM, &ecm_ipa_ctx->is_vlan_mode)) {
-		ECM_IPA_ERROR("couldn't acquire vlan mode, is ipa ready?\n");
-		goto fail_get_vlan_mode;
-	}
-	ECM_IPA_DEBUG("is vlan mode %d\n", ecm_ipa_ctx->is_vlan_mode);
-
-	result = ecm_ipa_rules_cfg
-		(ecm_ipa_ctx, params->host_ethaddr, params->device_ethaddr);
-	if (result) {
-		ECM_IPA_ERROR("fail on ipa rules set\n");
-		goto fail_rules_cfg;
-	}
-	ECM_IPA_DEBUG("Ethernet header insertion set\n");
-
-	netif_carrier_off(net);
-	ECM_IPA_DEBUG("netif_carrier_off() was called\n");
-
-	netif_stop_queue(ecm_ipa_ctx->net);
-	ECM_IPA_DEBUG("netif_stop_queue() was called");
-
-	result = register_netdev(net);
-	if (result) {
-		ECM_IPA_ERROR("register_netdev failed: %d\n", result);
-		goto fail_register_netdev;
-	}
-	ECM_IPA_DEBUG("register_netdev succeeded\n");
-
-	params->ecm_ipa_rx_dp_notify = ecm_ipa_packet_receive_notify;
-	params->ecm_ipa_tx_dp_notify = ecm_ipa_tx_complete_notify;
-	params->private = (void *)ecm_ipa_ctx;
-	params->skip_ep_cfg = false;
-	ecm_ipa_ctx->state = ECM_IPA_INITIALIZED;
-	ECM_IPA_STATE_DEBUG(ecm_ipa_ctx);
-
-	ECM_IPA_INFO("ECM_IPA was initialized successfully\n");
-
-	ECM_IPA_LOG_EXIT();
-
-	return 0;
-
-fail_register_netdev:
-	ecm_ipa_rules_destroy(ecm_ipa_ctx);
-fail_rules_cfg:
-fail_get_vlan_mode:
-fail_set_device_ethernet:
-	ecm_ipa_debugfs_destroy(ecm_ipa_ctx);
-fail_netdev_priv:
-	free_netdev(net);
-fail_alloc_etherdev:
-	return result;
-}
-EXPORT_SYMBOL(ecm_ipa_init);
-
-/**
- * ecm_ipa_connect() - notify ecm_ipa for IPA<->USB pipes connection
- * @usb_to_ipa_hdl: handle of IPA driver client for USB->IPA
- * @ipa_to_usb_hdl: handle of IPA driver client for IPA->USB
- * @priv: same value that was set by ecm_ipa_init(), this
- *  parameter holds the network device pointer.
- *
- * Once USB driver finishes the pipe connection between IPA core
- * and USB core this method shall be called in order to
- * allow ecm_ipa complete the data path configurations.
- * Caller should make sure that it is calling this function
- * from a context that allows it to handle device_ready_notify().
- * Detailed description:
- *  - configure the IPA end-points register
- *  - notify the Linux kernel for "carrier_on"
- *  After this function is done the driver state changes to "Connected".
- *  This API is expected to be called after ecm_ipa_init() or
- *  after a call to ecm_ipa_disconnect.
- */
-int ecm_ipa_connect(u32 usb_to_ipa_hdl, u32 ipa_to_usb_hdl, void *priv)
-{
-	struct ecm_ipa_dev *ecm_ipa_ctx = priv;
-	int next_state;
-	struct ipa_ecm_msg *ecm_msg;
-	struct ipa_msg_meta msg_meta;
-	int retval;
-	int ret;
-
-	ECM_IPA_LOG_ENTRY();
-	ret = 0;
-	NULL_CHECK(priv);
-	if (ret)
-		return ret;
-	ECM_IPA_DEBUG("usb_to_ipa_hdl = %d, ipa_to_usb_hdl = %d, priv=0x%pK\n",
-		      usb_to_ipa_hdl, ipa_to_usb_hdl, priv);
-
-	next_state = ecm_ipa_next_state(ecm_ipa_ctx->state, ECM_IPA_CONNECT);
-	if (next_state == ECM_IPA_INVALID) {
-		ECM_IPA_ERROR("can't call connect before calling initialize\n");
-		return -EPERM;
-	}
-	ecm_ipa_ctx->state = next_state;
-	ECM_IPA_STATE_DEBUG(ecm_ipa_ctx);
-
-	if (!ipa_is_client_handle_valid(usb_to_ipa_hdl)) {
-		ECM_IPA_ERROR
-			("usb_to_ipa_hdl(%d) is not a valid ipa handle\n",
-			usb_to_ipa_hdl);
-		return -EINVAL;
-	}
-	if (!ipa_is_client_handle_valid(ipa_to_usb_hdl)) {
-		ECM_IPA_ERROR
-			("ipa_to_usb_hdl(%d) is not a valid ipa handle\n",
-			ipa_to_usb_hdl);
-		return -EINVAL;
-	}
-
-	ecm_ipa_ctx->ipa_to_usb_hdl = ipa_to_usb_hdl;
-	ecm_ipa_ctx->usb_to_ipa_hdl = usb_to_ipa_hdl;
-
-	ecm_ipa_ctx->ipa_to_usb_client = ipa_get_client_mapping(ipa_to_usb_hdl);
-	if (ecm_ipa_ctx->ipa_to_usb_client < 0) {
-		ECM_IPA_ERROR(
-			"Error getting IPA->USB client from handle %d\n",
-			ecm_ipa_ctx->ipa_to_usb_client);
-		return -EINVAL;
-	}
-	ECM_IPA_DEBUG("ipa_to_usb_client = %d\n",
-		      ecm_ipa_ctx->ipa_to_usb_client);
-
-	ecm_ipa_ctx->usb_to_ipa_client = ipa_get_client_mapping(usb_to_ipa_hdl);
-	if (ecm_ipa_ctx->usb_to_ipa_client < 0) {
-		ECM_IPA_ERROR(
-			"Error getting USB->IPA client from handle %d\n",
-			ecm_ipa_ctx->usb_to_ipa_client);
-		return -EINVAL;
-	}
-	ECM_IPA_DEBUG("usb_to_ipa_client = %d\n",
-		      ecm_ipa_ctx->usb_to_ipa_client);
-
-	retval = ecm_ipa_register_pm_client(ecm_ipa_ctx);
-
-	if (retval) {
-		ECM_IPA_ERROR("fail register PM client\n");
-		return retval;
-	}
-	ECM_IPA_DEBUG("PM client registered\n");
-
-	retval = ecm_ipa_register_properties(ecm_ipa_ctx);
-	if (retval) {
-		ECM_IPA_ERROR("fail on properties set\n");
-		goto fail_register_pm;
-	}
-	ECM_IPA_DEBUG("ecm_ipa 2 Tx and 2 Rx properties were registered\n");
-
-	retval = ecm_ipa_ep_registers_cfg(usb_to_ipa_hdl, ipa_to_usb_hdl,
-		ecm_ipa_ctx->is_vlan_mode);
-	if (retval) {
-		ECM_IPA_ERROR("fail on ep cfg\n");
-		goto fail;
-	}
-	ECM_IPA_DEBUG("end-point configured\n");
-
-	netif_carrier_on(ecm_ipa_ctx->net);
-
-	ecm_msg = kzalloc(sizeof(*ecm_msg), GFP_KERNEL);
-	if (!ecm_msg) {
-		retval = -ENOMEM;
-		goto fail;
-	}
-
-	memset(&msg_meta, 0, sizeof(struct ipa_msg_meta));
-	msg_meta.msg_type = ECM_CONNECT;
-	msg_meta.msg_len = sizeof(struct ipa_ecm_msg);
-	strlcpy(ecm_msg->name, ecm_ipa_ctx->net->name,
-		IPA_RESOURCE_NAME_MAX);
-	ecm_msg->ifindex = ecm_ipa_ctx->net->ifindex;
-
-	retval = ipa_send_msg(&msg_meta, ecm_msg, ecm_ipa_msg_free_cb);
-	if (retval) {
-		ECM_IPA_ERROR("fail to send ECM_CONNECT message\n");
-		kfree(ecm_msg);
-		goto fail;
-	}
-
-	if (!netif_carrier_ok(ecm_ipa_ctx->net)) {
-		ECM_IPA_ERROR("netif_carrier_ok error\n");
-		retval = -EBUSY;
-		goto fail;
-	}
-	ECM_IPA_DEBUG("carrier_on notified\n");
-
-	if (ecm_ipa_ctx->state == ECM_IPA_CONNECTED_AND_UP)
-		ecm_ipa_enable_data_path(ecm_ipa_ctx);
-	else
-		ECM_IPA_DEBUG("data path was not enabled yet\n");
-
-	ECM_IPA_INFO("ECM_IPA was connected successfully\n");
-
-	ECM_IPA_LOG_EXIT();
-
-	return 0;
-
-fail:
-	ecm_ipa_deregister_properties();
-fail_register_pm:
-	ecm_ipa_deregister_pm_client(ecm_ipa_ctx);
-	return retval;
-}
-EXPORT_SYMBOL(ecm_ipa_connect);
-
-/**
- * ecm_ipa_open() - notify Linux network stack to start sending packets
- * @net: the network interface supplied by the network stack
- *
- * Linux uses this API to notify the driver that the network interface
- * transitions to the up state.
- * The driver will instruct the Linux network stack to start
- * delivering data packets.
- */
-static int ecm_ipa_open(struct net_device *net)
-{
-	struct ecm_ipa_dev *ecm_ipa_ctx;
-	int next_state;
-
-	ECM_IPA_LOG_ENTRY();
-
-	ecm_ipa_ctx = netdev_priv(net);
-
-	next_state = ecm_ipa_next_state(ecm_ipa_ctx->state, ECM_IPA_OPEN);
-	if (next_state == ECM_IPA_INVALID) {
-		ECM_IPA_ERROR("can't bring driver up before initialize\n");
-		return -EPERM;
-	}
-	ecm_ipa_ctx->state = next_state;
-	ECM_IPA_STATE_DEBUG(ecm_ipa_ctx);
-
-	if (ecm_ipa_ctx->state == ECM_IPA_CONNECTED_AND_UP)
-		ecm_ipa_enable_data_path(ecm_ipa_ctx);
-	else
-		ECM_IPA_DEBUG("data path was not enabled yet\n");
-
-	ECM_IPA_LOG_EXIT();
-
-	return 0;
-}
-
-/**
- * ecm_ipa_start_xmit() - send data from APPs to USB core via IPA core
- * @skb: packet received from Linux network stack
- * @net: the network device being used to send this packet
- *
- * Several conditions needed in order to send the packet to IPA:
- * - Transmit queue for the network driver is currently
- *   in "send" state
- * - The driver internal state is in "UP" state.
- * - Filter Tx switch is turned off
- * - Outstanding high boundary did not reach.
- *
- * In case all of the above conditions are met, the network driver will
- * send the packet by using the IPA API for Tx.
- * In case the outstanding packet high boundary is reached, the driver will
- * stop the send queue until enough packet were proceeded by the IPA core.
- */
-static netdev_tx_t ecm_ipa_start_xmit
-	(struct sk_buff *skb, struct net_device *net)
-{
-	int ret;
-	netdev_tx_t status = NETDEV_TX_BUSY;
-	struct ecm_ipa_dev *ecm_ipa_ctx = netdev_priv(net);
-
-	netif_trans_update(net);
-
-	ECM_IPA_DEBUG_XMIT
-		("Tx, len=%d, skb->protocol=%d, outstanding=%d\n",
-		skb->len, skb->protocol,
-		atomic_read(&ecm_ipa_ctx->outstanding_pkts));
-
-	if (unlikely(netif_queue_stopped(net))) {
-		ECM_IPA_ERROR("interface queue is stopped\n");
-		goto out;
-	}
-
-	if (unlikely(ecm_ipa_ctx->state != ECM_IPA_CONNECTED_AND_UP)) {
-		ECM_IPA_ERROR("Missing pipe connected and/or iface up\n");
-		return NETDEV_TX_BUSY;
-	}
-
-	ret = ipa_pm_activate(ecm_ipa_ctx->pm_hdl);
-	if (ret) {
-		ECM_IPA_DEBUG("Failed to activate PM client\n");
-		netif_stop_queue(net);
-		goto fail_pm_activate;
-	}
-
-	if (atomic_read(&ecm_ipa_ctx->outstanding_pkts) >=
-					ecm_ipa_ctx->outstanding_high) {
-		ECM_IPA_DEBUG
-			("outstanding high (%d)- stopping\n",
-			ecm_ipa_ctx->outstanding_high);
-		netif_stop_queue(net);
-		status = NETDEV_TX_BUSY;
-		goto out;
-	}
-
-	if (ecm_ipa_ctx->is_vlan_mode)
-		if (unlikely(skb->protocol != htons(ETH_P_8021Q)))
-			ECM_IPA_DEBUG(
-				"ether_type != ETH_P_8021Q && vlan, prot = 0x%X\n"
-				, skb->protocol);
-
-	ret = ipa_tx_dp(ecm_ipa_ctx->ipa_to_usb_client, skb, NULL);
-	if (ret) {
-		ECM_IPA_ERROR("ipa transmit failed (%d)\n", ret);
-		goto fail_tx_packet;
-	}
-
-	atomic_inc(&ecm_ipa_ctx->outstanding_pkts);
-
-	status = NETDEV_TX_OK;
-	goto out;
-
-fail_tx_packet:
-out:
-	ipa_pm_deferred_deactivate(ecm_ipa_ctx->pm_hdl);
-fail_pm_activate:
-	return status;
-}
-
-/**
- * ecm_ipa_packet_receive_notify() - Rx notify
- *
- * @priv: ecm driver context
- * @evt: event type
- * @data: data provided with event
- *
- * IPA will pass a packet to the Linux network stack with skb->data pointing
- * to Ethernet packet frame.
- */
-static void ecm_ipa_packet_receive_notify
-	(void *priv, enum ipa_dp_evt_type evt, unsigned long data)
-{
-	struct sk_buff *skb = (struct sk_buff *)data;
-	struct ecm_ipa_dev *ecm_ipa_ctx = priv;
-	int result;
-	unsigned int packet_len;
-
-	if (!skb) {
-		ECM_IPA_ERROR("Bad SKB received from IPA driver\n");
-		return;
-	}
-
-	packet_len = skb->len;
-	ECM_IPA_DEBUG("packet RX, len=%d\n", skb->len);
-
-	if (unlikely(ecm_ipa_ctx->state != ECM_IPA_CONNECTED_AND_UP)) {
-		ECM_IPA_DEBUG("Missing pipe connected and/or iface up\n");
-		return;
-	}
-
-	if (evt != IPA_RECEIVE)	{
-		ECM_IPA_ERROR("A none IPA_RECEIVE event in ecm_ipa_receive\n");
-		return;
-	}
-
-	skb->dev = ecm_ipa_ctx->net;
-	skb->protocol = eth_type_trans(skb, ecm_ipa_ctx->net);
-
-	result = netif_rx(skb);
-	if (result)
-		ECM_IPA_ERROR("fail on netif_rx\n");
-	ecm_ipa_ctx->net->stats.rx_packets++;
-	ecm_ipa_ctx->net->stats.rx_bytes += packet_len;
-}
-
-/** ecm_ipa_stop() - called when network device transitions to the down
- *     state.
- *  @net: the network device being stopped.
- *
- * This API is used by Linux network stack to notify the network driver that
- * its state was changed to "down"
- * The driver will stop the "send" queue and change its internal
- * state to "Connected".
- */
-static int ecm_ipa_stop(struct net_device *net)
-{
-	struct ecm_ipa_dev *ecm_ipa_ctx = netdev_priv(net);
-	int next_state;
-
-	ECM_IPA_LOG_ENTRY();
-
-	next_state = ecm_ipa_next_state(ecm_ipa_ctx->state, ECM_IPA_STOP);
-	if (next_state == ECM_IPA_INVALID) {
-		ECM_IPA_ERROR("can't do network interface down without up\n");
-		return -EPERM;
-	}
-	ecm_ipa_ctx->state = next_state;
-	ECM_IPA_STATE_DEBUG(ecm_ipa_ctx);
-
-	netif_stop_queue(net);
-	ECM_IPA_DEBUG("network device stopped\n");
-
-	ECM_IPA_LOG_EXIT();
-	return 0;
-}
-
-/** ecm_ipa_disconnect() - called when the USB cable is unplugged.
- * @priv: same value that was set by ecm_ipa_init(), this
- *  parameter holds the network device pointer.
- *
- * Once the USB cable is unplugged the USB driver will notify the network
- * interface driver.
- * The internal driver state will returned to its initialized state and
- * Linux network stack will be informed for carrier off and the send queue
- * will be stopped.
- */
-int ecm_ipa_disconnect(void *priv)
-{
-	struct ecm_ipa_dev *ecm_ipa_ctx = priv;
-	int next_state;
-	struct ipa_ecm_msg *ecm_msg;
-	struct ipa_msg_meta msg_meta;
-	int retval;
-	int outstanding_dropped_pkts;
-	int ret;
-
-	ECM_IPA_LOG_ENTRY();
-	ret = 0;
-	NULL_CHECK(ecm_ipa_ctx);
-	if (ret)
-		return ret;
-	ECM_IPA_DEBUG("priv=0x%pK\n", priv);
-
-	next_state = ecm_ipa_next_state(ecm_ipa_ctx->state, ECM_IPA_DISCONNECT);
-	if (next_state == ECM_IPA_INVALID) {
-		ECM_IPA_ERROR("can't disconnect before connect\n");
-		return -EPERM;
-	}
-	ecm_ipa_ctx->state = next_state;
-	ECM_IPA_STATE_DEBUG(ecm_ipa_ctx);
-
-	netif_carrier_off(ecm_ipa_ctx->net);
-	ECM_IPA_DEBUG("carrier_off notifcation was sent\n");
-
-	ecm_msg = kzalloc(sizeof(*ecm_msg), GFP_KERNEL);
-	if (!ecm_msg)
-		return -ENOMEM;
-
-	memset(&msg_meta, 0, sizeof(struct ipa_msg_meta));
-	msg_meta.msg_type = ECM_DISCONNECT;
-	msg_meta.msg_len = sizeof(struct ipa_ecm_msg);
-	strlcpy(ecm_msg->name, ecm_ipa_ctx->net->name,
-		IPA_RESOURCE_NAME_MAX);
-	ecm_msg->ifindex = ecm_ipa_ctx->net->ifindex;
-
-	retval = ipa_send_msg(&msg_meta, ecm_msg, ecm_ipa_msg_free_cb);
-	if (retval) {
-		ECM_IPA_ERROR("fail to send ECM_DISCONNECT message\n");
-		kfree(ecm_msg);
-		return -EPERM;
-	}
-
-	netif_stop_queue(ecm_ipa_ctx->net);
-	ECM_IPA_DEBUG("queue stopped\n");
-
-	ecm_ipa_deregister_pm_client(ecm_ipa_ctx);
-
-	outstanding_dropped_pkts =
-		atomic_read(&ecm_ipa_ctx->outstanding_pkts);
-	ecm_ipa_ctx->net->stats.tx_errors += outstanding_dropped_pkts;
-	atomic_set(&ecm_ipa_ctx->outstanding_pkts, 0);
-
-	ECM_IPA_INFO("ECM_IPA was disconnected successfully\n");
-
-	ECM_IPA_LOG_EXIT();
-
-	return 0;
-}
-EXPORT_SYMBOL(ecm_ipa_disconnect);
-
-/**
- * ecm_ipa_cleanup() - unregister the network interface driver and free
- *  internal data structs.
- * @priv: same value that was set by ecm_ipa_init(), this
- *   parameter holds the network device pointer.
- *
- * This function shall be called once the network interface is not
- * needed anymore, e.g: when the USB composition does not support ECM.
- * This function shall be called after the pipes were disconnected.
- * Detailed description:
- *  -  remove the debugfs entries
- *  - deregister the network interface from Linux network stack
- *  - free all internal data structs
- */
-void ecm_ipa_cleanup(void *priv)
-{
-	struct ecm_ipa_dev *ecm_ipa_ctx = priv;
-	int next_state;
-
-	ECM_IPA_LOG_ENTRY();
-
-	ECM_IPA_DEBUG("priv=0x%pK\n", priv);
-
-	if (!ecm_ipa_ctx) {
-		ECM_IPA_ERROR("ecm_ipa_ctx NULL pointer\n");
-		return;
-	}
-
-	next_state = ecm_ipa_next_state(ecm_ipa_ctx->state, ECM_IPA_CLEANUP);
-	if (next_state == ECM_IPA_INVALID) {
-		ECM_IPA_ERROR("can't clean driver without cable disconnect\n");
-		return;
-	}
-	ecm_ipa_ctx->state = next_state;
-	ECM_IPA_STATE_DEBUG(ecm_ipa_ctx);
-
-	ecm_ipa_rules_destroy(ecm_ipa_ctx);
-	ecm_ipa_debugfs_destroy(ecm_ipa_ctx);
-
-	unregister_netdev(ecm_ipa_ctx->net);
-	free_netdev(ecm_ipa_ctx->net);
-
-	ECM_IPA_INFO("ECM_IPA was destroyed successfully\n");
-
-	ECM_IPA_LOG_EXIT();
-}
-EXPORT_SYMBOL(ecm_ipa_cleanup);
-
-static void ecm_ipa_enable_data_path(struct ecm_ipa_dev *ecm_ipa_ctx)
-{
-	if (ecm_ipa_ctx->device_ready_notify) {
-		ecm_ipa_ctx->device_ready_notify();
-		ECM_IPA_DEBUG("USB device_ready_notify() was called\n");
-	} else {
-		ECM_IPA_DEBUG("device_ready_notify() not supplied\n");
-	}
-
-	netif_start_queue(ecm_ipa_ctx->net);
-	ECM_IPA_DEBUG("queue started\n");
-}
-
-static void ecm_ipa_prepare_header_insertion(
-	int eth_type,
-	const char *hdr_name, struct ipa_hdr_add *add_hdr,
-	const void *dst_mac, const void *src_mac, bool is_vlan_mode)
-{
-	struct ethhdr *eth_hdr;
-	struct vlan_ethhdr *eth_vlan_hdr;
-
-	ECM_IPA_LOG_ENTRY();
-
-	add_hdr->is_partial = 0;
-	strlcpy(add_hdr->name, hdr_name, IPA_RESOURCE_NAME_MAX);
-	add_hdr->is_eth2_ofst_valid = true;
-	add_hdr->eth2_ofst = 0;
-
-	if (is_vlan_mode) {
-		eth_vlan_hdr = (struct vlan_ethhdr *)add_hdr->hdr;
-		memcpy(eth_vlan_hdr->h_dest, dst_mac, ETH_ALEN);
-		memcpy(eth_vlan_hdr->h_source, src_mac, ETH_ALEN);
-		eth_vlan_hdr->h_vlan_encapsulated_proto =
-			htons(eth_type);
-		eth_vlan_hdr->h_vlan_proto = htons(ETH_P_8021Q);
-		add_hdr->hdr_len = VLAN_ETH_HLEN;
-		add_hdr->type = IPA_HDR_L2_802_1Q;
-	} else {
-		eth_hdr = (struct ethhdr *)add_hdr->hdr;
-		memcpy(eth_hdr->h_dest, dst_mac, ETH_ALEN);
-		memcpy(eth_hdr->h_source, src_mac, ETH_ALEN);
-		eth_hdr->h_proto = htons(eth_type);
-		add_hdr->hdr_len = ETH_HLEN;
-		add_hdr->type = IPA_HDR_L2_ETHERNET_II;
-	}
-	ECM_IPA_LOG_EXIT();
-}
-
-/**
- * ecm_ipa_rules_cfg() - set header insertion and register Tx/Rx properties
- *				Headers will be committed to HW
- * @ecm_ipa_ctx: main driver context parameters
- * @dst_mac: destination MAC address
- * @src_mac: source MAC address
- *
- * Returns negative errno, or zero on success
- */
-static int ecm_ipa_rules_cfg
-	(struct ecm_ipa_dev *ecm_ipa_ctx,
-	const void *dst_mac, const void *src_mac)
-{
-	struct ipa_ioc_add_hdr *hdrs;
-	struct ipa_hdr_add *ipv4_hdr;
-	struct ipa_hdr_add *ipv6_hdr;
-	int result = 0;
-
-	ECM_IPA_LOG_ENTRY();
-	hdrs = kzalloc
-		(sizeof(*hdrs) + sizeof(*ipv4_hdr) + sizeof(*ipv6_hdr),
-			GFP_KERNEL);
-	if (!hdrs) {
-		result = -ENOMEM;
-		goto out;
-	}
-
-	ipv4_hdr = &hdrs->hdr[0];
-	ecm_ipa_prepare_header_insertion(
-		ETH_P_IP, ECM_IPA_IPV4_HDR_NAME,
-		ipv4_hdr, dst_mac, src_mac, ecm_ipa_ctx->is_vlan_mode);
-
-	ipv6_hdr = &hdrs->hdr[1];
-	ecm_ipa_prepare_header_insertion(
-		ETH_P_IPV6, ECM_IPA_IPV6_HDR_NAME,
-		ipv6_hdr, dst_mac, src_mac, ecm_ipa_ctx->is_vlan_mode);
-
-	hdrs->commit = 1;
-	hdrs->num_hdrs = 2;
-	result = ipa_add_hdr(hdrs);
-	if (result) {
-		ECM_IPA_ERROR("Fail on Header-Insertion(%d)\n", result);
-		goto out_free_mem;
-	}
-	if (ipv4_hdr->status) {
-		ECM_IPA_ERROR
-			("Fail on Header-Insertion ipv4(%d)\n",
-			ipv4_hdr->status);
-		result = ipv4_hdr->status;
-		goto out_free_mem;
-	}
-	if (ipv6_hdr->status) {
-		ECM_IPA_ERROR
-			("Fail on Header-Insertion ipv6(%d)\n",
-			ipv6_hdr->status);
-		result = ipv6_hdr->status;
-		goto out_free_mem;
-	}
-	ecm_ipa_ctx->eth_ipv4_hdr_hdl = ipv4_hdr->hdr_hdl;
-	ecm_ipa_ctx->eth_ipv6_hdr_hdl = ipv6_hdr->hdr_hdl;
-	ECM_IPA_LOG_EXIT();
-out_free_mem:
-	kfree(hdrs);
-out:
-	return result;
-}
-
-/**
- * ecm_ipa_rules_destroy() - remove the IPA core configuration done for
- *  the driver data path.
- *  @ecm_ipa_ctx: the driver context
- *
- *  Revert the work done on ecm_ipa_rules_cfg.
- */
-static void ecm_ipa_rules_destroy(struct ecm_ipa_dev *ecm_ipa_ctx)
-{
-	struct ipa_ioc_del_hdr *del_hdr;
-	struct ipa_hdr_del *ipv4;
-	struct ipa_hdr_del *ipv6;
-	int result;
-
-	del_hdr = kzalloc(sizeof(*del_hdr) + sizeof(*ipv4) +
-			sizeof(*ipv6), GFP_KERNEL);
-	if (!del_hdr)
-		return;
-	del_hdr->commit = 1;
-	del_hdr->num_hdls = 2;
-	ipv4 = &del_hdr->hdl[0];
-	ipv4->hdl = ecm_ipa_ctx->eth_ipv4_hdr_hdl;
-	ipv6 = &del_hdr->hdl[1];
-	ipv6->hdl = ecm_ipa_ctx->eth_ipv6_hdr_hdl;
-	result = ipa_del_hdr(del_hdr);
-	if (result || ipv4->status || ipv6->status)
-		ECM_IPA_ERROR("ipa_del_hdr failed\n");
-	kfree(del_hdr);
-}
-
-/* ecm_ipa_register_properties() - set Tx/Rx properties for ipacm
- *
- * Register ecm0 interface with 2 Tx properties and 2 Rx properties:
- * The 2 Tx properties are for data flowing from IPA to USB, they
- * have Header-Insertion properties both for Ipv4 and Ipv6 Ethernet framing.
- * The 2 Rx properties are for data flowing from USB to IPA, they have
- * simple rule which always "hit".
- *
- */
-static int ecm_ipa_register_properties(struct ecm_ipa_dev *ecm_ipa_ctx)
-{
-	struct ipa_tx_intf tx_properties = {0};
-	struct ipa_ioc_tx_intf_prop properties[2] = { {0}, {0} };
-	struct ipa_ioc_tx_intf_prop *ipv4_property;
-	struct ipa_ioc_tx_intf_prop *ipv6_property;
-	struct ipa_ioc_rx_intf_prop rx_ioc_properties[2] = { {0}, {0} };
-	struct ipa_rx_intf rx_properties = {0};
-	struct ipa_ioc_rx_intf_prop *rx_ipv4_property;
-	struct ipa_ioc_rx_intf_prop *rx_ipv6_property;
-	enum ipa_hdr_l2_type hdr_l2_type = IPA_HDR_L2_ETHERNET_II;
-	int result = 0;
-
-	ECM_IPA_LOG_ENTRY();
-
-	if (ecm_ipa_ctx->is_vlan_mode)
-		hdr_l2_type = IPA_HDR_L2_802_1Q;
-
-	tx_properties.prop = properties;
-	ipv4_property = &tx_properties.prop[0];
-	ipv4_property->ip = IPA_IP_v4;
-	ipv4_property->dst_pipe = ecm_ipa_ctx->ipa_to_usb_client;
-	strlcpy
-		(ipv4_property->hdr_name, ECM_IPA_IPV4_HDR_NAME,
-		IPA_RESOURCE_NAME_MAX);
-	ipv4_property->hdr_l2_type = hdr_l2_type;
-	ipv6_property = &tx_properties.prop[1];
-	ipv6_property->ip = IPA_IP_v6;
-	ipv6_property->dst_pipe = ecm_ipa_ctx->ipa_to_usb_client;
-	ipv6_property->hdr_l2_type = hdr_l2_type;
-	strlcpy
-		(ipv6_property->hdr_name, ECM_IPA_IPV6_HDR_NAME,
-		IPA_RESOURCE_NAME_MAX);
-	tx_properties.num_props = 2;
-
-	rx_properties.prop = rx_ioc_properties;
-	rx_ipv4_property = &rx_properties.prop[0];
-	rx_ipv4_property->ip = IPA_IP_v4;
-	rx_ipv4_property->attrib.attrib_mask = 0;
-	rx_ipv4_property->src_pipe = ecm_ipa_ctx->usb_to_ipa_client;
-	rx_ipv4_property->hdr_l2_type = hdr_l2_type;
-	rx_ipv6_property = &rx_properties.prop[1];
-	rx_ipv6_property->ip = IPA_IP_v6;
-	rx_ipv6_property->attrib.attrib_mask = 0;
-	rx_ipv6_property->src_pipe = ecm_ipa_ctx->usb_to_ipa_client;
-	rx_ipv6_property->hdr_l2_type = hdr_l2_type;
-	rx_properties.num_props = 2;
-
-	result = ipa_register_intf("ecm0", &tx_properties, &rx_properties);
-	if (result)
-		ECM_IPA_ERROR("fail on Tx/Rx properties registration\n");
-
-	ECM_IPA_LOG_EXIT();
-
-	return result;
-}
-
-static void ecm_ipa_deregister_properties(void)
-{
-	int result;
-
-	ECM_IPA_LOG_ENTRY();
-	result = ipa_deregister_intf("ecm0");
-	if (result)
-		ECM_IPA_DEBUG("Fail on Tx prop deregister\n");
-	ECM_IPA_LOG_EXIT();
-}
-
-/**
- * ecm_ipa_configure() - make IPA core end-point specific configuration
- * @usb_to_ipa_hdl: handle of usb_to_ipa end-point for IPA driver
- * @ipa_to_usb_hdl: handle of ipa_to_usb end-point for IPA driver
- * @host_ethaddr: host Ethernet address in network order
- * @device_ethaddr: device Ethernet address in network order
- *
- * Configure the usb_to_ipa and ipa_to_usb end-point registers
- * - USB->IPA end-point: disable de-aggregation, enable link layer
- *   header removal (Ethernet removal), source NATing and default routing.
- * - IPA->USB end-point: disable aggregation, add link layer header (Ethernet)
- * - allocate Ethernet device
- * - register to Linux network stack
- *
- * Returns negative errno, or zero on success
- */
-
-static struct net_device_stats *ecm_ipa_get_stats(struct net_device *net)
-{
-	return &net->stats;
-}
-
-static void ecm_ipa_pm_cb(void *p, enum ipa_pm_cb_event event)
-{
-	struct ecm_ipa_dev *ecm_ipa_ctx = p;
-
-	ECM_IPA_LOG_ENTRY();
-	if (event != IPA_PM_CLIENT_ACTIVATED) {
-		ECM_IPA_ERROR("unexpected event %d\n", event);
-		WARN_ON(1);
-		return;
-	}
-
-	if (netif_queue_stopped(ecm_ipa_ctx->net)) {
-		ECM_IPA_DEBUG("Resource Granted - starting queue\n");
-		netif_start_queue(ecm_ipa_ctx->net);
-	}
-	ECM_IPA_LOG_EXIT();
-}
-
-static int ecm_ipa_register_pm_client(struct ecm_ipa_dev *ecm_ipa_ctx)
-{
-	int result;
-	struct ipa_pm_register_params pm_reg;
-
-	memset(&pm_reg, 0, sizeof(pm_reg));
-	pm_reg.name = ecm_ipa_ctx->net->name;
-	pm_reg.user_data = ecm_ipa_ctx;
-	pm_reg.callback = ecm_ipa_pm_cb;
-	pm_reg.group = IPA_PM_GROUP_APPS;
-	result = ipa_pm_register(&pm_reg, &ecm_ipa_ctx->pm_hdl);
-	if (result) {
-		ECM_IPA_ERROR("failed to create IPA PM client %d\n", result);
-		return result;
-	}
-	return 0;
-}
-
-static void ecm_ipa_deregister_pm_client(struct ecm_ipa_dev *ecm_ipa_ctx)
-{
-	ipa_pm_deactivate_sync(ecm_ipa_ctx->pm_hdl);
-	ipa_pm_deregister(ecm_ipa_ctx->pm_hdl);
-	ecm_ipa_ctx->pm_hdl = ~0;
-}
-
-
-/**
- * ecm_ipa_tx_complete_notify() - Rx notify
- *
- * @priv: ecm driver context
- * @evt: event type
- * @data: data provided with event
- *
- * Check that the packet is the one we sent and release it
- * This function will be called in defered context in IPA wq.
- */
-static void ecm_ipa_tx_complete_notify
-		(void *priv,
-		enum ipa_dp_evt_type evt,
-		unsigned long data)
-{
-	struct sk_buff *skb = (struct sk_buff *)data;
-	struct ecm_ipa_dev *ecm_ipa_ctx = priv;
-
-	if (!skb) {
-		ECM_IPA_ERROR("Bad SKB received from IPA driver\n");
-		return;
-	}
-
-	if (!ecm_ipa_ctx) {
-		ECM_IPA_ERROR("ecm_ipa_ctx is NULL pointer\n");
-		return;
-	}
-
-	ECM_IPA_DEBUG
-		("Tx-complete, len=%d, skb->prot=%d, outstanding=%d\n",
-		skb->len, skb->protocol,
-		atomic_read(&ecm_ipa_ctx->outstanding_pkts));
-
-	if (evt != IPA_WRITE_DONE) {
-		ECM_IPA_ERROR("unsupported event on Tx callback\n");
-		return;
-	}
-
-	if (unlikely(ecm_ipa_ctx->state != ECM_IPA_CONNECTED_AND_UP)) {
-		ECM_IPA_DEBUG
-			("dropping Tx-complete pkt, state=%s",
-			ecm_ipa_state_string(ecm_ipa_ctx->state));
-		goto out;
-	}
-
-	ecm_ipa_ctx->net->stats.tx_packets++;
-	ecm_ipa_ctx->net->stats.tx_bytes += skb->len;
-
-	if (atomic_read(&ecm_ipa_ctx->outstanding_pkts) > 0)
-		atomic_dec(&ecm_ipa_ctx->outstanding_pkts);
-
-	if
-		(netif_queue_stopped(ecm_ipa_ctx->net) &&
-		netif_carrier_ok(ecm_ipa_ctx->net) &&
-		atomic_read(&ecm_ipa_ctx->outstanding_pkts)
-		< (ecm_ipa_ctx->outstanding_low)) {
-		ECM_IPA_DEBUG
-			("outstanding low (%d) - waking up queue\n",
-			ecm_ipa_ctx->outstanding_low);
-		netif_wake_queue(ecm_ipa_ctx->net);
-	}
-
-out:
-	dev_kfree_skb_any(skb);
-}
-
-static void ecm_ipa_tx_timeout(struct net_device *net)
-{
-	struct ecm_ipa_dev *ecm_ipa_ctx = netdev_priv(net);
-
-	ECM_IPA_ERROR
-		("possible IPA stall was detected, %d outstanding",
-		atomic_read(&ecm_ipa_ctx->outstanding_pkts));
-
-	net->stats.tx_errors++;
-}
-
-static int ecm_ipa_debugfs_atomic_open(struct inode *inode, struct file *file)
-{
-	struct ecm_ipa_dev *ecm_ipa_ctx = inode->i_private;
-
-	ECM_IPA_LOG_ENTRY();
-	file->private_data = &ecm_ipa_ctx->outstanding_pkts;
-	ECM_IPA_LOG_EXIT();
-	return 0;
-}
-
-static ssize_t ecm_ipa_debugfs_atomic_read
-	(struct file *file, char __user *ubuf, size_t count, loff_t *ppos)
-{
-	int nbytes;
-	u8 atomic_str[DEBUGFS_TEMP_BUF_SIZE] = {0};
-	atomic_t *atomic_var = file->private_data;
-
-	nbytes = scnprintf
-		(atomic_str, sizeof(atomic_str), "%d\n",
-			atomic_read(atomic_var));
-	return simple_read_from_buffer(ubuf, count, ppos, atomic_str, nbytes);
-}
-
-#ifdef CONFIG_DEBUG_FS
-
-static void ecm_ipa_debugfs_init(struct ecm_ipa_dev *ecm_ipa_ctx)
-{
-	const mode_t flags_read_write = 0666;
-	const mode_t flags_read_only = 0444;
-	struct dentry *file;
-
-	ECM_IPA_LOG_ENTRY();
-
-	if (!ecm_ipa_ctx)
-		return;
-
-	ecm_ipa_ctx->directory = debugfs_create_dir("ecm_ipa", NULL);
-	if (!ecm_ipa_ctx->directory) {
-		ECM_IPA_ERROR("could not create debugfs directory entry\n");
-		goto fail_directory;
-	}
-	file = debugfs_create_u8
-		("outstanding_high", flags_read_write,
-		ecm_ipa_ctx->directory, &ecm_ipa_ctx->outstanding_high);
-	if (!file) {
-		ECM_IPA_ERROR("could not create outstanding_high file\n");
-		goto fail_file;
-	}
-	file = debugfs_create_u8
-		("outstanding_low", flags_read_write,
-		ecm_ipa_ctx->directory, &ecm_ipa_ctx->outstanding_low);
-	if (!file) {
-		ECM_IPA_ERROR("could not create outstanding_low file\n");
-		goto fail_file;
-	}
-	file = debugfs_create_file
-		("outstanding", flags_read_only,
-		ecm_ipa_ctx->directory,
-		ecm_ipa_ctx, &ecm_ipa_debugfs_atomic_ops);
-	if (!file) {
-		ECM_IPA_ERROR("could not create outstanding file\n");
-		goto fail_file;
-	}
-
-	file = debugfs_create_bool("is_vlan_mode", flags_read_only,
-		ecm_ipa_ctx->directory, &ecm_ipa_ctx->is_vlan_mode);
-	if (!file) {
-		ECM_IPA_ERROR("could not create is_vlan_mode file\n");
-		goto fail_file;
-	}
-
-	ECM_IPA_DEBUG("debugfs entries were created\n");
-	ECM_IPA_LOG_EXIT();
-
-	return;
-fail_file:
-	debugfs_remove_recursive(ecm_ipa_ctx->directory);
-fail_directory:
-	return;
-}
-
-static void ecm_ipa_debugfs_destroy(struct ecm_ipa_dev *ecm_ipa_ctx)
-{
-	debugfs_remove_recursive(ecm_ipa_ctx->directory);
-}
-
-#else /* !CONFIG_DEBUG_FS*/
-
-static void ecm_ipa_debugfs_init(struct ecm_ipa_dev *ecm_ipa_ctx) {}
-
-static void ecm_ipa_debugfs_destroy(struct ecm_ipa_dev *ecm_ipa_ctx) {}
-
-#endif /* CONFIG_DEBUG_FS */
-
-/**
- * ecm_ipa_ep_cfg() - configure the USB endpoints for ECM
- *
- * @usb_to_ipa_hdl: handle received from ipa_connect
- * @ipa_to_usb_hdl: handle received from ipa_connect
- * @is_vlan_mode - should driver work in vlan mode?
- *
- * USB to IPA pipe:
- *  - No de-aggregation
- *  - Remove Ethernet header
- *  - SRC NAT
- *  - Default routing(0)
- * IPA to USB Pipe:
- *  - No aggregation
- *  - Add Ethernet header
- */
-static int ecm_ipa_ep_registers_cfg(u32 usb_to_ipa_hdl, u32 ipa_to_usb_hdl,
-	bool is_vlan_mode)
-{
-	int result = 0;
-	struct ipa_ep_cfg usb_to_ipa_ep_cfg;
-	struct ipa_ep_cfg ipa_to_usb_ep_cfg;
-	uint8_t hdr_add = 0;
-
-
-	ECM_IPA_LOG_ENTRY();
-	if (is_vlan_mode)
-		hdr_add = VLAN_HLEN;
-	memset(&usb_to_ipa_ep_cfg, 0, sizeof(struct ipa_ep_cfg));
-	usb_to_ipa_ep_cfg.aggr.aggr_en = IPA_BYPASS_AGGR;
-	usb_to_ipa_ep_cfg.hdr.hdr_len = ETH_HLEN + hdr_add;
-	usb_to_ipa_ep_cfg.nat.nat_en = IPA_SRC_NAT;
-	usb_to_ipa_ep_cfg.route.rt_tbl_hdl = 0;
-	usb_to_ipa_ep_cfg.mode.dst = IPA_CLIENT_A5_LAN_WAN_CONS;
-	usb_to_ipa_ep_cfg.mode.mode = IPA_BASIC;
-
-	/* enable hdr_metadata_reg_valid */
-	usb_to_ipa_ep_cfg.hdr.hdr_metadata_reg_valid = true;
-
-	result = ipa_cfg_ep(usb_to_ipa_hdl, &usb_to_ipa_ep_cfg);
-	if (result) {
-		ECM_IPA_ERROR("failed to configure USB to IPA point\n");
-		goto out;
-	}
-	memset(&ipa_to_usb_ep_cfg, 0, sizeof(struct ipa_ep_cfg));
-	ipa_to_usb_ep_cfg.aggr.aggr_en = IPA_BYPASS_AGGR;
-	ipa_to_usb_ep_cfg.hdr.hdr_len = ETH_HLEN + hdr_add;
-	ipa_to_usb_ep_cfg.nat.nat_en = IPA_BYPASS_NAT;
-	result = ipa_cfg_ep(ipa_to_usb_hdl, &ipa_to_usb_ep_cfg);
-	if (result) {
-		ECM_IPA_ERROR("failed to configure IPA to USB end-point\n");
-		goto out;
-	}
-	ECM_IPA_DEBUG("end-point registers successfully configured\n");
-out:
-	ECM_IPA_LOG_EXIT();
-	return result;
-}
-
-/**
- * ecm_ipa_set_device_ethernet_addr() - set device etherenet address
- * @dev_ethaddr: device etherenet address
- *
- * Returns 0 for success, negative otherwise
- */
-static int ecm_ipa_set_device_ethernet_addr
-	(u8 *dev_ethaddr, u8 device_ethaddr[])
-{
-	if (!is_valid_ether_addr(device_ethaddr))
-		return -EINVAL;
-	memcpy(dev_ethaddr, device_ethaddr, ETH_ALEN);
-	ECM_IPA_DEBUG("device ethernet address: %pM\n", dev_ethaddr);
-	return 0;
-}
-
-/** ecm_ipa_next_state - return the next state of the driver
- * @current_state: the current state of the driver
- * @operation: an enum which represent the operation being made on the driver
- *  by its API.
- *
- * This function implements the driver internal state machine.
- * Its decisions are based on the driver current state and the operation
- * being made.
- * In case the operation is invalid this state machine will return
- * the value ECM_IPA_INVALID to inform the caller for a forbidden sequence.
- */
-static enum ecm_ipa_state ecm_ipa_next_state
-	(enum ecm_ipa_state current_state, enum ecm_ipa_operation operation)
-{
-	int next_state = ECM_IPA_INVALID;
-
-	switch (current_state) {
-	case ECM_IPA_UNLOADED:
-		if (operation == ECM_IPA_INITIALIZE)
-			next_state = ECM_IPA_INITIALIZED;
-		break;
-	case ECM_IPA_INITIALIZED:
-		if (operation == ECM_IPA_CONNECT)
-			next_state = ECM_IPA_CONNECTED;
-		else if (operation == ECM_IPA_OPEN)
-			next_state = ECM_IPA_UP;
-		else if (operation == ECM_IPA_CLEANUP)
-			next_state = ECM_IPA_UNLOADED;
-		break;
-	case ECM_IPA_CONNECTED:
-		if (operation == ECM_IPA_DISCONNECT)
-			next_state = ECM_IPA_INITIALIZED;
-		else if (operation == ECM_IPA_OPEN)
-			next_state = ECM_IPA_CONNECTED_AND_UP;
-		break;
-	case ECM_IPA_UP:
-		if (operation == ECM_IPA_STOP)
-			next_state = ECM_IPA_INITIALIZED;
-		else if (operation == ECM_IPA_CONNECT)
-			next_state = ECM_IPA_CONNECTED_AND_UP;
-		else if (operation == ECM_IPA_CLEANUP)
-			next_state = ECM_IPA_UNLOADED;
-		break;
-	case ECM_IPA_CONNECTED_AND_UP:
-		if (operation == ECM_IPA_STOP)
-			next_state = ECM_IPA_CONNECTED;
-		else if (operation == ECM_IPA_DISCONNECT)
-			next_state = ECM_IPA_UP;
-		break;
-	default:
-		ECM_IPA_ERROR("State is not supported\n");
-		break;
-	}
-
-	ECM_IPA_DEBUG
-		("state transition ( %s -> %s )- %s\n",
-		ecm_ipa_state_string(current_state),
-		ecm_ipa_state_string(next_state),
-		next_state == ECM_IPA_INVALID ? "Forbidden" : "Allowed");
-
-	return next_state;
-}
-
-/**
- * ecm_ipa_state_string - return the state string representation
- * @state: enum which describe the state
- */
-static const char *ecm_ipa_state_string(enum ecm_ipa_state state)
-{
-	switch (state) {
-	case ECM_IPA_UNLOADED:
-		return "ECM_IPA_UNLOADED";
-	case ECM_IPA_INITIALIZED:
-		return "ECM_IPA_INITIALIZED";
-	case ECM_IPA_CONNECTED:
-		return "ECM_IPA_CONNECTED";
-	case ECM_IPA_UP:
-		return "ECM_IPA_UP";
-	case ECM_IPA_CONNECTED_AND_UP:
-		return "ECM_IPA_CONNECTED_AND_UP";
-	default:
-		return "Not supported";
-	}
-}
-
-/**
- * ecm_ipa_init_module() - module initialization
- *
- */
-static int ecm_ipa_init_module(void)
-{
-	ECM_IPA_LOG_ENTRY();
-	ipa_ecm_logbuf = ipc_log_context_create(IPA_ECM_IPC_LOG_PAGES,
-			"ipa_ecm", 0);
-	if (ipa_ecm_logbuf == NULL)
-		ECM_IPA_DEBUG("failed to create IPC log, continue...\n");
-	ECM_IPA_LOG_EXIT();
-	return 0;
-}
-
-/**
- * ecm_ipa_cleanup_module() - module cleanup
- *
- */
-static void ecm_ipa_cleanup_module(void)
-{
-	ECM_IPA_LOG_ENTRY();
-	if (ipa_ecm_logbuf)
-		ipc_log_context_destroy(ipa_ecm_logbuf);
-	ipa_ecm_logbuf = NULL;
-	ECM_IPA_LOG_EXIT();
-}
-
-MODULE_LICENSE("GPL v2");
-MODULE_DESCRIPTION("ECM IPA network interface");
-
-late_initcall(ecm_ipa_init_module);
-module_exit(ecm_ipa_cleanup_module);

ipa/ipa_clients/ipa_gsb.c

@@ -1,1228 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/debugfs.h>
-#include <linux/export.h>
-#include <linux/fs.h>
-#include <linux/if_ether.h>
-#include <linux/ioctl.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/msm_ipa.h>
-#include <linux/mutex.h>
-#include <linux/skbuff.h>
-#include <linux/types.h>
-#include <linux/ipv6.h>
-#include <net/addrconf.h>
-#include <linux/ipa.h>
-#include <linux/cdev.h>
-#include <linux/ipa_odu_bridge.h>
-#include "../ipa_common_i.h"
-#ifdef CONFIG_IPA3
-#include "../ipa_v3/ipa_pm.h"
-#endif
-
-#define IPA_GSB_DRV_NAME "ipa_gsb"
-
-#define MAX_SUPPORTED_IFACE 5
-
-#define IPA_GSB_DBG(fmt, args...) \
-	do { \
-		pr_debug(IPA_GSB_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-			IPA_GSB_DRV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			IPA_GSB_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-#define IPA_GSB_DBG_LOW(fmt, args...) \
-	do { \
-		pr_debug(IPA_GSB_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			IPA_GSB_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-#define IPA_GSB_ERR(fmt, args...) \
-	do { \
-		pr_err(IPA_GSB_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-			IPA_GSB_DRV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			IPA_GSB_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-#define IPA_GSB_MAX_MSG_LEN 512
-static char dbg_buff[IPA_GSB_MAX_MSG_LEN];
-
-#define IPA_GSB_SKB_HEADROOM 256
-#define IPA_GSB_SKB_DUMMY_HEADER 42
-#define IPA_GSB_AGGR_BYTE_LIMIT 14
-#define IPA_GSB_AGGR_TIME_LIMIT 1000 /* 1000 us */
-
-static struct dentry *dent;
-static struct dentry *dfile_stats;
-
-/**
- * struct stats - driver statistics,
- * @num_ul_packets: number of uplink packets
- * @num_dl_packets: number of downlink packets
- * @num_insufficient_headroom_packets: number of
-	packets with insufficient headroom
- */
-struct stats {
-	u64 num_ul_packets;
-	u64 num_dl_packets;
-	u64 num_insufficient_headroom_packets;
-};
-
-/**
- * struct ipa_gsb_mux_hdr - ipa gsb mux header,
- * @iface_hdl: interface handle
- * @qmap_id: qmap id
- * @pkt_size: packet size
- */
-struct ipa_gsb_mux_hdr {
-	u8 iface_hdl;
-	u8 qmap_id;
-	u16 pkt_size;
-};
-
-/**
- * struct ipa_gsb_iface_info - GSB interface information
- * @netdev_name: network interface name
- * @device_ethaddr: network interface ethernet address
- * @priv: client's private data. to be used in client's callbacks
- * @tx_dp_notify: client callback for handling IPA ODU_PROD callback
- * @send_dl_skb: client callback for sending skb in downlink direction
- * @iface_stats: statistics, how many packets were transmitted
- * using the SW bridge.
- * @partial_hdr_hdl: handle for partial header
- * @wakeup_request: client callback to wakeup
- * @is_conencted: is interface connected ?
- * @is_resumed: is interface resumed ?
- * @iface_hdl: interface handle
- */
-struct ipa_gsb_iface_info {
-	char netdev_name[IPA_RESOURCE_NAME_MAX];
-	u8 device_ethaddr[ETH_ALEN];
-	void *priv;
-	ipa_notify_cb tx_dp_notify;
-	int (*send_dl_skb)(void *priv, struct sk_buff *skb);
-	struct stats iface_stats;
-	uint32_t partial_hdr_hdl[IPA_IP_MAX];
-	void (*wakeup_request)(void *cl_priv);
-	bool is_connected;
-	bool is_resumed;
-	u8 iface_hdl;
-};
-
-/**
- * struct ipa_gsb_context - GSB driver context information
- * @logbuf: buffer of ipc logging
- * @logbuf_low: buffer of ipc logging (low priority)
- * @lock: global mutex lock for global variables
- * @prod_hdl: handle for prod pipe
- * @cons_hdl: handle for cons pipe
- * @ipa_sys_desc_size: sys pipe desc size
- * @num_iface: number of interface
- * @iface_hdl: interface handles
- * @num_connected_iface: number of connected interface
- * @num_resumed_iface: number of resumed interface
- * @iface: interface information
- * @iface_lock: interface mutex lock for control path
- * @iface_spinlock: interface spinlock for data path
- * @pm_hdl: IPA PM handle
- */
-struct ipa_gsb_context {
-	void *logbuf;
-	void *logbuf_low;
-	struct mutex lock;
-	u32 prod_hdl;
-	u32 cons_hdl;
-	u32 ipa_sys_desc_size;
-	int num_iface;
-	bool iface_hdl[MAX_SUPPORTED_IFACE];
-	int num_connected_iface;
-	int num_resumed_iface;
-	struct ipa_gsb_iface_info *iface[MAX_SUPPORTED_IFACE];
-	struct mutex iface_lock[MAX_SUPPORTED_IFACE];
-	spinlock_t iface_spinlock[MAX_SUPPORTED_IFACE];
-	u32 pm_hdl;
-	atomic_t disconnect_in_progress;
-	atomic_t suspend_in_progress;
-};
-
-static struct ipa_gsb_context *ipa_gsb_ctx;
-
-#ifdef CONFIG_DEBUG_FS
-static ssize_t ipa_gsb_debugfs_stats(struct file *file,
-				  char __user *ubuf,
-				  size_t count,
-				  loff_t *ppos)
-{
-	int i, nbytes = 0;
-	struct ipa_gsb_iface_info *iface = NULL;
-	struct stats iface_stats;
-
-	for (i = 0; i < MAX_SUPPORTED_IFACE; i++) {
-		iface = ipa_gsb_ctx->iface[i];
-		if (iface != NULL) {
-			iface_stats = iface->iface_stats;
-			nbytes += scnprintf(&dbg_buff[nbytes],
-				IPA_GSB_MAX_MSG_LEN - nbytes,
-				"netdev: %s\n",
-				iface->netdev_name);
-
-			nbytes += scnprintf(&dbg_buff[nbytes],
-				IPA_GSB_MAX_MSG_LEN - nbytes,
-				"UL packets: %lld\n",
-				iface_stats.num_ul_packets);
-
-			nbytes += scnprintf(&dbg_buff[nbytes],
-				IPA_GSB_MAX_MSG_LEN - nbytes,
-				"DL packets: %lld\n",
-				iface_stats.num_dl_packets);
-
-			nbytes += scnprintf(&dbg_buff[nbytes],
-				IPA_GSB_MAX_MSG_LEN - nbytes,
-				"packets with insufficient headroom: %lld\n",
-				iface_stats.num_insufficient_headroom_packets);
-		}
-	}
-	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, nbytes);
-}
-
-static const struct file_operations ipa_gsb_stats_ops = {
-	.read = ipa_gsb_debugfs_stats,
-};
-
-static void ipa_gsb_debugfs_init(void)
-{
-	const mode_t read_only_mode = 00444;
-
-	dent = debugfs_create_dir("ipa_gsb", NULL);
-	if (IS_ERR(dent)) {
-		IPA_GSB_ERR("fail to create folder ipa_gsb\n");
-		return;
-	}
-
-	dfile_stats =
-		debugfs_create_file("stats", read_only_mode, dent,
-					NULL, &ipa_gsb_stats_ops);
-	if (!dfile_stats || IS_ERR(dfile_stats)) {
-		IPA_GSB_ERR("fail to create file stats\n");
-		goto fail;
-	}
-
-	return;
-
-fail:
-	debugfs_remove_recursive(dent);
-}
-
-static void ipa_gsb_debugfs_destroy(void)
-{
-	debugfs_remove_recursive(dent);
-}
-#else
-static void ipa_gsb_debugfs_init(void)
-{
-}
-
-static void ipa_gsb_debugfs_destroy(void)
-{
-}
-#endif
-
-static int ipa_gsb_driver_init(struct odu_bridge_params *params)
-{
-	int i;
-
-	if (!ipa_is_ready()) {
-		IPA_GSB_ERR("IPA is not ready\n");
-		return -EFAULT;
-	}
-
-	ipa_gsb_ctx = kzalloc(sizeof(*ipa_gsb_ctx),
-		GFP_KERNEL);
-
-	if (!ipa_gsb_ctx)
-		return -ENOMEM;
-
-	mutex_init(&ipa_gsb_ctx->lock);
-	for (i = 0; i < MAX_SUPPORTED_IFACE; i++) {
-		mutex_init(&ipa_gsb_ctx->iface_lock[i]);
-		spin_lock_init(&ipa_gsb_ctx->iface_spinlock[i]);
-	}
-	ipa_gsb_debugfs_init();
-
-	return 0;
-}
-
-static int ipa_gsb_commit_partial_hdr(struct ipa_gsb_iface_info *iface_info)
-{
-	int i;
-	struct ipa_ioc_add_hdr *hdr;
-
-	if (!iface_info) {
-		IPA_GSB_ERR("invalid input\n");
-		return -EINVAL;
-	}
-
-	hdr = kzalloc(sizeof(struct ipa_ioc_add_hdr) +
-		2 * sizeof(struct ipa_hdr_add), GFP_KERNEL);
-	if (!hdr)
-		return -ENOMEM;
-
-	hdr->commit = 1;
-	hdr->num_hdrs = 2;
-
-	snprintf(hdr->hdr[0].name, sizeof(hdr->hdr[0].name),
-			 "%s_ipv4", iface_info->netdev_name);
-	snprintf(hdr->hdr[1].name, sizeof(hdr->hdr[1].name),
-			 "%s_ipv6", iface_info->netdev_name);
-	/*
-	 * partial header:
-	 * [hdl][QMAP ID][pkt size][Dummy Header][ETH header]
-	 */
-	for (i = IPA_IP_v4; i < IPA_IP_MAX; i++) {
-		/*
-		 * Optimization: add dummy header to reserve space
-		 * for rndis header, so we can do the skb_clone
-		 * instead of deep copy.
-		 */
-		hdr->hdr[i].hdr_len = ETH_HLEN +
-			sizeof(struct ipa_gsb_mux_hdr) +
-			IPA_GSB_SKB_DUMMY_HEADER;
-		hdr->hdr[i].type = IPA_HDR_L2_ETHERNET_II;
-		hdr->hdr[i].is_partial = 1;
-		hdr->hdr[i].is_eth2_ofst_valid = 1;
-		hdr->hdr[i].eth2_ofst = sizeof(struct ipa_gsb_mux_hdr) +
-			IPA_GSB_SKB_DUMMY_HEADER;
-		/* populate iface handle */
-		hdr->hdr[i].hdr[0] = iface_info->iface_hdl;
-		/* populate src ETH address */
-		memcpy(&hdr->hdr[i].hdr[10 + IPA_GSB_SKB_DUMMY_HEADER],
-			iface_info->device_ethaddr, 6);
-		/* populate Ethertype */
-		if (i == IPA_IP_v4)
-			*(u16 *)(hdr->hdr[i].hdr + 16 +
-				IPA_GSB_SKB_DUMMY_HEADER) = htons(ETH_P_IP);
-		else
-			*(u16 *)(hdr->hdr[i].hdr + 16 +
-				IPA_GSB_SKB_DUMMY_HEADER) = htons(ETH_P_IPV6);
-	}
-
-	if (ipa_add_hdr(hdr)) {
-		IPA_GSB_ERR("fail to add partial headers\n");
-		kfree(hdr);
-		return -EFAULT;
-	}
-
-	for (i = IPA_IP_v4; i < IPA_IP_MAX; i++)
-		iface_info->partial_hdr_hdl[i] =
-			hdr->hdr[i].hdr_hdl;
-
-	IPA_GSB_DBG("added partial hdr hdl for ipv4: %d\n",
-		iface_info->partial_hdr_hdl[IPA_IP_v4]);
-	IPA_GSB_DBG("added partial hdr hdl for ipv6: %d\n",
-		iface_info->partial_hdr_hdl[IPA_IP_v6]);
-
-	kfree(hdr);
-	return 0;
-}
-
-static void ipa_gsb_delete_partial_hdr(struct ipa_gsb_iface_info *iface_info)
-{
-	struct ipa_ioc_del_hdr *del_hdr;
-
-	del_hdr = kzalloc(sizeof(struct ipa_ioc_del_hdr) +
-		2 * sizeof(struct ipa_hdr_del), GFP_KERNEL);
-	if (!del_hdr)
-		return;
-
-	del_hdr->commit = 1;
-	del_hdr->num_hdls = 2;
-	del_hdr->hdl[IPA_IP_v4].hdl = iface_info->partial_hdr_hdl[IPA_IP_v4];
-	del_hdr->hdl[IPA_IP_v6].hdl = iface_info->partial_hdr_hdl[IPA_IP_v6];
-
-	if (ipa_del_hdr(del_hdr) != 0)
-		IPA_GSB_ERR("failed to delete partial hdr\n");
-
-	IPA_GSB_DBG("deleted partial hdr hdl for ipv4: %d\n",
-		iface_info->partial_hdr_hdl[IPA_IP_v4]);
-	IPA_GSB_DBG("deleted partial hdr hdl for ipv6: %d\n",
-		iface_info->partial_hdr_hdl[IPA_IP_v6]);
-
-	kfree(del_hdr);
-}
-
-static int ipa_gsb_reg_intf_props(struct ipa_gsb_iface_info *iface_info)
-{
-	struct ipa_tx_intf tx;
-	struct ipa_rx_intf rx;
-	struct ipa_ioc_tx_intf_prop tx_prop[2];
-	struct ipa_ioc_rx_intf_prop rx_prop[2];
-
-	/* populate tx prop */
-	tx.num_props = 2;
-	tx.prop = tx_prop;
-
-	memset(tx_prop, 0, sizeof(tx_prop));
-	tx_prop[0].ip = IPA_IP_v4;
-	tx_prop[0].dst_pipe = IPA_CLIENT_ODU_EMB_CONS;
-	tx_prop[0].hdr_l2_type = IPA_HDR_L2_ETHERNET_II;
-	snprintf(tx_prop[0].hdr_name, sizeof(tx_prop[0].hdr_name),
-			 "%s_ipv4", iface_info->netdev_name);
-
-	tx_prop[1].ip = IPA_IP_v6;
-	tx_prop[1].dst_pipe = IPA_CLIENT_ODU_EMB_CONS;
-	tx_prop[1].hdr_l2_type = IPA_HDR_L2_ETHERNET_II;
-	snprintf(tx_prop[1].hdr_name, sizeof(tx_prop[1].hdr_name),
-			 "%s_ipv6", iface_info->netdev_name);
-
-	/* populate rx prop */
-	rx.num_props = 2;
-	rx.prop = rx_prop;
-
-	memset(rx_prop, 0, sizeof(rx_prop));
-	rx_prop[0].ip = IPA_IP_v4;
-	rx_prop[0].src_pipe = IPA_CLIENT_ODU_PROD;
-	rx_prop[0].hdr_l2_type = IPA_HDR_L2_ETHERNET_II;
-	rx_prop[0].attrib.attrib_mask |= IPA_FLT_META_DATA;
-	rx_prop[0].attrib.meta_data = iface_info->iface_hdl;
-	rx_prop[0].attrib.meta_data_mask = 0xFF;
-
-	rx_prop[1].ip = IPA_IP_v6;
-	rx_prop[1].src_pipe = IPA_CLIENT_ODU_PROD;
-	rx_prop[1].hdr_l2_type = IPA_HDR_L2_ETHERNET_II;
-	rx_prop[1].attrib.attrib_mask |= IPA_FLT_META_DATA;
-	rx_prop[1].attrib.meta_data = iface_info->iface_hdl;
-	rx_prop[1].attrib.meta_data_mask = 0xFF;
-
-	if (ipa_register_intf(iface_info->netdev_name, &tx, &rx)) {
-		IPA_GSB_ERR("fail to add interface prop\n");
-		return -EFAULT;
-	}
-
-	return 0;
-}
-
-static void ipa_gsb_dereg_intf_props(struct ipa_gsb_iface_info *iface_info)
-{
-	if (ipa_deregister_intf(iface_info->netdev_name) != 0)
-		IPA_GSB_ERR("fail to dereg intf props\n");
-
-	IPA_GSB_DBG("deregistered iface props for %s\n",
-		iface_info->netdev_name);
-}
-
-static void ipa_gsb_pm_cb(void *user_data, enum ipa_pm_cb_event event)
-{
-	int i;
-
-	if (event != IPA_PM_REQUEST_WAKEUP) {
-		IPA_GSB_ERR("Unexpected event %d\n", event);
-		WARN_ON(1);
-		return;
-	}
-
-	IPA_GSB_DBG_LOW("wake up clients\n");
-	for (i = 0; i < MAX_SUPPORTED_IFACE; i++)
-		if (ipa_gsb_ctx->iface[i] != NULL)
-			ipa_gsb_ctx->iface[i]->wakeup_request(
-				ipa_gsb_ctx->iface[i]->priv);
-}
-
-static int ipa_gsb_register_pm(void)
-{
-	struct ipa_pm_register_params reg_params;
-	int ret;
-
-	memset(&reg_params, 0, sizeof(reg_params));
-	reg_params.name = "ipa_gsb";
-	reg_params.callback = ipa_gsb_pm_cb;
-	reg_params.user_data = NULL;
-	reg_params.group = IPA_PM_GROUP_DEFAULT;
-
-	ret = ipa_pm_register(&reg_params,
-		&ipa_gsb_ctx->pm_hdl);
-	if (ret) {
-		IPA_GSB_ERR("fail to register with PM %d\n", ret);
-		goto fail_pm_reg;
-	}
-	IPA_GSB_DBG("ipa pm hdl: %d\n", ipa_gsb_ctx->pm_hdl);
-
-	ret = ipa_pm_associate_ipa_cons_to_client(ipa_gsb_ctx->pm_hdl,
-		IPA_CLIENT_ODU_EMB_CONS);
-	if (ret) {
-		IPA_GSB_ERR("fail to associate cons with PM %d\n", ret);
-		goto fail_pm_cons;
-	}
-
-	return 0;
-
-fail_pm_cons:
-	ipa_pm_deregister(ipa_gsb_ctx->pm_hdl);
-	ipa_gsb_ctx->pm_hdl = ~0;
-fail_pm_reg:
-	return ret;
-}
-
-int ipa_bridge_init(struct ipa_bridge_init_params *params, u32 *hdl)
-{
-	int i, ret;
-	struct ipa_gsb_iface_info *new_intf;
-
-	if (!params || !params->wakeup_request || !hdl ||
-		!params->info.netdev_name || !params->info.tx_dp_notify ||
-		!params->info.send_dl_skb) {
-		IPA_GSB_ERR("Invalid parameters\n");
-		return -EINVAL;
-	}
-
-	IPA_GSB_DBG("netdev_name: %s\n", params->info.netdev_name);
-
-	if (ipa_gsb_ctx == NULL) {
-		ret = ipa_gsb_driver_init(&params->info);
-		if (ret) {
-			IPA_GSB_ERR("fail to init ipa gsb driver\n");
-			return -EFAULT;
-		}
-		ipa_gsb_ctx->ipa_sys_desc_size =
-			params->info.ipa_desc_size;
-		IPA_GSB_DBG("desc size: %d\n", ipa_gsb_ctx->ipa_sys_desc_size);
-	}
-
-	mutex_lock(&ipa_gsb_ctx->lock);
-
-	if (params->info.ipa_desc_size != ipa_gsb_ctx->ipa_sys_desc_size) {
-		IPA_GSB_ERR("unmatch: orig desc size %d, new desc size %d\n",
-			ipa_gsb_ctx->ipa_sys_desc_size,
-			params->info.ipa_desc_size);
-		mutex_unlock(&ipa_gsb_ctx->lock);
-		return -EFAULT;
-	}
-
-	for (i = 0; i < MAX_SUPPORTED_IFACE; i++)
-		if (ipa_gsb_ctx->iface[i] != NULL &&
-			strnlen(ipa_gsb_ctx->iface[i]->netdev_name,
-					IPA_RESOURCE_NAME_MAX) ==
-			strnlen(params->info.netdev_name,
-					IPA_RESOURCE_NAME_MAX) &&
-			strcmp(ipa_gsb_ctx->iface[i]->netdev_name,
-				params->info.netdev_name) == 0) {
-			IPA_GSB_ERR("intf was added before.\n");
-			mutex_unlock(&ipa_gsb_ctx->lock);
-			return -EFAULT;
-		}
-
-	if (ipa_gsb_ctx->num_iface == MAX_SUPPORTED_IFACE) {
-		IPA_GSB_ERR("reached maximum supported interfaces");
-		mutex_unlock(&ipa_gsb_ctx->lock);
-		return -EFAULT;
-	}
-
-	for (i = 0; i < MAX_SUPPORTED_IFACE; i++)
-		if (!ipa_gsb_ctx->iface_hdl[i]) {
-			ipa_gsb_ctx->iface_hdl[i] = true;
-			*hdl = i;
-			IPA_GSB_DBG("iface hdl: %d\n", *hdl);
-			break;
-		}
-
-	IPA_GSB_DBG("intf was not added before, proceed.\n");
-	new_intf = kzalloc(sizeof(*new_intf), GFP_KERNEL);
-	if (new_intf == NULL) {
-		ret = -ENOMEM;
-		goto fail_alloc_mem;
-	}
-
-	strlcpy(new_intf->netdev_name, params->info.netdev_name,
-		sizeof(new_intf->netdev_name));
-	new_intf->wakeup_request = params->wakeup_request;
-	new_intf->priv = params->info.priv;
-	new_intf->tx_dp_notify = params->info.tx_dp_notify;
-	new_intf->send_dl_skb = params->info.send_dl_skb;
-	new_intf->iface_hdl = *hdl;
-	memcpy(new_intf->device_ethaddr, params->info.device_ethaddr,
-		sizeof(new_intf->device_ethaddr));
-
-	if (ipa_gsb_commit_partial_hdr(new_intf) != 0) {
-		IPA_GSB_ERR("fail to commit partial hdrs\n");
-		ret = -EFAULT;
-		goto fail_partial_hdr;
-	}
-
-	if (ipa_gsb_reg_intf_props(new_intf) != 0) {
-		IPA_GSB_ERR("fail to register interface props\n");
-		ret = -EFAULT;
-		goto fail_reg_intf_props;
-	}
-
-	if (ipa_gsb_ctx->num_iface == 0) {
-		ret = ipa_gsb_register_pm();
-		if (ret) {
-			IPA_GSB_ERR("fail to register with IPA PM %d\n", ret);
-			ret = -EFAULT;
-			goto fail_register_pm;
-		}
-	}
-
-	ipa_gsb_ctx->iface[*hdl] = new_intf;
-	ipa_gsb_ctx->num_iface++;
-	IPA_GSB_DBG("num_iface %d\n", ipa_gsb_ctx->num_iface);
-	mutex_unlock(&ipa_gsb_ctx->lock);
-	return 0;
-
-fail_register_pm:
-	ipa_gsb_dereg_intf_props(new_intf);
-fail_reg_intf_props:
-	ipa_gsb_delete_partial_hdr(new_intf);
-fail_partial_hdr:
-	kfree(new_intf);
-fail_alloc_mem:
-	ipa_gsb_ctx->iface_hdl[*hdl] = false;
-	mutex_unlock(&ipa_gsb_ctx->lock);
-	return ret;
-}
-EXPORT_SYMBOL(ipa_bridge_init);
-
-static void ipa_gsb_deregister_pm(void)
-{
-	IPA_GSB_DBG("deregister ipa pm hdl: %d\n", ipa_gsb_ctx->pm_hdl);
-	ipa_pm_deactivate_sync(ipa_gsb_ctx->pm_hdl);
-	ipa_pm_deregister(ipa_gsb_ctx->pm_hdl);
-	ipa_gsb_ctx->pm_hdl = ~0;
-}
-
-int ipa_bridge_cleanup(u32 hdl)
-{
-	int i;
-
-	if (!ipa_gsb_ctx) {
-		IPA_GSB_ERR("ipa_gsb_ctx was not initialized\n");
-		return -EFAULT;
-	}
-
-	if (hdl >= MAX_SUPPORTED_IFACE) {
-		IPA_GSB_ERR("invalid hdl: %d\n", hdl);
-		return -EINVAL;
-	}
-
-	mutex_lock(&ipa_gsb_ctx->iface_lock[hdl]);
-	if (!ipa_gsb_ctx->iface[hdl]) {
-		IPA_GSB_ERR("fail to find interface, hdl: %d\n", hdl);
-		mutex_unlock(&ipa_gsb_ctx->iface_lock[hdl]);
-		return -EFAULT;
-	}
-
-	IPA_GSB_DBG("client hdl: %d\n", hdl);
-
-	if (ipa_gsb_ctx->iface[hdl]->is_connected) {
-		IPA_GSB_ERR("cannot cleanup when iface is connected\n");
-		mutex_unlock(&ipa_gsb_ctx->iface_lock[hdl]);
-		return -EFAULT;
-	}
-	ipa_gsb_dereg_intf_props(ipa_gsb_ctx->iface[hdl]);
-	ipa_gsb_delete_partial_hdr(ipa_gsb_ctx->iface[hdl]);
-	spin_lock_bh(&ipa_gsb_ctx->iface_spinlock[hdl]);
-	kfree(ipa_gsb_ctx->iface[hdl]);
-	ipa_gsb_ctx->iface[hdl] = NULL;
-	ipa_gsb_ctx->iface_hdl[hdl] = false;
-	spin_unlock_bh(&ipa_gsb_ctx->iface_spinlock[hdl]);
-	mutex_unlock(&ipa_gsb_ctx->iface_lock[hdl]);
-	mutex_lock(&ipa_gsb_ctx->lock);
-	ipa_gsb_ctx->num_iface--;
-	IPA_GSB_DBG("num_iface %d\n", ipa_gsb_ctx->num_iface);
-	if (ipa_gsb_ctx->num_iface == 0) {
-		ipa_gsb_deregister_pm();
-		ipa_gsb_debugfs_destroy();
-		ipc_log_context_destroy(ipa_gsb_ctx->logbuf);
-		ipc_log_context_destroy(ipa_gsb_ctx->logbuf_low);
-		mutex_unlock(&ipa_gsb_ctx->lock);
-		mutex_destroy(&ipa_gsb_ctx->lock);
-		for (i = 0; i < MAX_SUPPORTED_IFACE; i++)
-			mutex_destroy(&ipa_gsb_ctx->iface_lock[i]);
-		kfree(ipa_gsb_ctx);
-		ipa_gsb_ctx = NULL;
-		return 0;
-	}
-	mutex_unlock(&ipa_gsb_ctx->lock);
-	return 0;
-}
-EXPORT_SYMBOL(ipa_bridge_cleanup);
-
-static void ipa_gsb_cons_cb(void *priv, enum ipa_dp_evt_type evt,
-	unsigned long data)
-{
-	struct sk_buff *skb;
-	struct sk_buff *skb2;
-	struct ipa_gsb_mux_hdr *mux_hdr;
-	u16 pkt_size, pad_byte;
-	u8 hdl;
-
-	if (evt != IPA_RECEIVE) {
-		IPA_GSB_ERR("unexpected event\n");
-		WARN_ON(1);
-		return;
-	}
-
-	skb = (struct sk_buff *)data;
-
-	if (skb == NULL) {
-		IPA_GSB_ERR("unexpected NULL data\n");
-		WARN_ON(1);
-		return;
-	}
-
-	while (skb->len) {
-		mux_hdr = (struct ipa_gsb_mux_hdr *)skb->data;
-		pkt_size = mux_hdr->pkt_size;
-		/* 4-byte padding */
-		pad_byte = ((pkt_size + sizeof(*mux_hdr) + ETH_HLEN +
-			3 + IPA_GSB_SKB_DUMMY_HEADER) & ~3) -
-			(pkt_size + sizeof(*mux_hdr) +
-			ETH_HLEN + IPA_GSB_SKB_DUMMY_HEADER);
-		hdl = mux_hdr->iface_hdl;
-		if (hdl >= MAX_SUPPORTED_IFACE) {
-			IPA_GSB_ERR("invalid hdl: %d\n", hdl);
-			break;
-		}
-		IPA_GSB_DBG_LOW("pkt_size: %d, pad_byte: %d, hdl: %d\n",
-			pkt_size, pad_byte, hdl);
-
-		/* remove 4 byte mux header AND dummy header*/
-		skb_pull(skb, sizeof(*mux_hdr) + IPA_GSB_SKB_DUMMY_HEADER);
-
-		skb2 = skb_clone(skb, GFP_KERNEL);
-		if (!skb2) {
-			IPA_GSB_ERR("skb_clone failed\n");
-			WARN_ON(1);
-			break;
-		}
-		skb_trim(skb2, pkt_size + ETH_HLEN);
-		spin_lock_bh(&ipa_gsb_ctx->iface_spinlock[hdl]);
-		if (ipa_gsb_ctx->iface[hdl] != NULL) {
-			ipa_gsb_ctx->iface[hdl]->send_dl_skb(
-				ipa_gsb_ctx->iface[hdl]->priv, skb2);
-			ipa_gsb_ctx->iface[hdl]->iface_stats.num_dl_packets++;
-			spin_unlock_bh(&ipa_gsb_ctx->iface_spinlock[hdl]);
-			skb_pull(skb, pkt_size + ETH_HLEN + pad_byte);
-		} else {
-			IPA_GSB_ERR("Invalid hdl: %d, drop the skb\n", hdl);
-			spin_unlock_bh(&ipa_gsb_ctx->iface_spinlock[hdl]);
-			dev_kfree_skb_any(skb2);
-			break;
-		}
-	}
-
-	if (skb) {
-		dev_kfree_skb_any(skb);
-		skb = NULL;
-	}
-}
-
-static void ipa_gsb_tx_dp_notify(void *priv, enum ipa_dp_evt_type evt,
-	unsigned long data)
-{
-	struct sk_buff *skb;
-	struct ipa_gsb_mux_hdr *mux_hdr;
-	u8 hdl;
-
-	skb = (struct sk_buff *)data;
-
-	if (skb == NULL) {
-		IPA_GSB_ERR("unexpected NULL data\n");
-		WARN_ON(1);
-		return;
-	}
-
-	if (evt != IPA_WRITE_DONE && evt != IPA_RECEIVE) {
-		IPA_GSB_ERR("unexpected event: %d\n", evt);
-		dev_kfree_skb_any(skb);
-		return;
-	}
-
-	/* fetch iface handle from header */
-	mux_hdr = (struct ipa_gsb_mux_hdr *)skb->data;
-	/* change to host order */
-	*(u32 *)mux_hdr = ntohl(*(u32 *)mux_hdr);
-	hdl = mux_hdr->iface_hdl;
-	if ((hdl < 0) || (hdl >= MAX_SUPPORTED_IFACE) ||
-		!ipa_gsb_ctx->iface[hdl]) {
-		IPA_GSB_ERR("invalid hdl: %d and cb, drop the skb\n", hdl);
-		dev_kfree_skb_any(skb);
-		return;
-	}
-	IPA_GSB_DBG_LOW("evt: %d, hdl in tx_dp_notify: %d\n", evt, hdl);
-
-	/* remove 4 byte mux header */
-	skb_pull(skb, sizeof(struct ipa_gsb_mux_hdr));
-	ipa_gsb_ctx->iface[hdl]->tx_dp_notify(
-	   ipa_gsb_ctx->iface[hdl]->priv, evt,
-	   (unsigned long)skb);
-}
-
-static int ipa_gsb_connect_sys_pipe(void)
-{
-	struct ipa_sys_connect_params prod_params;
-	struct ipa_sys_connect_params cons_params;
-	int res;
-
-	memset(&prod_params, 0, sizeof(prod_params));
-	memset(&cons_params, 0, sizeof(cons_params));
-
-	/* configure RX EP */
-	prod_params.client = IPA_CLIENT_ODU_PROD;
-	prod_params.ipa_ep_cfg.hdr.hdr_len =
-		ETH_HLEN + sizeof(struct ipa_gsb_mux_hdr);
-	prod_params.ipa_ep_cfg.nat.nat_en = IPA_SRC_NAT;
-	prod_params.ipa_ep_cfg.hdr.hdr_ofst_metadata_valid = 1;
-	prod_params.ipa_ep_cfg.hdr.hdr_ofst_metadata = 0;
-	prod_params.desc_fifo_sz = ipa_gsb_ctx->ipa_sys_desc_size;
-	prod_params.priv = NULL;
-	prod_params.notify = ipa_gsb_tx_dp_notify;
-	res = ipa_setup_sys_pipe(&prod_params,
-		&ipa_gsb_ctx->prod_hdl);
-	if (res) {
-		IPA_GSB_ERR("fail to setup prod sys pipe %d\n", res);
-		goto fail_prod;
-	}
-
-	/* configure TX EP */
-	cons_params.client = IPA_CLIENT_ODU_EMB_CONS;
-	cons_params.ipa_ep_cfg.hdr.hdr_len =
-		ETH_HLEN + sizeof(struct ipa_gsb_mux_hdr) +
-		IPA_GSB_SKB_DUMMY_HEADER;
-	cons_params.ipa_ep_cfg.hdr.hdr_ofst_pkt_size_valid = 1;
-	cons_params.ipa_ep_cfg.hdr.hdr_ofst_pkt_size = 2;
-	cons_params.ipa_ep_cfg.hdr_ext.hdr_pad_to_alignment = 2;
-	cons_params.ipa_ep_cfg.hdr_ext.hdr_little_endian = true;
-	cons_params.ipa_ep_cfg.nat.nat_en = IPA_BYPASS_NAT;
-	/* setup aggregation */
-	cons_params.ipa_ep_cfg.aggr.aggr_en = IPA_ENABLE_AGGR;
-	cons_params.ipa_ep_cfg.aggr.aggr = IPA_GENERIC;
-	cons_params.ipa_ep_cfg.aggr.aggr_time_limit =
-		IPA_GSB_AGGR_TIME_LIMIT;
-	cons_params.ipa_ep_cfg.aggr.aggr_byte_limit =
-		IPA_GSB_AGGR_BYTE_LIMIT;
-	cons_params.desc_fifo_sz = ipa_gsb_ctx->ipa_sys_desc_size;
-	cons_params.priv = NULL;
-	cons_params.notify = ipa_gsb_cons_cb;
-	res = ipa_setup_sys_pipe(&cons_params,
-		&ipa_gsb_ctx->cons_hdl);
-	if (res) {
-		IPA_GSB_ERR("fail to setup cons sys pipe %d\n", res);
-		goto fail_cons;
-	}
-
-	IPA_GSB_DBG("prod_hdl = %d, cons_hdl = %d\n",
-		ipa_gsb_ctx->prod_hdl, ipa_gsb_ctx->cons_hdl);
-
-	return 0;
-
-fail_cons:
-	ipa_teardown_sys_pipe(ipa_gsb_ctx->prod_hdl);
-	ipa_gsb_ctx->prod_hdl = 0;
-fail_prod:
-	return res;
-}
-
-int ipa_bridge_connect(u32 hdl)
-{
-	int ret;
-
-	if (!ipa_gsb_ctx) {
-		IPA_GSB_ERR("ipa_gsb_ctx was not initialized\n");
-		return -EFAULT;
-	}
-
-	if (hdl >= MAX_SUPPORTED_IFACE) {
-		IPA_GSB_ERR("invalid hdl: %d\n", hdl);
-		return -EINVAL;
-	}
-
-	IPA_GSB_DBG("client hdl: %d\n", hdl);
-
-	mutex_lock(&ipa_gsb_ctx->iface_lock[hdl]);
-	if (!ipa_gsb_ctx->iface[hdl]) {
-		IPA_GSB_ERR("fail to find interface, hdl: %d\n", hdl);
-		mutex_unlock(&ipa_gsb_ctx->iface_lock[hdl]);
-		return -EFAULT;
-	}
-
-	if (ipa_gsb_ctx->iface[hdl]->is_connected) {
-		IPA_GSB_DBG("iface was already connected\n");
-		mutex_unlock(&ipa_gsb_ctx->iface_lock[hdl]);
-		return 0;
-	}
-
-	mutex_lock(&ipa_gsb_ctx->lock);
-	if (ipa_gsb_ctx->num_connected_iface == 0) {
-		ret = ipa_pm_activate_sync(ipa_gsb_ctx->pm_hdl);
-		if (ret) {
-			IPA_GSB_ERR("failed to activate ipa pm\n");
-			mutex_unlock(&ipa_gsb_ctx->lock);
-			mutex_unlock(&ipa_gsb_ctx->iface_lock[hdl]);
-			return ret;
-		}
-		ret = ipa_gsb_connect_sys_pipe();
-		if (ret) {
-			IPA_GSB_ERR("fail to connect pipe\n");
-			mutex_unlock(&ipa_gsb_ctx->lock);
-			mutex_unlock(&ipa_gsb_ctx->iface_lock[hdl]);
-			return ret;
-		}
-	}
-
-	/* connect = connect + resume */
-	ipa_gsb_ctx->iface[hdl]->is_connected = true;
-	ipa_gsb_ctx->iface[hdl]->is_resumed = true;
-
-	ipa_gsb_ctx->num_connected_iface++;
-	IPA_GSB_DBG("connected iface: %d\n",
-		ipa_gsb_ctx->num_connected_iface);
-	ipa_gsb_ctx->num_resumed_iface++;
-	IPA_GSB_DBG("num resumed iface: %d\n",
-		ipa_gsb_ctx->num_resumed_iface);
-	mutex_unlock(&ipa_gsb_ctx->lock);
-	mutex_unlock(&ipa_gsb_ctx->iface_lock[hdl]);
-	return 0;
-}
-EXPORT_SYMBOL(ipa_bridge_connect);
-
-static int ipa_gsb_disconnect_sys_pipe(void)
-{
-	int ret;
-
-	IPA_GSB_DBG("prod_hdl = %d, cons_hdl = %d\n",
-		ipa_gsb_ctx->prod_hdl, ipa_gsb_ctx->cons_hdl);
-
-	ret = ipa_teardown_sys_pipe(ipa_gsb_ctx->prod_hdl);
-	if (ret) {
-		IPA_GSB_ERR("failed to tear down prod pipe\n");
-		return -EFAULT;
-	}
-	ipa_gsb_ctx->prod_hdl = 0;
-
-	ret = ipa_teardown_sys_pipe(ipa_gsb_ctx->cons_hdl);
-	if (ret) {
-		IPA_GSB_ERR("failed to tear down cons pipe\n");
-		return -EFAULT;
-	}
-	ipa_gsb_ctx->cons_hdl = 0;
-
-	return 0;
-}
-
-int ipa_bridge_disconnect(u32 hdl)
-{
-	int ret = 0;
-
-	if (!ipa_gsb_ctx) {
-		IPA_GSB_ERR("ipa_gsb_ctx was not initialized\n");
-		return -EFAULT;
-	}
-
-	if (hdl >= MAX_SUPPORTED_IFACE) {
-		IPA_GSB_ERR("invalid hdl: %d\n", hdl);
-		return -EINVAL;
-	}
-
-	IPA_GSB_DBG("client hdl: %d\n", hdl);
-
-	mutex_lock(&ipa_gsb_ctx->iface_lock[hdl]);
-	atomic_set(&ipa_gsb_ctx->disconnect_in_progress, 1);
-
-	if (!ipa_gsb_ctx->iface[hdl]) {
-		IPA_GSB_ERR("fail to find interface, hdl: %d\n", hdl);
-		ret = -EFAULT;
-		goto fail;
-	}
-
-	if (!ipa_gsb_ctx->iface[hdl]->is_connected) {
-		IPA_GSB_DBG("iface was not connected\n");
-		ret = 0;
-		goto fail;
-	}
-
-	mutex_lock(&ipa_gsb_ctx->lock);
-	if (ipa_gsb_ctx->num_connected_iface == 1) {
-		ret = ipa_gsb_disconnect_sys_pipe();
-		if (ret) {
-			IPA_GSB_ERR("fail to discon pipes\n");
-			ret = -EFAULT;
-			goto fail;
-		}
-
-		ret = ipa_pm_deactivate_sync(ipa_gsb_ctx->pm_hdl);
-		if (ret) {
-			IPA_GSB_ERR("failed to deactivate ipa pm\n");
-			ret = -EFAULT;
-			goto fail;
-		}
-	}
-
-	/* disconnect = suspend + disconnect */
-	ipa_gsb_ctx->iface[hdl]->is_connected = false;
-	ipa_gsb_ctx->num_connected_iface--;
-	IPA_GSB_DBG("connected iface: %d\n",
-		ipa_gsb_ctx->num_connected_iface);
-
-	if (ipa_gsb_ctx->iface[hdl]->is_resumed) {
-		ipa_gsb_ctx->iface[hdl]->is_resumed = false;
-		ipa_gsb_ctx->num_resumed_iface--;
-		IPA_GSB_DBG("num resumed iface: %d\n",
-			ipa_gsb_ctx->num_resumed_iface);
-	}
-
-fail:
-	mutex_unlock(&ipa_gsb_ctx->lock);
-	atomic_set(&ipa_gsb_ctx->disconnect_in_progress, 0);
-	mutex_unlock(&ipa_gsb_ctx->iface_lock[hdl]);
-	return ret;
-}
-EXPORT_SYMBOL(ipa_bridge_disconnect);
-
-int ipa_bridge_resume(u32 hdl)
-{
-	int ret;
-
-	if (!ipa_gsb_ctx) {
-		IPA_GSB_ERR("ipa_gsb_ctx was not initialized\n");
-		return -EFAULT;
-	}
-
-	if (hdl >= MAX_SUPPORTED_IFACE) {
-		IPA_GSB_ERR("invalid hdl: %d\n", hdl);
-		return -EINVAL;
-	}
-
-	IPA_GSB_DBG_LOW("client hdl: %d\n", hdl);
-
-	mutex_lock(&ipa_gsb_ctx->iface_lock[hdl]);
-	if (!ipa_gsb_ctx->iface[hdl]) {
-		IPA_GSB_ERR("fail to find interface, hdl: %d\n", hdl);
-		mutex_unlock(&ipa_gsb_ctx->iface_lock[hdl]);
-		return -EFAULT;
-	}
-
-	if (!ipa_gsb_ctx->iface[hdl]->is_connected) {
-		IPA_GSB_ERR("iface is not connected\n");
-		mutex_unlock(&ipa_gsb_ctx->iface_lock[hdl]);
-		return -EFAULT;
-	}
-
-	if (ipa_gsb_ctx->iface[hdl]->is_resumed) {
-		IPA_GSB_DBG_LOW("iface was already resumed\n");
-		mutex_unlock(&ipa_gsb_ctx->iface_lock[hdl]);
-		return 0;
-	}
-
-	mutex_lock(&ipa_gsb_ctx->lock);
-	if (ipa_gsb_ctx->num_resumed_iface == 0) {
-		ret = ipa_pm_activate_sync(ipa_gsb_ctx->pm_hdl);
-		if (ret) {
-			IPA_GSB_ERR("fail to activate ipa pm\n");
-			mutex_unlock(&ipa_gsb_ctx->lock);
-			mutex_unlock(&ipa_gsb_ctx->iface_lock[hdl]);
-			return ret;
-		}
-
-		ret = ipa_start_gsi_channel(
-			ipa_gsb_ctx->cons_hdl);
-		if (ret) {
-			IPA_GSB_ERR(
-				"fail to start con ep %d\n",
-				ret);
-			mutex_unlock(&ipa_gsb_ctx->lock);
-			mutex_unlock(&ipa_gsb_ctx->iface_lock[hdl]);
-			return ret;
-		}
-	}
-
-	ipa_gsb_ctx->iface[hdl]->is_resumed = true;
-	ipa_gsb_ctx->num_resumed_iface++;
-	IPA_GSB_DBG_LOW("num resumed iface: %d\n",
-		ipa_gsb_ctx->num_resumed_iface);
-
-	mutex_unlock(&ipa_gsb_ctx->lock);
-	mutex_unlock(&ipa_gsb_ctx->iface_lock[hdl]);
-	return 0;
-}
-EXPORT_SYMBOL(ipa_bridge_resume);
-
-int ipa_bridge_suspend(u32 hdl)
-{
-	int ret;
-
-	if (!ipa_gsb_ctx) {
-		IPA_GSB_ERR("ipa_gsb_ctx was not initialized\n");
-		return -EFAULT;
-	}
-
-	if (hdl >= MAX_SUPPORTED_IFACE) {
-		IPA_GSB_ERR("invalid hdl: %d\n", hdl);
-		return -EINVAL;
-	}
-
-	IPA_GSB_DBG_LOW("client hdl: %d\n", hdl);
-
-	mutex_lock(&ipa_gsb_ctx->iface_lock[hdl]);
-	atomic_set(&ipa_gsb_ctx->suspend_in_progress, 1);
-	if (!ipa_gsb_ctx->iface[hdl]) {
-		IPA_GSB_ERR("fail to find interface, hdl: %d\n", hdl);
-		atomic_set(&ipa_gsb_ctx->suspend_in_progress, 0);
-		mutex_unlock(&ipa_gsb_ctx->iface_lock[hdl]);
-		return -EFAULT;
-	}
-
-	if (!ipa_gsb_ctx->iface[hdl]->is_connected) {
-		IPA_GSB_ERR("iface is not connected\n");
-		atomic_set(&ipa_gsb_ctx->suspend_in_progress, 0);
-		mutex_unlock(&ipa_gsb_ctx->iface_lock[hdl]);
-		return -EFAULT;
-	}
-
-	if (!ipa_gsb_ctx->iface[hdl]->is_resumed) {
-		IPA_GSB_DBG_LOW("iface was already suspended\n");
-		atomic_set(&ipa_gsb_ctx->suspend_in_progress, 0);
-		mutex_unlock(&ipa_gsb_ctx->iface_lock[hdl]);
-		return 0;
-	}
-
-	mutex_lock(&ipa_gsb_ctx->lock);
-	if (ipa_gsb_ctx->num_resumed_iface == 1) {
-		ret = ipa_stop_gsi_channel(
-			ipa_gsb_ctx->cons_hdl);
-		if (ret) {
-			IPA_GSB_ERR(
-				"fail to stop cons ep %d\n",
-				ret);
-			atomic_set(&ipa_gsb_ctx->suspend_in_progress, 0);
-			mutex_unlock(&ipa_gsb_ctx->lock);
-			mutex_unlock(&ipa_gsb_ctx->iface_lock[hdl]);
-			return ret;
-		}
-
-		ret = ipa_pm_deactivate_sync(ipa_gsb_ctx->pm_hdl);
-		if (ret) {
-			IPA_GSB_ERR("fail to deactivate ipa pm\n");
-			ipa_start_gsi_channel(ipa_gsb_ctx->cons_hdl);
-			atomic_set(&ipa_gsb_ctx->suspend_in_progress, 0);
-			mutex_unlock(&ipa_gsb_ctx->lock);
-			mutex_unlock(&ipa_gsb_ctx->iface_lock[hdl]);
-			return ret;
-		}
-	}
-
-	ipa_gsb_ctx->iface[hdl]->is_resumed = false;
-	ipa_gsb_ctx->num_resumed_iface--;
-	IPA_GSB_DBG_LOW("num resumed iface: %d\n",
-		ipa_gsb_ctx->num_resumed_iface);
-	atomic_set(&ipa_gsb_ctx->suspend_in_progress, 0);
-	mutex_unlock(&ipa_gsb_ctx->lock);
-	mutex_unlock(&ipa_gsb_ctx->iface_lock[hdl]);
-	return 0;
-}
-EXPORT_SYMBOL(ipa_bridge_suspend);
-
-int ipa_bridge_set_perf_profile(u32 hdl, u32 bandwidth)
-{
-	int ret;
-
-	if (!ipa_gsb_ctx) {
-		IPA_GSB_ERR("ipa_gsb_ctx was not initialized\n");
-		return -EFAULT;
-	}
-
-	if (hdl >= MAX_SUPPORTED_IFACE) {
-		IPA_GSB_ERR("invalid hdl: %d\n", hdl);
-		return -EINVAL;
-	}
-
-	IPA_GSB_DBG("client hdl: %d, BW: %d\n", hdl, bandwidth);
-
-	mutex_lock(&ipa_gsb_ctx->iface_lock[hdl]);
-
-	ret = ipa_pm_set_throughput(ipa_gsb_ctx->pm_hdl,
-		bandwidth);
-	if (ret)
-		IPA_GSB_ERR("fail to set perf profile\n");
-
-	mutex_unlock(&ipa_gsb_ctx->iface_lock[hdl]);
-	return ret;
-}
-EXPORT_SYMBOL(ipa_bridge_set_perf_profile);
-
-int ipa_bridge_tx_dp(u32 hdl, struct sk_buff *skb,
-	struct ipa_tx_meta *metadata)
-{
-	struct ipa_gsb_mux_hdr *mux_hdr;
-	struct sk_buff *skb2;
-	struct stats iface_stats;
-	int ret;
-
-	IPA_GSB_DBG_LOW("client hdl: %d\n", hdl);
-
-	iface_stats = ipa_gsb_ctx->iface[hdl]->iface_stats;
-	if (!ipa_gsb_ctx->iface[hdl]) {
-		IPA_GSB_ERR("fail to find interface, hdl: %d\n", hdl);
-		return -EFAULT;
-	}
-
-	if (unlikely(atomic_read(&ipa_gsb_ctx->disconnect_in_progress))) {
-		IPA_GSB_ERR("ipa bridge disconnect_in_progress\n");
-		return -EFAULT;
-	}
-
-	if (unlikely(atomic_read(&ipa_gsb_ctx->suspend_in_progress))) {
-		IPA_GSB_ERR("ipa bridge suspend_in_progress\n");
-		return -EFAULT;
-	}
-
-	if (unlikely(!ipa_gsb_ctx->iface[hdl]->is_resumed)) {
-		IPA_GSB_ERR("iface %d was suspended\n", hdl);
-		return -EFAULT;
-	}
-
-	/* make sure skb has enough headroom */
-	if (unlikely(skb_headroom(skb) < sizeof(struct ipa_gsb_mux_hdr))) {
-		IPA_GSB_DBG_LOW("skb doesn't have enough headroom\n");
-		skb2 = skb_copy_expand(skb, sizeof(struct ipa_gsb_mux_hdr),
-			0, GFP_ATOMIC);
-		if (!skb2) {
-			dev_kfree_skb_any(skb);
-			return -ENOMEM;
-		}
-		dev_kfree_skb_any(skb);
-		skb = skb2;
-		iface_stats.num_insufficient_headroom_packets++;
-	}
-
-	/* add 4 byte header for mux */
-	mux_hdr = (struct ipa_gsb_mux_hdr *)skb_push(skb,
-		sizeof(struct ipa_gsb_mux_hdr));
-	mux_hdr->iface_hdl = (u8)hdl;
-	/* change to network order */
-	*(u32 *)mux_hdr = htonl(*(u32 *)mux_hdr);
-
-	ret = ipa_tx_dp(IPA_CLIENT_ODU_PROD, skb, metadata);
-	if (ret) {
-		IPA_GSB_ERR("tx dp failed %d\n", ret);
-		return -EFAULT;
-	}
-	ipa_gsb_ctx->iface[hdl]->iface_stats.num_ul_packets++;
-
-	return 0;
-}
-EXPORT_SYMBOL(ipa_bridge_tx_dp);
-
-MODULE_LICENSE("GPL v2");
-MODULE_DESCRIPTION("ipa gsb driver");

ipa/ipa_clients/ipa_mhi_client.c

@@ -1,2504 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2015-2019, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/debugfs.h>
-#include <linux/export.h>
-#include <linux/delay.h>
-#include <linux/module.h>
-#include <linux/mutex.h>
-#include <linux/ipa.h>
-#include <linux/msm_gsi.h>
-#include <linux/ipa_qmi_service_v01.h>
-#include <linux/ipa_mhi.h>
-#include "../ipa_common_i.h"
-#include "../ipa_v3/ipa_pm.h"
-
-#define IPA_MHI_DRV_NAME "ipa_mhi_client"
-
-#define IPA_MHI_DBG(fmt, args...) \
-	do { \
-		pr_debug(IPA_MHI_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-			IPA_MHI_DRV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			IPA_MHI_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-#define IPA_MHI_DBG_LOW(fmt, args...) \
-	do { \
-		pr_debug(IPA_MHI_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			IPA_MHI_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-
-#define IPA_MHI_ERR(fmt, args...) \
-	do { \
-		pr_err(IPA_MHI_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-				IPA_MHI_DRV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-				IPA_MHI_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-#define IPA_MHI_FUNC_ENTRY() \
-	IPA_MHI_DBG("ENTRY\n")
-#define IPA_MHI_FUNC_EXIT() \
-	IPA_MHI_DBG("EXIT\n")
-
-#define IPA_MHI_CH_EMPTY_TIMEOUT_MSEC 10
-
-#define IPA_MHI_SUSPEND_SLEEP_MIN 900
-#define IPA_MHI_SUSPEND_SLEEP_MAX 1100
-
-#define IPA_MHI_MAX_UL_CHANNELS 1
-#define IPA_MHI_MAX_DL_CHANNELS 2
-
-/* bit #40 in address should be asserted for MHI transfers over pcie */
-#define IPA_MHI_CLIENT_HOST_ADDR_COND(addr) \
-	((ipa_mhi_client_ctx->assert_bit40)?(IPA_MHI_HOST_ADDR(addr)):(addr))
-
-enum ipa_mhi_state {
-	IPA_MHI_STATE_INITIALIZED,
-	IPA_MHI_STATE_READY,
-	IPA_MHI_STATE_STARTED,
-	IPA_MHI_STATE_SUSPEND_IN_PROGRESS,
-	IPA_MHI_STATE_SUSPENDED,
-	IPA_MHI_STATE_RESUME_IN_PROGRESS,
-	IPA_MHI_STATE_MAX
-};
-
-static char *ipa_mhi_state_str[] = {
-	__stringify(IPA_MHI_STATE_INITIALIZED),
-	__stringify(IPA_MHI_STATE_READY),
-	__stringify(IPA_MHI_STATE_STARTED),
-	__stringify(IPA_MHI_STATE_SUSPEND_IN_PROGRESS),
-	__stringify(IPA_MHI_STATE_SUSPENDED),
-	__stringify(IPA_MHI_STATE_RESUME_IN_PROGRESS),
-};
-
-#define MHI_STATE_STR(state) \
-	(((state) >= 0 && (state) < IPA_MHI_STATE_MAX) ? \
-		ipa_mhi_state_str[(state)] : \
-		"INVALID")
-
-enum ipa_mhi_dma_dir {
-	IPA_MHI_DMA_TO_HOST,
-	IPA_MHI_DMA_FROM_HOST,
-};
-
-/**
- * struct ipa_mhi_channel_ctx - MHI Channel context
- * @valid: entry is valid
- * @id: MHI channel ID
- * @hdl: channel handle for uC
- * @client: IPA Client
- * @state: Channel state
- */
-struct ipa_mhi_channel_ctx {
-	bool valid;
-	u8 id;
-	u8 index;
-	enum ipa_client_type client;
-	enum ipa_hw_mhi_channel_states state;
-	bool stop_in_proc;
-	struct gsi_chan_info ch_info;
-	u64 channel_context_addr;
-	struct ipa_mhi_ch_ctx ch_ctx_host;
-	u64 event_context_addr;
-	struct ipa_mhi_ev_ctx ev_ctx_host;
-	bool brstmode_enabled;
-	union __packed gsi_channel_scratch ch_scratch;
-	unsigned long cached_gsi_evt_ring_hdl;
-};
-
-struct ipa_mhi_client_ctx {
-	enum ipa_mhi_state state;
-	spinlock_t state_lock;
-	mhi_client_cb cb_notify;
-	void *cb_priv;
-	bool trigger_wakeup;
-	bool wakeup_notified;
-	struct workqueue_struct *wq;
-	struct ipa_mhi_channel_ctx ul_channels[IPA_MHI_MAX_UL_CHANNELS];
-	struct ipa_mhi_channel_ctx dl_channels[IPA_MHI_MAX_DL_CHANNELS];
-	u32 total_channels;
-	struct ipa_mhi_msi_info msi;
-	u32 mmio_addr;
-	u32 first_ch_idx;
-	u32 first_er_idx;
-	u32 host_ctrl_addr;
-	u32 host_data_addr;
-	u64 channel_context_array_addr;
-	u64 event_context_array_addr;
-	u32 qmi_req_id;
-	u32 use_ipadma;
-	bool assert_bit40;
-	bool test_mode;
-	u32 pm_hdl;
-	u32 modem_pm_hdl;
-};
-
-static struct ipa_mhi_client_ctx *ipa_mhi_client_ctx;
-static DEFINE_MUTEX(mhi_client_general_mutex);
-
-#ifdef CONFIG_DEBUG_FS
-#define IPA_MHI_MAX_MSG_LEN 512
-static char dbg_buff[IPA_MHI_MAX_MSG_LEN];
-static struct dentry *dent;
-
-static char *ipa_mhi_channel_state_str[] = {
-	__stringify(IPA_HW_MHI_CHANNEL_STATE_DISABLE),
-	__stringify(IPA_HW_MHI_CHANNEL_STATE_ENABLE),
-	__stringify(IPA_HW_MHI_CHANNEL_STATE_RUN),
-	__stringify(IPA_HW_MHI_CHANNEL_STATE_SUSPEND),
-	__stringify(IPA_HW_MHI_CHANNEL_STATE_STOP),
-	__stringify(IPA_HW_MHI_CHANNEL_STATE_ERROR),
-};
-
-#define MHI_CH_STATE_STR(state) \
-	(((state) >= 0 && (state) <= IPA_HW_MHI_CHANNEL_STATE_ERROR) ? \
-	ipa_mhi_channel_state_str[(state)] : \
-	"INVALID")
-
-static int ipa_mhi_set_lock_unlock(bool is_lock)
-{
-	IPA_MHI_DBG("entry\n");
-	if (is_lock)
-		mutex_lock(&mhi_client_general_mutex);
-	else
-		mutex_unlock(&mhi_client_general_mutex);
-	IPA_MHI_DBG("exit\n");
-
-	return 0;
-}
-
-static int ipa_mhi_read_write_host(enum ipa_mhi_dma_dir dir, void *dev_addr,
-	u64 host_addr, int size)
-{
-	struct ipa_mem_buffer mem;
-	int res;
-	struct device *pdev;
-
-	IPA_MHI_FUNC_ENTRY();
-
-	if (ipa_mhi_client_ctx->use_ipadma) {
-		pdev = ipa_get_dma_dev();
-		host_addr = IPA_MHI_CLIENT_HOST_ADDR_COND(host_addr);
-
-		mem.size = size;
-		mem.base = dma_alloc_coherent(pdev, mem.size,
-			&mem.phys_base, GFP_KERNEL);
-		if (!mem.base) {
-			IPA_MHI_ERR(
-				"dma_alloc_coherent failed, DMA buff size %d\n"
-					, mem.size);
-			return -ENOMEM;
-		}
-
-		res = ipa_dma_enable();
-		if (res) {
-			IPA_MHI_ERR("failed to enable IPA DMA rc=%d\n", res);
-			goto fail_dma_enable;
-		}
-
-		if (dir == IPA_MHI_DMA_FROM_HOST) {
-			res = ipa_dma_sync_memcpy(mem.phys_base, host_addr,
-				size);
-			if (res) {
-				IPA_MHI_ERR(
-					"ipa_dma_sync_memcpy from host fail%d\n"
-					, res);
-				goto fail_memcopy;
-			}
-			memcpy(dev_addr, mem.base, size);
-		} else {
-			memcpy(mem.base, dev_addr, size);
-			res = ipa_dma_sync_memcpy(host_addr, mem.phys_base,
-				size);
-			if (res) {
-				IPA_MHI_ERR(
-					"ipa_dma_sync_memcpy to host fail %d\n"
-					, res);
-				goto fail_memcopy;
-			}
-		}
-		goto dma_succeed;
-	} else {
-		void *host_ptr;
-
-		if (!ipa_mhi_client_ctx->test_mode)
-			host_ptr = ioremap(host_addr, size);
-		else
-			host_ptr = phys_to_virt(host_addr);
-		if (!host_ptr) {
-			IPA_MHI_ERR("ioremap failed for 0x%llx\n", host_addr);
-			return -EFAULT;
-		}
-		if (dir == IPA_MHI_DMA_FROM_HOST)
-			memcpy(dev_addr, host_ptr, size);
-		else
-			memcpy(host_ptr, dev_addr, size);
-		if (!ipa_mhi_client_ctx->test_mode)
-			iounmap(host_ptr);
-	}
-
-	IPA_MHI_FUNC_EXIT();
-	return 0;
-
-dma_succeed:
-	IPA_MHI_FUNC_EXIT();
-	res = 0;
-fail_memcopy:
-	if (ipa_dma_disable())
-		IPA_MHI_ERR("failed to disable IPA DMA\n");
-fail_dma_enable:
-	dma_free_coherent(pdev, mem.size, mem.base, mem.phys_base);
-	return res;
-}
-
-static int ipa_mhi_print_channel_info(struct ipa_mhi_channel_ctx *channel,
-	char *buff, int len)
-{
-	int nbytes = 0;
-
-	if (channel->valid) {
-		nbytes += scnprintf(&buff[nbytes],
-			len - nbytes,
-			"channel idx=%d ch_id=%d client=%d state=%s\n",
-			channel->index, channel->id, channel->client,
-			MHI_CH_STATE_STR(channel->state));
-
-		nbytes += scnprintf(&buff[nbytes],
-			len - nbytes,
-			"	ch_ctx=%llx\n",
-			channel->channel_context_addr);
-
-		nbytes += scnprintf(&buff[nbytes],
-			len - nbytes,
-			"	gsi_evt_ring_hdl=%ld ev_ctx=%llx\n",
-			channel->cached_gsi_evt_ring_hdl,
-			channel->event_context_addr);
-	}
-	return nbytes;
-}
-
-static int ipa_mhi_print_host_channel_ctx_info(
-		struct ipa_mhi_channel_ctx *channel, char *buff, int len)
-{
-	int res, nbytes = 0;
-	struct ipa_mhi_ch_ctx ch_ctx_host;
-
-	memset(&ch_ctx_host, 0, sizeof(ch_ctx_host));
-
-	/* reading ch context from host */
-	res = ipa_mhi_read_write_host(IPA_MHI_DMA_FROM_HOST,
-		&ch_ctx_host, channel->channel_context_addr,
-		sizeof(ch_ctx_host));
-	if (res) {
-		nbytes += scnprintf(&buff[nbytes], len - nbytes,
-			"Failed to read from host %d\n", res);
-		return nbytes;
-	}
-
-	nbytes += scnprintf(&buff[nbytes], len - nbytes,
-		"ch_id: %d\n", channel->id);
-	nbytes += scnprintf(&buff[nbytes], len - nbytes,
-		"chstate: 0x%x\n", ch_ctx_host.chstate);
-	nbytes += scnprintf(&buff[nbytes], len - nbytes,
-		"brstmode: 0x%x\n", ch_ctx_host.brstmode);
-	nbytes += scnprintf(&buff[nbytes], len - nbytes,
-		"chtype: 0x%x\n", ch_ctx_host.chtype);
-	nbytes += scnprintf(&buff[nbytes], len - nbytes,
-		"erindex: 0x%x\n", ch_ctx_host.erindex);
-	nbytes += scnprintf(&buff[nbytes], len - nbytes,
-		"rbase: 0x%llx\n", ch_ctx_host.rbase);
-	nbytes += scnprintf(&buff[nbytes], len - nbytes,
-		"rlen: 0x%llx\n", ch_ctx_host.rlen);
-	nbytes += scnprintf(&buff[nbytes], len - nbytes,
-		"rp: 0x%llx\n", ch_ctx_host.rp);
-	nbytes += scnprintf(&buff[nbytes], len - nbytes,
-		"wp: 0x%llx\n", ch_ctx_host.wp);
-
-	return nbytes;
-}
-
-static ssize_t ipa_mhi_debugfs_stats(struct file *file,
-	char __user *ubuf,
-	size_t count,
-	loff_t *ppos)
-{
-	int nbytes = 0;
-	int i;
-	struct ipa_mhi_channel_ctx *channel;
-
-	nbytes += scnprintf(&dbg_buff[nbytes],
-		IPA_MHI_MAX_MSG_LEN - nbytes,
-		"IPA MHI state: %s\n",
-		MHI_STATE_STR(ipa_mhi_client_ctx->state));
-
-	for (i = 0; i < IPA_MHI_MAX_UL_CHANNELS; i++) {
-		channel = &ipa_mhi_client_ctx->ul_channels[i];
-		nbytes += ipa_mhi_print_channel_info(channel,
-			&dbg_buff[nbytes], IPA_MHI_MAX_MSG_LEN - nbytes);
-	}
-
-	for (i = 0; i < IPA_MHI_MAX_DL_CHANNELS; i++) {
-		channel = &ipa_mhi_client_ctx->dl_channels[i];
-		nbytes += ipa_mhi_print_channel_info(channel,
-			&dbg_buff[nbytes], IPA_MHI_MAX_MSG_LEN - nbytes);
-	}
-
-	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, nbytes);
-}
-
-static ssize_t ipa_mhi_debugfs_uc_stats(struct file *file,
-	char __user *ubuf,
-	size_t count,
-	loff_t *ppos)
-{
-	int nbytes = 0;
-
-	nbytes += ipa_uc_mhi_print_stats(dbg_buff, IPA_MHI_MAX_MSG_LEN);
-	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, nbytes);
-}
-
-static ssize_t ipa_mhi_debugfs_dump_host_ch_ctx_arr(struct file *file,
-	char __user *ubuf,
-	size_t count,
-	loff_t *ppos)
-{
-	int i, nbytes = 0;
-	struct ipa_mhi_channel_ctx *channel;
-
-	if (ipa_mhi_client_ctx->state == IPA_MHI_STATE_INITIALIZED ||
-	    ipa_mhi_client_ctx->state == IPA_MHI_STATE_READY) {
-		nbytes += scnprintf(&dbg_buff[nbytes],
-		IPA_MHI_MAX_MSG_LEN - nbytes,
-			"Cannot dump host channel context ");
-		nbytes += scnprintf(&dbg_buff[nbytes],
-				IPA_MHI_MAX_MSG_LEN - nbytes,
-				"before IPA MHI was STARTED\n");
-		return simple_read_from_buffer(ubuf, count, ppos,
-			dbg_buff, nbytes);
-	}
-	if (ipa_mhi_client_ctx->state == IPA_MHI_STATE_SUSPENDED) {
-		nbytes += scnprintf(&dbg_buff[nbytes],
-			IPA_MHI_MAX_MSG_LEN - nbytes,
-			"IPA MHI is suspended, cannot dump channel ctx array");
-		nbytes += scnprintf(&dbg_buff[nbytes],
-			IPA_MHI_MAX_MSG_LEN - nbytes,
-			" from host -PCIe can be in D3 state\n");
-		return simple_read_from_buffer(ubuf, count, ppos,
-			dbg_buff, nbytes);
-	}
-
-	nbytes += scnprintf(&dbg_buff[nbytes],
-			IPA_MHI_MAX_MSG_LEN - nbytes,
-			"channel contex array - dump from host\n");
-	nbytes += scnprintf(&dbg_buff[nbytes],
-			IPA_MHI_MAX_MSG_LEN - nbytes,
-			"***** UL channels *******\n");
-
-	for (i = 0; i < IPA_MHI_MAX_UL_CHANNELS; i++) {
-		channel = &ipa_mhi_client_ctx->ul_channels[i];
-		if (!channel->valid)
-			continue;
-		nbytes += ipa_mhi_print_host_channel_ctx_info(channel,
-			&dbg_buff[nbytes],
-			IPA_MHI_MAX_MSG_LEN - nbytes);
-	}
-
-	nbytes += scnprintf(&dbg_buff[nbytes],
-			IPA_MHI_MAX_MSG_LEN - nbytes,
-			"\n***** DL channels *******\n");
-
-	for (i = 0; i < IPA_MHI_MAX_DL_CHANNELS; i++) {
-		channel = &ipa_mhi_client_ctx->dl_channels[i];
-		if (!channel->valid)
-			continue;
-		nbytes += ipa_mhi_print_host_channel_ctx_info(channel,
-			&dbg_buff[nbytes], IPA_MHI_MAX_MSG_LEN - nbytes);
-	}
-
-	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, nbytes);
-}
-
-const struct file_operations ipa_mhi_stats_ops = {
-	.read = ipa_mhi_debugfs_stats,
-};
-
-const struct file_operations ipa_mhi_uc_stats_ops = {
-	.read = ipa_mhi_debugfs_uc_stats,
-};
-
-const struct file_operations ipa_mhi_dump_host_ch_ctx_ops = {
-	.read = ipa_mhi_debugfs_dump_host_ch_ctx_arr,
-};
-
-
-static void ipa_mhi_debugfs_init(void)
-{
-	const mode_t read_only_mode = 0444;
-	const mode_t read_write_mode = 0664;
-	struct dentry *file;
-
-	IPA_MHI_FUNC_ENTRY();
-
-	dent = debugfs_create_dir("ipa_mhi", 0);
-	if (IS_ERR(dent)) {
-		IPA_MHI_ERR("fail to create folder ipa_mhi\n");
-		return;
-	}
-
-	file = debugfs_create_file("stats", read_only_mode, dent,
-		0, &ipa_mhi_stats_ops);
-	if (!file || IS_ERR(file)) {
-		IPA_MHI_ERR("fail to create file stats\n");
-		goto fail;
-	}
-
-	file = debugfs_create_file("uc_stats", read_only_mode, dent,
-		0, &ipa_mhi_uc_stats_ops);
-	if (!file || IS_ERR(file)) {
-		IPA_MHI_ERR("fail to create file uc_stats\n");
-		goto fail;
-	}
-
-	file = debugfs_create_u32("use_ipadma", read_write_mode, dent,
-		&ipa_mhi_client_ctx->use_ipadma);
-	if (!file || IS_ERR(file)) {
-		IPA_MHI_ERR("fail to create file use_ipadma\n");
-		goto fail;
-	}
-
-	file = debugfs_create_file("dump_host_channel_ctx_array",
-		read_only_mode, dent, 0, &ipa_mhi_dump_host_ch_ctx_ops);
-	if (!file || IS_ERR(file)) {
-		IPA_MHI_ERR("fail to create file dump_host_channel_ctx_arr\n");
-		goto fail;
-	}
-
-	IPA_MHI_FUNC_EXIT();
-	return;
-fail:
-	debugfs_remove_recursive(dent);
-}
-
-#else
-static void ipa_mhi_debugfs_init(void) {}
-static void ipa_mhi_debugfs_destroy(void) {}
-#endif /* CONFIG_DEBUG_FS */
-
-static union IpaHwMhiDlUlSyncCmdData_t ipa_cached_dl_ul_sync_info;
-
-static void ipa_mhi_wq_notify_wakeup(struct work_struct *work);
-static DECLARE_WORK(ipa_mhi_notify_wakeup_work, ipa_mhi_wq_notify_wakeup);
-
-static void ipa_mhi_wq_notify_ready(struct work_struct *work);
-static DECLARE_WORK(ipa_mhi_notify_ready_work, ipa_mhi_wq_notify_ready);
-
-/**
- * ipa_mhi_notify_wakeup() - Schedule work to notify data available
- *
- * This function will schedule a work to notify data available event.
- * In case this function is called more than once, only one notification will
- * be sent to MHI client driver. No further notifications will be sent until
- * IPA MHI state will become STARTED.
- */
-static void ipa_mhi_notify_wakeup(void)
-{
-	IPA_MHI_FUNC_ENTRY();
-	if (ipa_mhi_client_ctx->wakeup_notified) {
-		IPA_MHI_DBG("wakeup already called\n");
-		return;
-	}
-	queue_work(ipa_mhi_client_ctx->wq, &ipa_mhi_notify_wakeup_work);
-	ipa_mhi_client_ctx->wakeup_notified = true;
-	IPA_MHI_FUNC_EXIT();
-}
-
-/**
- * ipa_mhi_wq_notify_wakeup() - Notify MHI client on data available
- *
- * This function is called from IPA MHI workqueue to notify
- * MHI client driver on data available event.
- */
-static void ipa_mhi_wq_notify_wakeup(struct work_struct *work)
-{
-	IPA_MHI_FUNC_ENTRY();
-	ipa_mhi_client_ctx->cb_notify(ipa_mhi_client_ctx->cb_priv,
-		IPA_MHI_EVENT_DATA_AVAILABLE, 0);
-	IPA_MHI_FUNC_EXIT();
-}
-
-/**
- * ipa_mhi_wq_notify_ready() - Notify MHI client on ready
- *
- * This function is called from IPA MHI workqueue to notify
- * MHI client driver on ready event when IPA uC is loaded
- */
-static void ipa_mhi_wq_notify_ready(struct work_struct *work)
-{
-	IPA_MHI_FUNC_ENTRY();
-	ipa_mhi_client_ctx->cb_notify(ipa_mhi_client_ctx->cb_priv,
-		IPA_MHI_EVENT_READY, 0);
-	IPA_MHI_FUNC_EXIT();
-}
-
-/**
- * ipa_mhi_notify_ready() - Schedule work to notify ready
- *
- * This function will schedule a work to notify ready event.
- */
-static void ipa_mhi_notify_ready(void)
-{
-	IPA_MHI_FUNC_ENTRY();
-	queue_work(ipa_mhi_client_ctx->wq, &ipa_mhi_notify_ready_work);
-	IPA_MHI_FUNC_EXIT();
-}
-
-/**
- * ipa_mhi_set_state() - Set new state to IPA MHI
- * @state: new state
- *
- * Sets a new state to IPA MHI if possible according to IPA MHI state machine.
- * In some state transitions a wakeup request will be triggered.
- *
- * Returns: 0 on success, -1 otherwise
- */
-static int ipa_mhi_set_state(enum ipa_mhi_state new_state)
-{
-	unsigned long flags;
-	int res = -EPERM;
-
-	spin_lock_irqsave(&ipa_mhi_client_ctx->state_lock, flags);
-	IPA_MHI_DBG("Current state: %s\n",
-			MHI_STATE_STR(ipa_mhi_client_ctx->state));
-
-	switch (ipa_mhi_client_ctx->state) {
-	case IPA_MHI_STATE_INITIALIZED:
-		if (new_state == IPA_MHI_STATE_READY) {
-			ipa_mhi_notify_ready();
-			res = 0;
-		}
-		break;
-
-	case IPA_MHI_STATE_READY:
-		if (new_state == IPA_MHI_STATE_READY)
-			res = 0;
-		if (new_state == IPA_MHI_STATE_STARTED)
-			res = 0;
-		break;
-
-	case IPA_MHI_STATE_STARTED:
-		if (new_state == IPA_MHI_STATE_INITIALIZED)
-			res = 0;
-		else if (new_state == IPA_MHI_STATE_SUSPEND_IN_PROGRESS)
-			res = 0;
-		break;
-
-	case IPA_MHI_STATE_SUSPEND_IN_PROGRESS:
-		if (new_state == IPA_MHI_STATE_SUSPENDED) {
-			if (ipa_mhi_client_ctx->trigger_wakeup) {
-				ipa_mhi_client_ctx->trigger_wakeup = false;
-				ipa_mhi_notify_wakeup();
-			}
-			res = 0;
-		} else if (new_state == IPA_MHI_STATE_STARTED) {
-			ipa_mhi_client_ctx->wakeup_notified = false;
-			ipa_mhi_client_ctx->trigger_wakeup = false;
-			res = 0;
-		}
-		break;
-
-	case IPA_MHI_STATE_SUSPENDED:
-		if (new_state == IPA_MHI_STATE_RESUME_IN_PROGRESS)
-			res = 0;
-		break;
-
-	case IPA_MHI_STATE_RESUME_IN_PROGRESS:
-		if (new_state == IPA_MHI_STATE_SUSPENDED) {
-			if (ipa_mhi_client_ctx->trigger_wakeup) {
-				ipa_mhi_client_ctx->trigger_wakeup = false;
-				ipa_mhi_notify_wakeup();
-			}
-			res = 0;
-		} else if (new_state == IPA_MHI_STATE_STARTED) {
-			ipa_mhi_client_ctx->trigger_wakeup = false;
-			ipa_mhi_client_ctx->wakeup_notified = false;
-			res = 0;
-		}
-		break;
-
-	default:
-		IPA_MHI_ERR("Invalid state %d\n", ipa_mhi_client_ctx->state);
-		WARN_ON(1);
-	}
-
-	if (res)
-		IPA_MHI_ERR("Invalid state change to %s\n",
-						MHI_STATE_STR(new_state));
-	else {
-		IPA_MHI_DBG("New state change to %s\n",
-						MHI_STATE_STR(new_state));
-		ipa_mhi_client_ctx->state = new_state;
-	}
-	spin_unlock_irqrestore(&ipa_mhi_client_ctx->state_lock, flags);
-	return res;
-}
-
-static void ipa_mhi_uc_ready_cb(void)
-{
-	IPA_MHI_FUNC_ENTRY();
-	ipa_mhi_set_state(IPA_MHI_STATE_READY);
-	IPA_MHI_FUNC_EXIT();
-}
-
-static void ipa_mhi_uc_wakeup_request_cb(void)
-{
-	unsigned long flags;
-
-	IPA_MHI_FUNC_ENTRY();
-	IPA_MHI_DBG("MHI state: %s\n",
-			MHI_STATE_STR(ipa_mhi_client_ctx->state));
-	spin_lock_irqsave(&ipa_mhi_client_ctx->state_lock, flags);
-	if (ipa_mhi_client_ctx->state == IPA_MHI_STATE_SUSPENDED)
-		ipa_mhi_notify_wakeup();
-	else if (ipa_mhi_client_ctx->state ==
-			IPA_MHI_STATE_SUSPEND_IN_PROGRESS)
-		/* wakeup event will be triggered after suspend finishes */
-		ipa_mhi_client_ctx->trigger_wakeup = true;
-
-	spin_unlock_irqrestore(&ipa_mhi_client_ctx->state_lock, flags);
-	IPA_MHI_FUNC_EXIT();
-}
-
-/**
- * ipa_mhi_start() - Start IPA MHI engine
- * @params: pcie addresses for MHI
- *
- * This function is called by MHI client driver on MHI engine start for
- * handling MHI accelerated channels. This function is called after
- * ipa_mhi_init() was called and can be called after MHI reset to restart MHI
- * engine. When this function returns device can move to M0 state.
- *
- * Return codes: 0	  : success
- *		 negative : error
- */
-int ipa_mhi_start(struct ipa_mhi_start_params *params)
-{
-	int res;
-	struct ipa_mhi_init_engine init_params;
-
-	IPA_MHI_FUNC_ENTRY();
-
-	if (!params) {
-		IPA_MHI_ERR("null args\n");
-		return -EINVAL;
-	}
-
-	if (!ipa_mhi_client_ctx) {
-		IPA_MHI_ERR("not initialized\n");
-		return -EPERM;
-	}
-
-	res = ipa_mhi_set_state(IPA_MHI_STATE_STARTED);
-	if (res) {
-		IPA_MHI_ERR("ipa_mhi_set_state %d\n", res);
-		return res;
-	}
-
-	ipa_mhi_client_ctx->host_ctrl_addr = params->host_ctrl_addr;
-	ipa_mhi_client_ctx->host_data_addr = params->host_data_addr;
-	ipa_mhi_client_ctx->channel_context_array_addr =
-		params->channel_context_array_addr;
-	ipa_mhi_client_ctx->event_context_array_addr =
-		params->event_context_array_addr;
-	IPA_MHI_DBG("host_ctrl_addr 0x%x\n",
-			ipa_mhi_client_ctx->host_ctrl_addr);
-	IPA_MHI_DBG("host_data_addr 0x%x\n",
-			ipa_mhi_client_ctx->host_data_addr);
-	IPA_MHI_DBG("channel_context_array_addr 0x%llx\n",
-		ipa_mhi_client_ctx->channel_context_array_addr);
-	IPA_MHI_DBG("event_context_array_addr 0x%llx\n",
-		ipa_mhi_client_ctx->event_context_array_addr);
-
-	res = ipa_pm_activate_sync(ipa_mhi_client_ctx->pm_hdl);
-	if (res) {
-		IPA_MHI_ERR("failed activate client %d\n", res);
-		goto fail_pm_activate;
-	}
-	res = ipa_pm_activate_sync(ipa_mhi_client_ctx->modem_pm_hdl);
-	if (res) {
-		IPA_MHI_ERR("failed activate modem client %d\n", res);
-		goto fail_pm_activate_modem;
-	}
-
-	/* gsi params */
-	init_params.gsi.first_ch_idx =
-			ipa_mhi_client_ctx->first_ch_idx;
-	/* uC params */
-	init_params.uC.first_ch_idx =
-			ipa_mhi_client_ctx->first_ch_idx;
-	init_params.uC.first_er_idx =
-			ipa_mhi_client_ctx->first_er_idx;
-	init_params.uC.host_ctrl_addr = params->host_ctrl_addr;
-	init_params.uC.host_data_addr = params->host_data_addr;
-	init_params.uC.mmio_addr = ipa_mhi_client_ctx->mmio_addr;
-	init_params.uC.msi = &ipa_mhi_client_ctx->msi;
-	init_params.uC.ipa_cached_dl_ul_sync_info =
-			&ipa_cached_dl_ul_sync_info;
-
-	res = ipa_mhi_init_engine(&init_params);
-	if (res) {
-		IPA_MHI_ERR("IPA core failed to start MHI %d\n", res);
-		goto fail_init_engine;
-	}
-
-	IPA_MHI_FUNC_EXIT();
-	return 0;
-
-fail_init_engine:
-	ipa_pm_deactivate_sync(ipa_mhi_client_ctx->modem_pm_hdl);
-fail_pm_activate_modem:
-	ipa_pm_deactivate_sync(ipa_mhi_client_ctx->pm_hdl);
-fail_pm_activate:
-	ipa_mhi_set_state(IPA_MHI_STATE_INITIALIZED);
-	return res;
-}
-
-/**
- * ipa_mhi_get_channel_context() - Get corresponding channel context
- * @ep: IPA ep
- * @channel_id: Channel ID
- *
- * This function will return the corresponding channel context or allocate new
- * one in case channel context for channel does not exist.
- */
-static struct ipa_mhi_channel_ctx *ipa_mhi_get_channel_context(
-	enum ipa_client_type client, u8 channel_id)
-{
-	int ch_idx;
-	struct ipa_mhi_channel_ctx *channels;
-	int max_channels;
-
-	if (IPA_CLIENT_IS_PROD(client)) {
-		channels = ipa_mhi_client_ctx->ul_channels;
-		max_channels = IPA_MHI_MAX_UL_CHANNELS;
-	} else {
-		channels = ipa_mhi_client_ctx->dl_channels;
-		max_channels = IPA_MHI_MAX_DL_CHANNELS;
-	}
-
-	/* find the channel context according to channel id */
-	for (ch_idx = 0; ch_idx < max_channels; ch_idx++) {
-		if (channels[ch_idx].valid &&
-		    channels[ch_idx].id == channel_id)
-			return &channels[ch_idx];
-	}
-
-	/* channel context does not exists, allocate a new one */
-	for (ch_idx = 0; ch_idx < max_channels; ch_idx++) {
-		if (!channels[ch_idx].valid)
-			break;
-	}
-
-	if (ch_idx == max_channels) {
-		IPA_MHI_ERR("no more channels available\n");
-		return NULL;
-	}
-
-	channels[ch_idx].valid = true;
-	channels[ch_idx].id = channel_id;
-	channels[ch_idx].index = ipa_mhi_client_ctx->total_channels++;
-	channels[ch_idx].client = client;
-	channels[ch_idx].state = IPA_HW_MHI_CHANNEL_STATE_INVALID;
-
-	return &channels[ch_idx];
-}
-
-/**
- * ipa_mhi_get_channel_context_by_clnt_hdl() - Get corresponding channel
- * context
- * @clnt_hdl: client handle as provided in ipa_mhi_connect_pipe()
- *
- * This function will return the corresponding channel context or NULL in case
- * that channel does not exist.
- */
-static struct ipa_mhi_channel_ctx *ipa_mhi_get_channel_context_by_clnt_hdl(
-	u32 clnt_hdl)
-{
-	int ch_idx;
-
-	for (ch_idx = 0; ch_idx < IPA_MHI_MAX_UL_CHANNELS; ch_idx++) {
-		if (ipa_mhi_client_ctx->ul_channels[ch_idx].valid &&
-		ipa_get_ep_mapping(
-			ipa_mhi_client_ctx->ul_channels[ch_idx].client)
-				== clnt_hdl)
-			return &ipa_mhi_client_ctx->ul_channels[ch_idx];
-	}
-
-	for (ch_idx = 0; ch_idx < IPA_MHI_MAX_DL_CHANNELS; ch_idx++) {
-		if (ipa_mhi_client_ctx->dl_channels[ch_idx].valid &&
-		ipa_get_ep_mapping(
-			ipa_mhi_client_ctx->dl_channels[ch_idx].client)
-				== clnt_hdl)
-			return &ipa_mhi_client_ctx->dl_channels[ch_idx];
-	}
-
-	return NULL;
-}
-
-static void ipa_mhi_dump_ch_ctx(struct ipa_mhi_channel_ctx *channel)
-{
-	IPA_MHI_DBG("ch_id %d\n", channel->id);
-	IPA_MHI_DBG("chstate 0x%x\n", channel->ch_ctx_host.chstate);
-	IPA_MHI_DBG("brstmode 0x%x\n", channel->ch_ctx_host.brstmode);
-	IPA_MHI_DBG("pollcfg 0x%x\n", channel->ch_ctx_host.pollcfg);
-	IPA_MHI_DBG("chtype 0x%x\n", channel->ch_ctx_host.chtype);
-	IPA_MHI_DBG("erindex 0x%x\n", channel->ch_ctx_host.erindex);
-	IPA_MHI_DBG("rbase 0x%llx\n", channel->ch_ctx_host.rbase);
-	IPA_MHI_DBG("rlen 0x%llx\n", channel->ch_ctx_host.rlen);
-	IPA_MHI_DBG("rp 0x%llx\n", channel->ch_ctx_host.rp);
-	IPA_MHI_DBG("wp 0x%llx\n", channel->ch_ctx_host.wp);
-}
-
-static void ipa_mhi_dump_ev_ctx(struct ipa_mhi_channel_ctx *channel)
-{
-	IPA_MHI_DBG("ch_id %d event id %d\n", channel->id,
-		channel->ch_ctx_host.erindex);
-
-	IPA_MHI_DBG("intmodc 0x%x\n", channel->ev_ctx_host.intmodc);
-	IPA_MHI_DBG("intmodt 0x%x\n", channel->ev_ctx_host.intmodt);
-	IPA_MHI_DBG("ertype 0x%x\n", channel->ev_ctx_host.ertype);
-	IPA_MHI_DBG("msivec 0x%x\n", channel->ev_ctx_host.msivec);
-	IPA_MHI_DBG("rbase 0x%llx\n", channel->ev_ctx_host.rbase);
-	IPA_MHI_DBG("rlen 0x%llx\n", channel->ev_ctx_host.rlen);
-	IPA_MHI_DBG("rp 0x%llx\n", channel->ev_ctx_host.rp);
-	IPA_MHI_DBG("wp 0x%llx\n", channel->ev_ctx_host.wp);
-}
-
-static int ipa_mhi_read_ch_ctx(struct ipa_mhi_channel_ctx *channel)
-{
-	int res;
-
-	res = ipa_mhi_read_write_host(IPA_MHI_DMA_FROM_HOST,
-		&channel->ch_ctx_host, channel->channel_context_addr,
-		sizeof(channel->ch_ctx_host));
-	if (res) {
-		IPA_MHI_ERR("ipa_mhi_read_write_host failed %d\n", res);
-		return res;
-
-	}
-	ipa_mhi_dump_ch_ctx(channel);
-
-	channel->event_context_addr =
-		ipa_mhi_client_ctx->event_context_array_addr +
-		channel->ch_ctx_host.erindex * sizeof(struct ipa_mhi_ev_ctx);
-	IPA_MHI_DBG("ch %d event_context_addr 0x%llx\n", channel->id,
-		channel->event_context_addr);
-
-	res = ipa_mhi_read_write_host(IPA_MHI_DMA_FROM_HOST,
-		&channel->ev_ctx_host, channel->event_context_addr,
-		sizeof(channel->ev_ctx_host));
-	if (res) {
-		IPA_MHI_ERR("ipa_mhi_read_write_host failed %d\n", res);
-		return res;
-
-	}
-	ipa_mhi_dump_ev_ctx(channel);
-
-	return 0;
-}
-
-static void ipa_mhi_gsi_ev_err_cb(struct gsi_evt_err_notify *notify)
-{
-	struct ipa_mhi_channel_ctx *channel = notify->user_data;
-
-	IPA_MHI_ERR("channel id=%d client=%d state=%d\n",
-		channel->id, channel->client, channel->state);
-	switch (notify->evt_id) {
-	case GSI_EVT_OUT_OF_BUFFERS_ERR:
-		IPA_MHI_ERR("Received GSI_EVT_OUT_OF_BUFFERS_ERR\n");
-		break;
-	case GSI_EVT_OUT_OF_RESOURCES_ERR:
-		IPA_MHI_ERR("Received GSI_EVT_OUT_OF_RESOURCES_ERR\n");
-		break;
-	case GSI_EVT_UNSUPPORTED_INTER_EE_OP_ERR:
-		IPA_MHI_ERR("Received GSI_EVT_UNSUPPORTED_INTER_EE_OP_ERR\n");
-		break;
-	case GSI_EVT_EVT_RING_EMPTY_ERR:
-		IPA_MHI_ERR("Received GSI_EVT_EVT_RING_EMPTY_ERR\n");
-		break;
-	default:
-		IPA_MHI_ERR("Unexpected err evt: %d\n", notify->evt_id);
-	}
-	IPA_MHI_ERR("err_desc=0x%x\n", notify->err_desc);
-	ipa_assert();
-}
-
-static void ipa_mhi_gsi_ch_err_cb(struct gsi_chan_err_notify *notify)
-{
-	struct ipa_mhi_channel_ctx *channel = notify->chan_user_data;
-
-	IPA_MHI_ERR("channel id=%d client=%d state=%d\n",
-		channel->id, channel->client, channel->state);
-	switch (notify->evt_id) {
-	case GSI_CHAN_INVALID_TRE_ERR:
-		IPA_MHI_ERR("Received GSI_CHAN_INVALID_TRE_ERR\n");
-		break;
-	case GSI_CHAN_NON_ALLOCATED_EVT_ACCESS_ERR:
-		IPA_MHI_ERR("Received GSI_CHAN_NON_ALLOCATED_EVT_ACCESS_ERR\n");
-		break;
-	case GSI_CHAN_OUT_OF_BUFFERS_ERR:
-		IPA_MHI_ERR("Received GSI_CHAN_OUT_OF_BUFFERS_ERR\n");
-		break;
-	case GSI_CHAN_OUT_OF_RESOURCES_ERR:
-		IPA_MHI_ERR("Received GSI_CHAN_OUT_OF_RESOURCES_ERR\n");
-		break;
-	case GSI_CHAN_UNSUPPORTED_INTER_EE_OP_ERR:
-		IPA_MHI_ERR("Received GSI_CHAN_UNSUPPORTED_INTER_EE_OP_ERR\n");
-		break;
-	case GSI_CHAN_HWO_1_ERR:
-		IPA_MHI_ERR("Received GSI_CHAN_HWO_1_ERR\n");
-		break;
-	default:
-		IPA_MHI_ERR("Unexpected err evt: %d\n", notify->evt_id);
-	}
-	IPA_MHI_ERR("err_desc=0x%x\n", notify->err_desc);
-	ipa_assert();
-}
-
-
-static bool ipa_mhi_gsi_channel_empty(struct ipa_mhi_channel_ctx *channel)
-{
-	IPA_MHI_FUNC_ENTRY();
-
-	if (!channel->stop_in_proc) {
-		IPA_MHI_DBG("Channel is not in STOP_IN_PROC\n");
-		return true;
-	}
-
-	if (ipa_mhi_stop_gsi_channel(channel->client)) {
-		channel->stop_in_proc = false;
-		return true;
-	}
-
-	return false;
-}
-
-/**
- * ipa_mhi_wait_for_ul_empty_timeout() - wait for pending packets in uplink
- * @msecs: timeout to wait
- *
- * This function will poll until there are no packets pending in uplink channels
- * or timeout occurred.
- *
- * Return code: true - no pending packets in uplink channels
- *		false - timeout occurred
- */
-static bool ipa_mhi_wait_for_ul_empty_timeout(unsigned int msecs)
-{
-	unsigned long jiffies_timeout = msecs_to_jiffies(msecs);
-	unsigned long jiffies_start = jiffies;
-	bool empty = false;
-	int i;
-
-	IPA_MHI_FUNC_ENTRY();
-	while (!empty) {
-		empty = true;
-		for (i = 0; i < IPA_MHI_MAX_UL_CHANNELS; i++) {
-			if (!ipa_mhi_client_ctx->ul_channels[i].valid)
-				continue;
-			if (ipa_get_transport_type() ==
-			    IPA_TRANSPORT_TYPE_GSI)
-				empty &= ipa_mhi_gsi_channel_empty(
-					&ipa_mhi_client_ctx->ul_channels[i]);
-			else
-				empty &= ipa_mhi_sps_channel_empty(
-				ipa_mhi_client_ctx->ul_channels[i].client);
-		}
-
-		if (time_after(jiffies, jiffies_start + jiffies_timeout)) {
-			IPA_MHI_DBG("finished waiting for UL empty\n");
-			break;
-		}
-
-		if (ipa_get_transport_type() == IPA_TRANSPORT_TYPE_GSI &&
-		    IPA_MHI_MAX_UL_CHANNELS == 1)
-			usleep_range(IPA_GSI_CHANNEL_STOP_SLEEP_MIN_USEC,
-			IPA_GSI_CHANNEL_STOP_SLEEP_MAX_USEC);
-	}
-
-	IPA_MHI_DBG("IPA UL is %s\n", (empty) ? "empty" : "not empty");
-
-	IPA_MHI_FUNC_EXIT();
-	return empty;
-}
-
-static int ipa_mhi_enable_force_clear(u32 request_id, bool throttle_source)
-{
-	struct ipa_enable_force_clear_datapath_req_msg_v01 req;
-	int i;
-	int res;
-
-	IPA_MHI_FUNC_ENTRY();
-	memset(&req, 0, sizeof(req));
-	req.request_id = request_id;
-	req.source_pipe_bitmask = 0;
-	for (i = 0; i < IPA_MHI_MAX_UL_CHANNELS; i++) {
-		if (!ipa_mhi_client_ctx->ul_channels[i].valid)
-			continue;
-		req.source_pipe_bitmask |= 1 << ipa_get_ep_mapping(
-				ipa_mhi_client_ctx->ul_channels[i].client);
-	}
-	if (throttle_source) {
-		req.throttle_source_valid = 1;
-		req.throttle_source = 1;
-	}
-	IPA_MHI_DBG("req_id=0x%x src_pipe_btmk=0x%x throt_src=%d\n",
-		req.request_id, req.source_pipe_bitmask,
-		req.throttle_source);
-	res = ipa_qmi_enable_force_clear_datapath_send(&req);
-	if (res) {
-		IPA_MHI_ERR(
-			"ipa_qmi_enable_force_clear_datapath_send failed %d\n"
-				, res);
-		return res;
-	}
-
-	IPA_MHI_FUNC_EXIT();
-	return 0;
-}
-
-static int ipa_mhi_disable_force_clear(u32 request_id)
-{
-	struct ipa_disable_force_clear_datapath_req_msg_v01 req;
-	int res;
-
-	IPA_MHI_FUNC_ENTRY();
-	memset(&req, 0, sizeof(req));
-	req.request_id = request_id;
-	IPA_MHI_DBG("req_id=0x%x\n", req.request_id);
-	res = ipa_qmi_disable_force_clear_datapath_send(&req);
-	if (res) {
-		IPA_MHI_ERR(
-			"ipa_qmi_disable_force_clear_datapath_send failed %d\n"
-				, res);
-		return res;
-	}
-
-	IPA_MHI_FUNC_EXIT();
-	return 0;
-}
-
-static void ipa_mhi_set_holb_on_dl_channels(bool enable,
-	struct ipa_ep_cfg_holb old_holb[])
-{
-	int i;
-	struct ipa_ep_cfg_holb ep_holb;
-	int ep_idx;
-	int res;
-
-	for (i = 0; i < IPA_MHI_MAX_DL_CHANNELS; i++) {
-		if (!ipa_mhi_client_ctx->dl_channels[i].valid)
-			continue;
-		if (ipa_mhi_client_ctx->dl_channels[i].state ==
-			IPA_HW_MHI_CHANNEL_STATE_INVALID)
-			continue;
-		ep_idx = ipa_get_ep_mapping(
-			ipa_mhi_client_ctx->dl_channels[i].client);
-		if (-1 == ep_idx) {
-			IPA_MHI_ERR("Client %u is not mapped\n",
-				ipa_mhi_client_ctx->dl_channels[i].client);
-			ipa_assert();
-			return;
-		}
-		memset(&ep_holb, 0, sizeof(ep_holb));
-		if (enable) {
-			ipa_get_holb(ep_idx, &old_holb[i]);
-			ep_holb.en = 1;
-			ep_holb.tmr_val = 0;
-		} else {
-			ep_holb = old_holb[i];
-		}
-		res = ipa_cfg_ep_holb(ep_idx, &ep_holb);
-		if (res) {
-			IPA_MHI_ERR("ipa_cfg_ep_holb failed %d\n", res);
-			ipa_assert();
-			return;
-		}
-	}
-}
-
-static int ipa_mhi_suspend_gsi_channel(struct ipa_mhi_channel_ctx *channel)
-{
-	int clnt_hdl;
-	int res;
-
-	IPA_MHI_FUNC_ENTRY();
-	clnt_hdl = ipa_get_ep_mapping(channel->client);
-	if (clnt_hdl < 0)
-		return -EFAULT;
-
-	res = ipa_stop_gsi_channel(clnt_hdl);
-	if (res != 0 && res != -GSI_STATUS_AGAIN &&
-	    res != -GSI_STATUS_TIMED_OUT) {
-		IPA_MHI_ERR("GSI stop channel failed %d\n", res);
-		return -EFAULT;
-	}
-
-	/* check if channel was stopped completely */
-	if (res)
-		channel->stop_in_proc = true;
-
-	IPA_MHI_DBG("GSI channel is %s\n", (channel->stop_in_proc) ?
-		"STOP_IN_PROC" : "STOP");
-
-	IPA_MHI_FUNC_EXIT();
-	return 0;
-}
-
-static int ipa_mhi_reset_ul_channel(struct ipa_mhi_channel_ctx *channel)
-{
-	int res;
-	bool empty;
-	struct ipa_ep_cfg_holb old_ep_holb[IPA_MHI_MAX_DL_CHANNELS];
-
-	IPA_MHI_FUNC_ENTRY();
-	if (ipa_get_transport_type() == IPA_TRANSPORT_TYPE_GSI) {
-		res = ipa_mhi_suspend_gsi_channel(channel);
-		if (res) {
-			IPA_MHI_ERR("ipa_mhi_suspend_gsi_channel failed %d\n",
-				 res);
-			return res;
-		}
-	} else {
-		res = ipa_uc_mhi_reset_channel(channel->index);
-		if (res) {
-			IPA_MHI_ERR("ipa_uc_mhi_reset_channel failed %d\n",
-				res);
-			return res;
-		}
-	}
-
-	empty = ipa_mhi_wait_for_ul_empty_timeout(
-			IPA_MHI_CH_EMPTY_TIMEOUT_MSEC);
-	if (!empty) {
-		IPA_MHI_DBG("%s not empty\n",
-			(ipa_get_transport_type() ==
-				IPA_TRANSPORT_TYPE_GSI) ? "GSI" : "BAM");
-		res = ipa_mhi_enable_force_clear(
-				ipa_mhi_client_ctx->qmi_req_id, false);
-		if (res) {
-			IPA_MHI_ERR("ipa_mhi_enable_force_clear failed %d\n",
-				res);
-			ipa_assert();
-			return res;
-		}
-
-		if (ipa_get_transport_type() == IPA_TRANSPORT_TYPE_GSI) {
-			empty = ipa_mhi_wait_for_ul_empty_timeout(
-				IPA_MHI_CH_EMPTY_TIMEOUT_MSEC);
-
-			IPA_MHI_DBG("empty=%d\n", empty);
-		} else {
-			/* enable packet drop on all DL channels */
-			ipa_mhi_set_holb_on_dl_channels(true, old_ep_holb);
-			ipa_generate_tag_process();
-			/* disable packet drop on all DL channels */
-			ipa_mhi_set_holb_on_dl_channels(false, old_ep_holb);
-
-			res = ipa_disable_sps_pipe(channel->client);
-			if (res) {
-				IPA_MHI_ERR("sps_pipe_disable fail %d\n", res);
-				ipa_assert();
-				return res;
-			}
-		}
-
-		res =
-		ipa_mhi_disable_force_clear(ipa_mhi_client_ctx->qmi_req_id);
-		if (res) {
-			IPA_MHI_ERR("ipa_mhi_disable_force_clear failed %d\n",
-				res);
-			ipa_assert();
-			return res;
-		}
-		ipa_mhi_client_ctx->qmi_req_id++;
-	}
-
-	res = ipa_mhi_reset_channel_internal(channel->client);
-	if (res) {
-		IPA_MHI_ERR("ipa_mhi_reset_ul_channel_internal failed %d\n"
-				, res);
-		return res;
-	}
-
-	IPA_MHI_FUNC_EXIT();
-
-	return 0;
-}
-
-static int ipa_mhi_reset_dl_channel(struct ipa_mhi_channel_ctx *channel)
-{
-	int res;
-
-	IPA_MHI_FUNC_ENTRY();
-	if (ipa_get_transport_type() == IPA_TRANSPORT_TYPE_GSI) {
-		res = ipa_mhi_suspend_gsi_channel(channel);
-		if (res) {
-			IPA_MHI_ERR("ipa_mhi_suspend_gsi_channel failed %d\n"
-					, res);
-			return res;
-		}
-
-		res = ipa_mhi_reset_channel_internal(channel->client);
-		if (res) {
-			IPA_MHI_ERR(
-				"ipa_mhi_reset_ul_channel_internal failed %d\n"
-				, res);
-			return res;
-		}
-	} else {
-		res = ipa_mhi_reset_channel_internal(channel->client);
-		if (res) {
-			IPA_MHI_ERR(
-				"ipa_mhi_reset_ul_channel_internal failed %d\n"
-				, res);
-			return res;
-		}
-
-		res = ipa_uc_mhi_reset_channel(channel->index);
-		if (res) {
-			IPA_MHI_ERR("ipa_uc_mhi_reset_channel failed %d\n",
-				res);
-			ipa_mhi_start_channel_internal(channel->client);
-			return res;
-		}
-	}
-
-	IPA_MHI_FUNC_EXIT();
-	return 0;
-}
-
-static int ipa_mhi_reset_channel(struct ipa_mhi_channel_ctx *channel)
-{
-	int res;
-
-	IPA_MHI_FUNC_ENTRY();
-	if (IPA_CLIENT_IS_PROD(channel->client))
-		res = ipa_mhi_reset_ul_channel(channel);
-	else
-		res = ipa_mhi_reset_dl_channel(channel);
-	if (res) {
-		IPA_MHI_ERR("failed to reset channel error %d\n", res);
-		return res;
-	}
-
-	channel->state = IPA_HW_MHI_CHANNEL_STATE_DISABLE;
-
-	if (ipa_get_transport_type() == IPA_TRANSPORT_TYPE_GSI) {
-		res = ipa_mhi_read_write_host(IPA_MHI_DMA_TO_HOST,
-			&channel->state, channel->channel_context_addr +
-				offsetof(struct ipa_mhi_ch_ctx, chstate),
-				sizeof(((struct ipa_mhi_ch_ctx *)0)->chstate));
-		if (res) {
-			IPA_MHI_ERR("ipa_mhi_read_write_host failed %d\n", res);
-			return res;
-		}
-	}
-
-	IPA_MHI_FUNC_EXIT();
-	return 0;
-}
-
-/**
- * ipa_mhi_connect_pipe() - Connect pipe to IPA and start corresponding
- * MHI channel
- * @in: connect parameters
- * @clnt_hdl: [out] client handle for this pipe
- *
- * This function is called by MHI client driver on MHI channel start.
- * This function is called after MHI engine was started.
- *
- * Return codes: 0	  : success
- *		 negative : error
- */
-int ipa_mhi_connect_pipe(struct ipa_mhi_connect_params *in, u32 *clnt_hdl)
-{
-	int res;
-	unsigned long flags;
-	struct ipa_mhi_channel_ctx *channel = NULL;
-
-	IPA_MHI_FUNC_ENTRY();
-
-	if (!in || !clnt_hdl) {
-		IPA_MHI_ERR("NULL args\n");
-		return -EINVAL;
-	}
-
-	if (in->sys.client >= IPA_CLIENT_MAX) {
-		IPA_MHI_ERR("bad param client:%d\n", in->sys.client);
-		return -EINVAL;
-	}
-
-	if (!IPA_CLIENT_IS_MHI(in->sys.client)) {
-		IPA_MHI_ERR(
-			"Invalid MHI client, client: %d\n", in->sys.client);
-		return -EINVAL;
-	}
-
-	IPA_MHI_DBG("channel=%d\n", in->channel_id);
-
-	spin_lock_irqsave(&ipa_mhi_client_ctx->state_lock, flags);
-	if (!ipa_mhi_client_ctx ||
-			ipa_mhi_client_ctx->state != IPA_MHI_STATE_STARTED) {
-		IPA_MHI_ERR("IPA MHI was not started\n");
-		spin_unlock_irqrestore(&ipa_mhi_client_ctx->state_lock, flags);
-		return -EINVAL;
-	}
-	spin_unlock_irqrestore(&ipa_mhi_client_ctx->state_lock, flags);
-
-	channel = ipa_mhi_get_channel_context(in->sys.client, in->channel_id);
-	if (!channel) {
-		IPA_MHI_ERR("ipa_mhi_get_channel_context failed\n");
-		return -EINVAL;
-	}
-
-	if (channel->state != IPA_HW_MHI_CHANNEL_STATE_INVALID &&
-	    channel->state != IPA_HW_MHI_CHANNEL_STATE_DISABLE) {
-		IPA_MHI_ERR("Invalid channel state %d\n", channel->state);
-		return -EFAULT;
-	}
-
-	channel->channel_context_addr =
-		ipa_mhi_client_ctx->channel_context_array_addr +
-			channel->id * sizeof(struct ipa_mhi_ch_ctx);
-
-	/* for event context address index needs to read from host */
-
-	IPA_MHI_DBG("client %d channelIndex %d channelID %d, state %d\n",
-		channel->client, channel->index, channel->id, channel->state);
-	IPA_MHI_DBG("channel_context_addr 0x%llx cached_gsi_evt_ring_hdl %lu\n",
-		channel->channel_context_addr,
-		channel->cached_gsi_evt_ring_hdl);
-
-	IPA_ACTIVE_CLIENTS_INC_EP(in->sys.client);
-
-	mutex_lock(&mhi_client_general_mutex);
-	if (ipa_get_transport_type() == IPA_TRANSPORT_TYPE_GSI) {
-		struct ipa_mhi_connect_params_internal internal;
-
-		IPA_MHI_DBG("reading ch/ev context from host\n");
-		res = ipa_mhi_read_ch_ctx(channel);
-		if (res) {
-			IPA_MHI_ERR("ipa_mhi_read_ch_ctx failed %d\n", res);
-			goto fail_start_channel;
-		}
-
-		internal.channel_id = in->channel_id;
-		internal.sys = &in->sys;
-		internal.start.gsi.state = channel->state;
-		internal.start.gsi.msi = &ipa_mhi_client_ctx->msi;
-		internal.start.gsi.ev_ctx_host = &channel->ev_ctx_host;
-		internal.start.gsi.event_context_addr =
-				channel->event_context_addr;
-		internal.start.gsi.ch_ctx_host = &channel->ch_ctx_host;
-		internal.start.gsi.channel_context_addr =
-				channel->channel_context_addr;
-		internal.start.gsi.ch_err_cb = ipa_mhi_gsi_ch_err_cb;
-		internal.start.gsi.channel = (void *)channel;
-		internal.start.gsi.ev_err_cb = ipa_mhi_gsi_ev_err_cb;
-		internal.start.gsi.assert_bit40 =
-				ipa_mhi_client_ctx->assert_bit40;
-		internal.start.gsi.mhi = &channel->ch_scratch.mhi;
-		internal.start.gsi.cached_gsi_evt_ring_hdl =
-				&channel->cached_gsi_evt_ring_hdl;
-		internal.start.gsi.evchid = channel->index;
-
-		res = ipa_connect_mhi_pipe(&internal, clnt_hdl);
-		if (res) {
-			IPA_MHI_ERR("ipa_connect_mhi_pipe failed %d\n", res);
-			goto fail_connect_pipe;
-		}
-		channel->state = IPA_HW_MHI_CHANNEL_STATE_RUN;
-		channel->brstmode_enabled =
-				channel->ch_scratch.mhi.burst_mode_enabled;
-
-		res = ipa_mhi_read_write_host(IPA_MHI_DMA_TO_HOST,
-			&channel->state, channel->channel_context_addr +
-				offsetof(struct ipa_mhi_ch_ctx, chstate),
-				sizeof(((struct ipa_mhi_ch_ctx *)0)->chstate));
-		if (res) {
-			IPA_MHI_ERR("ipa_mhi_read_write_host failed\n");
-			mutex_unlock(&mhi_client_general_mutex);
-			IPA_ACTIVE_CLIENTS_DEC_EP(in->sys.client);
-			return res;
-
-		}
-	} else {
-		struct ipa_mhi_connect_params_internal internal;
-
-		internal.channel_id = in->channel_id;
-		internal.sys = &in->sys;
-		internal.start.uC.index = channel->index;
-		internal.start.uC.id = channel->id;
-		internal.start.uC.state = channel->state;
-		res = ipa_connect_mhi_pipe(&internal, clnt_hdl);
-		if (res) {
-			IPA_MHI_ERR("ipa_connect_mhi_pipe failed %d\n", res);
-			goto fail_connect_pipe;
-		}
-		channel->state = IPA_HW_MHI_CHANNEL_STATE_RUN;
-	}
-	mutex_unlock(&mhi_client_general_mutex);
-
-	if (!in->sys.keep_ipa_awake)
-		IPA_ACTIVE_CLIENTS_DEC_EP(in->sys.client);
-
-	IPA_MHI_FUNC_EXIT();
-
-	return 0;
-fail_connect_pipe:
-	mutex_unlock(&mhi_client_general_mutex);
-	ipa_mhi_reset_channel(channel);
-fail_start_channel:
-	IPA_ACTIVE_CLIENTS_DEC_EP(in->sys.client);
-	return -EPERM;
-}
-
-/**
- * ipa_mhi_disconnect_pipe() - Disconnect pipe from IPA and reset corresponding
- * MHI channel
- * @clnt_hdl: client handle for this pipe
- *
- * This function is called by MHI client driver on MHI channel reset.
- * This function is called after MHI channel was started.
- * This function is doing the following:
- *	- Send command to uC/GSI to reset corresponding MHI channel
- *	- Configure IPA EP control
- *
- * Return codes: 0	  : success
- *		 negative : error
- */
-int ipa_mhi_disconnect_pipe(u32 clnt_hdl)
-{
-	int res;
-	enum ipa_client_type client;
-	static struct ipa_mhi_channel_ctx *channel;
-
-	IPA_MHI_FUNC_ENTRY();
-
-	if (!ipa_mhi_client_ctx) {
-		IPA_MHI_ERR("IPA MHI was not initialized\n");
-		return -EINVAL;
-	}
-
-	client = ipa_get_client_mapping(clnt_hdl);
-
-	if (!IPA_CLIENT_IS_MHI(client)) {
-		IPA_MHI_ERR("invalid IPA MHI client, client: %d\n", client);
-		return -EINVAL;
-	}
-
-	channel = ipa_mhi_get_channel_context_by_clnt_hdl(clnt_hdl);
-	if (!channel) {
-		IPA_MHI_ERR("invalid clnt index\n");
-		return -EINVAL;
-	}
-
-	IPA_ACTIVE_CLIENTS_INC_EP(ipa_get_client_mapping(clnt_hdl));
-
-	res = ipa_mhi_reset_channel(channel);
-	if (res) {
-		IPA_MHI_ERR("ipa_mhi_reset_channel failed %d\n", res);
-		goto fail_reset_channel;
-	}
-
-	mutex_lock(&mhi_client_general_mutex);
-	res = ipa_disconnect_mhi_pipe(clnt_hdl);
-	if (res) {
-		IPA_MHI_ERR(
-			"IPA core driver failed to disconnect the pipe hdl %d, res %d"
-				, clnt_hdl, res);
-		goto fail_disconnect_pipe;
-	}
-	mutex_unlock(&mhi_client_general_mutex);
-
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa_get_client_mapping(clnt_hdl));
-
-	IPA_MHI_DBG("client (ep: %d) disconnected\n", clnt_hdl);
-	IPA_MHI_FUNC_EXIT();
-	return 0;
-fail_disconnect_pipe:
-	mutex_unlock(&mhi_client_general_mutex);
-fail_reset_channel:
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa_get_client_mapping(clnt_hdl));
-	return res;
-}
-
-static int ipa_mhi_suspend_channels(struct ipa_mhi_channel_ctx *channels,
-	int max_channels)
-{
-	int i;
-	int res;
-
-	IPA_MHI_FUNC_ENTRY();
-	for (i = 0; i < max_channels; i++) {
-		if (!channels[i].valid)
-			continue;
-		if (channels[i].state !=
-		    IPA_HW_MHI_CHANNEL_STATE_RUN)
-			continue;
-		IPA_MHI_DBG("suspending channel %d\n",
-			channels[i].id);
-
-		if (ipa_get_transport_type() == IPA_TRANSPORT_TYPE_GSI)
-			res = ipa_mhi_suspend_gsi_channel(
-				&channels[i]);
-		else
-			res = ipa_uc_mhi_suspend_channel(
-				channels[i].index);
-
-		if (res) {
-			IPA_MHI_ERR("failed to suspend channel %d error %d\n",
-				i, res);
-			return res;
-		}
-		channels[i].state =
-			IPA_HW_MHI_CHANNEL_STATE_SUSPEND;
-	}
-
-	IPA_MHI_FUNC_EXIT();
-	return 0;
-}
-
-static int ipa_mhi_stop_event_update_channels(
-		struct ipa_mhi_channel_ctx *channels, int max_channels)
-{
-	int i;
-	int res;
-
-	if (ipa_get_transport_type() == IPA_TRANSPORT_TYPE_GSI)
-		return 0;
-
-	IPA_MHI_FUNC_ENTRY();
-	for (i = 0; i < max_channels; i++) {
-		if (!channels[i].valid)
-			continue;
-		if (channels[i].state !=
-		    IPA_HW_MHI_CHANNEL_STATE_SUSPEND)
-			continue;
-		IPA_MHI_DBG("stop update event channel %d\n",
-			channels[i].id);
-		res = ipa_uc_mhi_stop_event_update_channel(
-			channels[i].index);
-		if (res) {
-			IPA_MHI_ERR("failed stop event channel %d error %d\n",
-				i, res);
-			return res;
-		}
-	}
-
-	IPA_MHI_FUNC_EXIT();
-	return 0;
-}
-
-static bool ipa_mhi_check_pending_packets_from_host(void)
-{
-	int i;
-	int res;
-	struct ipa_mhi_channel_ctx *channel;
-
-	IPA_MHI_FUNC_ENTRY();
-	for (i = 0; i < IPA_MHI_MAX_UL_CHANNELS; i++) {
-		channel = &ipa_mhi_client_ctx->ul_channels[i];
-		if (!channel->valid)
-			continue;
-
-		res = ipa_mhi_query_ch_info(channel->client,
-				&channel->ch_info);
-		if (res) {
-			IPA_MHI_ERR("gsi_query_channel_info failed\n");
-			return true;
-		}
-		res = ipa_mhi_read_ch_ctx(channel);
-		if (res) {
-			IPA_MHI_ERR("ipa_mhi_read_ch_ctx failed %d\n", res);
-			return true;
-		}
-
-		if (channel->ch_info.rp != channel->ch_ctx_host.wp) {
-			IPA_MHI_DBG("There are pending packets from host\n");
-			IPA_MHI_DBG("device rp 0x%llx host 0x%llx\n",
-				channel->ch_info.rp, channel->ch_ctx_host.wp);
-
-			return true;
-		}
-	}
-
-	IPA_MHI_FUNC_EXIT();
-	return false;
-}
-
-static int ipa_mhi_resume_channels(bool LPTransitionRejected,
-		struct ipa_mhi_channel_ctx *channels, int max_channels)
-{
-	int i;
-	int res;
-	struct ipa_mhi_channel_ctx *channel;
-
-	IPA_MHI_FUNC_ENTRY();
-	for (i = 0; i < max_channels; i++) {
-		if (!channels[i].valid)
-			continue;
-		if (channels[i].state !=
-		    IPA_HW_MHI_CHANNEL_STATE_SUSPEND)
-			continue;
-		channel = &channels[i];
-		IPA_MHI_DBG("resuming channel %d\n", channel->id);
-
-		res = ipa_mhi_resume_channels_internal(channel->client,
-			LPTransitionRejected, channel->brstmode_enabled,
-			channel->ch_scratch, channel->index);
-
-		if (res) {
-			IPA_MHI_ERR("failed to resume channel %d error %d\n",
-				i, res);
-			return res;
-		}
-
-		channel->stop_in_proc = false;
-		channel->state = IPA_HW_MHI_CHANNEL_STATE_RUN;
-	}
-
-	IPA_MHI_FUNC_EXIT();
-	return 0;
-}
-
-/**
- * ipa_mhi_suspend_ul() - Suspend MHI accelerated up link channels
- * @force:
- *	false: in case of data pending in IPA, MHI channels will not be
- *		suspended and function will fail.
- *	true:  in case of data pending in IPA, make sure no further access from
- *		IPA to PCIe is possible. In this case suspend cannot fail.
- *
- *
- * This function is called by MHI client driver on MHI suspend.
- * This function is called after MHI channel was started.
- * When this function returns device can move to M1/M2/M3/D3cold state.
- *
- * Return codes: 0	  : success
- *		 negative : error
- */
-static int ipa_mhi_suspend_ul(bool force, bool *empty, bool *force_clear)
-{
-	int res;
-
-	*force_clear = false;
-
-	res = ipa_mhi_suspend_channels(ipa_mhi_client_ctx->ul_channels,
-		IPA_MHI_MAX_UL_CHANNELS);
-	if (res) {
-		IPA_MHI_ERR("ipa_mhi_suspend_ul_channels failed %d\n", res);
-		goto fail_suspend_ul_channel;
-	}
-
-	*empty = ipa_mhi_wait_for_ul_empty_timeout(
-			IPA_MHI_CH_EMPTY_TIMEOUT_MSEC);
-
-	if (!*empty) {
-		if (force) {
-			res = ipa_mhi_enable_force_clear(
-				ipa_mhi_client_ctx->qmi_req_id, false);
-			if (res) {
-				IPA_MHI_ERR("failed to enable force clear\n");
-				ipa_assert();
-				return res;
-			}
-			*force_clear = true;
-			IPA_MHI_DBG("force clear datapath enabled\n");
-
-			*empty = ipa_mhi_wait_for_ul_empty_timeout(
-				IPA_MHI_CH_EMPTY_TIMEOUT_MSEC);
-			IPA_MHI_DBG("empty=%d\n", *empty);
-			if (!*empty && ipa_get_transport_type()
-				== IPA_TRANSPORT_TYPE_GSI) {
-				IPA_MHI_ERR("Failed to suspend UL channels\n");
-				if (ipa_mhi_client_ctx->test_mode) {
-					res = -EAGAIN;
-					goto fail_suspend_ul_channel;
-				}
-
-				ipa_assert();
-			}
-		} else {
-			IPA_MHI_DBG("IPA not empty\n");
-			res = -EAGAIN;
-			goto fail_suspend_ul_channel;
-		}
-	}
-
-	if (*force_clear) {
-		res =
-		ipa_mhi_disable_force_clear(ipa_mhi_client_ctx->qmi_req_id);
-		if (res) {
-			IPA_MHI_ERR("failed to disable force clear\n");
-			ipa_assert();
-			return res;
-		}
-		IPA_MHI_DBG("force clear datapath disabled\n");
-		ipa_mhi_client_ctx->qmi_req_id++;
-	}
-
-	if (!force && ipa_get_transport_type() == IPA_TRANSPORT_TYPE_GSI) {
-		if (ipa_mhi_check_pending_packets_from_host()) {
-			res = -EAGAIN;
-			goto fail_suspend_ul_channel;
-		}
-	}
-
-	res = ipa_mhi_stop_event_update_channels(
-		ipa_mhi_client_ctx->ul_channels, IPA_MHI_MAX_UL_CHANNELS);
-	if (res) {
-		IPA_MHI_ERR(
-			"ipa_mhi_stop_event_update_ul_channels failed %d\n",
-			res);
-		goto fail_suspend_ul_channel;
-	}
-
-	return 0;
-
-fail_suspend_ul_channel:
-	return res;
-}
-
-static bool ipa_mhi_has_open_aggr_frame(void)
-{
-	struct ipa_mhi_channel_ctx *channel;
-	int i;
-
-	for (i = 0; i < IPA_MHI_MAX_DL_CHANNELS; i++) {
-		channel = &ipa_mhi_client_ctx->dl_channels[i];
-
-		if (!channel->valid)
-			continue;
-
-		if (ipa_has_open_aggr_frame(channel->client))
-			return true;
-	}
-
-	return false;
-}
-
-static void ipa_mhi_update_host_ch_state(bool update_rp)
-{
-	int i;
-	int res;
-	struct ipa_mhi_channel_ctx *channel;
-
-	for (i = 0; i < IPA_MHI_MAX_UL_CHANNELS; i++) {
-		channel = &ipa_mhi_client_ctx->ul_channels[i];
-		if (!channel->valid)
-			continue;
-
-		if (update_rp) {
-			res = ipa_mhi_query_ch_info(channel->client,
-				&channel->ch_info);
-			if (res) {
-				IPA_MHI_ERR("gsi_query_channel_info failed\n");
-				ipa_assert();
-				return;
-			}
-
-			res = ipa_mhi_read_write_host(IPA_MHI_DMA_TO_HOST,
-				&channel->ch_info.rp,
-				channel->channel_context_addr +
-					offsetof(struct ipa_mhi_ch_ctx, rp),
-				sizeof(channel->ch_info.rp));
-			if (res) {
-				IPA_MHI_ERR("ipa_mhi_read_write_host failed\n");
-				ipa_assert();
-				return;
-			}
-		}
-
-		res = ipa_mhi_read_write_host(IPA_MHI_DMA_TO_HOST,
-			&channel->state, channel->channel_context_addr +
-				offsetof(struct ipa_mhi_ch_ctx, chstate),
-			sizeof(((struct ipa_mhi_ch_ctx *)0)->chstate));
-		if (res) {
-			IPA_MHI_ERR("ipa_mhi_read_write_host failed\n");
-			ipa_assert();
-			return;
-		}
-		IPA_MHI_DBG("Updated UL CH=%d state to %s on host\n",
-			i, MHI_CH_STATE_STR(channel->state));
-	}
-
-	for (i = 0; i < IPA_MHI_MAX_DL_CHANNELS; i++) {
-		channel = &ipa_mhi_client_ctx->dl_channels[i];
-		if (!channel->valid)
-			continue;
-
-		if (update_rp) {
-			res = ipa_mhi_query_ch_info(channel->client,
-				&channel->ch_info);
-			if (res) {
-				IPA_MHI_ERR("gsi_query_channel_info failed\n");
-				ipa_assert();
-				return;
-			}
-
-			res = ipa_mhi_read_write_host(IPA_MHI_DMA_TO_HOST,
-				&channel->ch_info.rp,
-				channel->channel_context_addr +
-					offsetof(struct ipa_mhi_ch_ctx, rp),
-				sizeof(channel->ch_info.rp));
-			if (res) {
-				IPA_MHI_ERR("ipa_mhi_read_write_host failed\n");
-				ipa_assert();
-				return;
-			}
-		}
-
-		res = ipa_mhi_read_write_host(IPA_MHI_DMA_TO_HOST,
-			&channel->state, channel->channel_context_addr +
-			offsetof(struct ipa_mhi_ch_ctx, chstate),
-			sizeof(((struct ipa_mhi_ch_ctx *)0)->chstate));
-		if (res) {
-			IPA_MHI_ERR("ipa_mhi_read_write_host failed\n");
-			ipa_assert();
-			return;
-		}
-		IPA_MHI_DBG("Updated DL CH=%d state to %s on host\n",
-			i, MHI_CH_STATE_STR(channel->state));
-	}
-}
-
-static int ipa_mhi_suspend_dl(bool force)
-{
-	int res;
-
-	res = ipa_mhi_suspend_channels(ipa_mhi_client_ctx->dl_channels,
-		IPA_MHI_MAX_DL_CHANNELS);
-	if (res) {
-		IPA_MHI_ERR(
-			"ipa_mhi_suspend_channels for dl failed %d\n", res);
-		goto fail_suspend_dl_channel;
-	}
-
-	res = ipa_mhi_stop_event_update_channels
-			(ipa_mhi_client_ctx->dl_channels,
-			IPA_MHI_MAX_DL_CHANNELS);
-	if (res) {
-		IPA_MHI_ERR("failed to stop event update on DL %d\n", res);
-		goto fail_stop_event_update_dl_channel;
-	}
-
-	if (ipa_get_transport_type() == IPA_TRANSPORT_TYPE_GSI) {
-		if (ipa_mhi_has_open_aggr_frame()) {
-			IPA_MHI_DBG("There is an open aggr frame\n");
-			if (force) {
-				ipa_mhi_client_ctx->trigger_wakeup = true;
-			} else {
-				res = -EAGAIN;
-				goto fail_stop_event_update_dl_channel;
-			}
-		}
-	}
-
-	return 0;
-
-fail_stop_event_update_dl_channel:
-		ipa_mhi_resume_channels(true,
-				ipa_mhi_client_ctx->dl_channels,
-				IPA_MHI_MAX_DL_CHANNELS);
-fail_suspend_dl_channel:
-		return res;
-}
-
-/**
- * ipa_mhi_suspend() - Suspend MHI accelerated channels
- * @force:
- *	false: in case of data pending in IPA, MHI channels will not be
- *		suspended and function will fail.
- *	true:  in case of data pending in IPA, make sure no further access from
- *		IPA to PCIe is possible. In this case suspend cannot fail.
- *
- * This function is called by MHI client driver on MHI suspend.
- * This function is called after MHI channel was started.
- * When this function returns device can move to M1/M2/M3/D3cold state.
- *
- * Return codes: 0	  : success
- *		 negative : error
- */
-int ipa_mhi_suspend(bool force)
-{
-	int res;
-	bool empty;
-	bool force_clear;
-
-	IPA_MHI_FUNC_ENTRY();
-
-	res = ipa_mhi_set_state(IPA_MHI_STATE_SUSPEND_IN_PROGRESS);
-	if (res) {
-		IPA_MHI_ERR("ipa_mhi_set_state failed %d\n", res);
-		return res;
-	}
-
-	res = ipa_mhi_suspend_dl(force);
-	if (res) {
-		IPA_MHI_ERR("ipa_mhi_suspend_dl failed %d\n", res);
-		goto fail_suspend_dl_channel;
-	}
-
-	usleep_range(IPA_MHI_SUSPEND_SLEEP_MIN, IPA_MHI_SUSPEND_SLEEP_MAX);
-
-	res = ipa_mhi_suspend_ul(force, &empty, &force_clear);
-	if (res) {
-		IPA_MHI_ERR("ipa_mhi_suspend_ul failed %d\n", res);
-		goto fail_suspend_ul_channel;
-	}
-
-	if (ipa_get_transport_type() == IPA_TRANSPORT_TYPE_GSI)
-		ipa_mhi_update_host_ch_state(true);
-
-	/*
-	 * hold IPA clocks and release them after all
-	 * IPA PM clients are deactivated to make sure tag process
-	 * will not start
-	 */
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	res = ipa_pm_deactivate_sync(ipa_mhi_client_ctx->pm_hdl);
-	if (res) {
-		IPA_MHI_ERR("fail to deactivate client %d\n", res);
-		goto fail_deactivate_pm;
-	}
-	res = ipa_pm_deactivate_sync(ipa_mhi_client_ctx->modem_pm_hdl);
-	if (res) {
-		IPA_MHI_ERR("fail to deactivate client %d\n", res);
-		goto fail_deactivate_modem_pm;
-	}
-	usleep_range(IPA_MHI_SUSPEND_SLEEP_MIN, IPA_MHI_SUSPEND_SLEEP_MAX);
-
-	if (!empty)
-		ipa_set_tag_process_before_gating(false);
-
-	res = ipa_mhi_set_state(IPA_MHI_STATE_SUSPENDED);
-	if (res) {
-		IPA_MHI_ERR("ipa_mhi_set_state failed %d\n", res);
-		goto fail_release_cons;
-	}
-
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-	IPA_MHI_FUNC_EXIT();
-	return 0;
-
-fail_release_cons:
-	ipa_pm_activate_sync(ipa_mhi_client_ctx->modem_pm_hdl);
-fail_deactivate_modem_pm:
-	ipa_pm_activate_sync(ipa_mhi_client_ctx->pm_hdl);
-fail_deactivate_pm:
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-fail_suspend_ul_channel:
-	ipa_mhi_resume_channels(true, ipa_mhi_client_ctx->ul_channels,
-		IPA_MHI_MAX_UL_CHANNELS);
-	if (force_clear) {
-		if (
-		ipa_mhi_disable_force_clear(ipa_mhi_client_ctx->qmi_req_id)) {
-			IPA_MHI_ERR("failed to disable force clear\n");
-			ipa_assert();
-		}
-		IPA_MHI_DBG("force clear datapath disabled\n");
-		ipa_mhi_client_ctx->qmi_req_id++;
-	}
-fail_suspend_dl_channel:
-	ipa_mhi_resume_channels(true, ipa_mhi_client_ctx->dl_channels,
-		IPA_MHI_MAX_DL_CHANNELS);
-	ipa_mhi_set_state(IPA_MHI_STATE_STARTED);
-	return res;
-}
-
-/**
- * ipa_mhi_resume() - Resume MHI accelerated channels
- *
- * This function is called by MHI client driver on MHI resume.
- * This function is called after MHI channel was suspended.
- * When this function returns device can move to M0 state.
- * This function is doing the following:
- *	- Send command to uC/GSI to resume corresponding MHI channel
- *	- Activate PM clients
- *	- Resume data to IPA
- *
- * Return codes: 0	  : success
- *		 negative : error
- */
-int ipa_mhi_resume(void)
-{
-	int res;
-
-	IPA_MHI_FUNC_ENTRY();
-
-	res = ipa_mhi_set_state(IPA_MHI_STATE_RESUME_IN_PROGRESS);
-	if (res) {
-		IPA_MHI_ERR("ipa_mhi_set_state failed %d\n", res);
-		return res;
-	}
-
-	res = ipa_pm_activate_sync(ipa_mhi_client_ctx->pm_hdl);
-	if (res) {
-		IPA_MHI_ERR("fail to activate client %d\n", res);
-		goto fail_pm_activate;
-	}
-
-	res = ipa_pm_activate_sync(ipa_mhi_client_ctx->modem_pm_hdl);
-	if (res) {
-		IPA_MHI_ERR("fail to activate client %d\n", res);
-		goto fail_pm_activate_modem;
-	}
-
-	/* resume all UL channels */
-	res = ipa_mhi_resume_channels(false,
-					ipa_mhi_client_ctx->ul_channels,
-					IPA_MHI_MAX_UL_CHANNELS);
-	if (res) {
-		IPA_MHI_ERR("ipa_mhi_resume_ul_channels failed %d\n", res);
-		goto fail_resume_ul_channels;
-	}
-
-	res = ipa_mhi_resume_channels(false,
-				ipa_mhi_client_ctx->dl_channels,
-				IPA_MHI_MAX_DL_CHANNELS);
-	if (res) {
-		IPA_MHI_ERR("ipa_mhi_resume_dl_channels failed %d\n",
-			res);
-		goto fail_resume_dl_channels;
-	}
-
-	if (ipa_get_transport_type() == IPA_TRANSPORT_TYPE_GSI)
-		ipa_mhi_update_host_ch_state(false);
-
-	res = ipa_mhi_set_state(IPA_MHI_STATE_STARTED);
-	if (res) {
-		IPA_MHI_ERR("ipa_mhi_set_state failed %d\n", res);
-		goto fail_set_state;
-	}
-
-	IPA_MHI_FUNC_EXIT();
-	return 0;
-
-fail_set_state:
-	ipa_mhi_suspend_channels(ipa_mhi_client_ctx->dl_channels,
-		IPA_MHI_MAX_DL_CHANNELS);
-fail_resume_dl_channels:
-	ipa_mhi_suspend_channels(ipa_mhi_client_ctx->ul_channels,
-		IPA_MHI_MAX_UL_CHANNELS);
-fail_resume_ul_channels:
-	ipa_pm_deactivate_sync(ipa_mhi_client_ctx->modem_pm_hdl);
-fail_pm_activate_modem:
-	ipa_pm_deactivate_sync(ipa_mhi_client_ctx->pm_hdl);
-fail_pm_activate:
-	ipa_mhi_set_state(IPA_MHI_STATE_SUSPENDED);
-	return res;
-}
-
-
-static int  ipa_mhi_destroy_channels(struct ipa_mhi_channel_ctx *channels,
-	int num_of_channels)
-{
-	struct ipa_mhi_channel_ctx *channel;
-	int i, res;
-	u32 clnt_hdl;
-
-	for (i = 0; i < num_of_channels; i++) {
-		channel = &channels[i];
-		if (!channel->valid)
-			continue;
-		if (channel->state == IPA_HW_MHI_CHANNEL_STATE_INVALID)
-			continue;
-		if (channel->state != IPA_HW_MHI_CHANNEL_STATE_DISABLE) {
-			clnt_hdl = ipa_get_ep_mapping(channel->client);
-			IPA_MHI_DBG("disconnect pipe (ep: %d)\n", clnt_hdl);
-			res = ipa_mhi_disconnect_pipe(clnt_hdl);
-			if (res) {
-				IPA_MHI_ERR(
-					"failed to disconnect pipe %d, err %d\n"
-					, clnt_hdl, res);
-				goto fail;
-			}
-		}
-		res = ipa_mhi_destroy_channel(channel->client);
-		if (res) {
-			IPA_MHI_ERR(
-				"ipa_mhi_destroy_channel failed %d"
-					, res);
-			goto fail;
-		}
-	}
-	return 0;
-fail:
-	return res;
-}
-
-/**
- * ipa_mhi_destroy_all_channels() - Destroy MHI IPA channels
- *
- * This function is called by IPA MHI client driver on MHI reset to destroy all
- * IPA MHI channels.
- */
-int ipa_mhi_destroy_all_channels(void)
-{
-	int res;
-
-	IPA_MHI_FUNC_ENTRY();
-	/* reset all UL and DL acc channels and its accociated event rings */
-	res = ipa_mhi_destroy_channels(ipa_mhi_client_ctx->ul_channels,
-		IPA_MHI_MAX_UL_CHANNELS);
-	if (res) {
-		IPA_MHI_ERR("ipa_mhi_destroy_channels(ul_channels) failed %d\n",
-			res);
-		return -EPERM;
-	}
-	IPA_MHI_DBG("All UL channels are disconnected\n");
-
-	res = ipa_mhi_destroy_channels(ipa_mhi_client_ctx->dl_channels,
-		IPA_MHI_MAX_DL_CHANNELS);
-	if (res) {
-		IPA_MHI_ERR("ipa_mhi_destroy_channels(dl_channels) failed %d\n",
-			res);
-		return -EPERM;
-	}
-	IPA_MHI_DBG("All DL channels are disconnected\n");
-
-	IPA_MHI_FUNC_EXIT();
-	return 0;
-}
-
-static void ipa_mhi_debugfs_destroy(void)
-{
-	debugfs_remove_recursive(dent);
-}
-
-static void ipa_mhi_deregister_pm(void)
-{
-	ipa_pm_deactivate_sync(ipa_mhi_client_ctx->pm_hdl);
-	ipa_pm_deregister(ipa_mhi_client_ctx->pm_hdl);
-	ipa_mhi_client_ctx->pm_hdl = ~0;
-
-	ipa_pm_deactivate_sync(ipa_mhi_client_ctx->modem_pm_hdl);
-	ipa_pm_deregister(ipa_mhi_client_ctx->modem_pm_hdl);
-	ipa_mhi_client_ctx->modem_pm_hdl = ~0;
-}
-
-/**
- * ipa_mhi_destroy() - Destroy MHI IPA
- *
- * This function is called by MHI client driver on MHI reset to destroy all IPA
- * MHI resources.
- * When this function returns ipa_mhi can re-initialize.
- */
-void ipa_mhi_destroy(void)
-{
-	int res;
-
-	IPA_MHI_FUNC_ENTRY();
-	if (!ipa_mhi_client_ctx) {
-		IPA_MHI_DBG("IPA MHI was not initialized, already destroyed\n");
-		return;
-	}
-
-	ipa_deregister_client_callback(IPA_CLIENT_MHI_PROD);
-
-	/* reset all UL and DL acc channels and its accociated event rings */
-	if (ipa_get_transport_type() == IPA_TRANSPORT_TYPE_GSI) {
-		res = ipa_mhi_destroy_all_channels();
-		if (res) {
-			IPA_MHI_ERR("ipa_mhi_destroy_all_channels failed %d\n",
-				res);
-			goto fail;
-		}
-	}
-	IPA_MHI_DBG("All channels are disconnected\n");
-
-	if (ipa_get_transport_type() == IPA_TRANSPORT_TYPE_SPS) {
-		IPA_MHI_DBG("cleanup uC MHI\n");
-		ipa_uc_mhi_cleanup();
-	}
-
-	ipa_mhi_deregister_pm();
-	ipa_dma_destroy();
-	ipa_mhi_debugfs_destroy();
-	destroy_workqueue(ipa_mhi_client_ctx->wq);
-	kfree(ipa_mhi_client_ctx);
-	ipa_mhi_client_ctx = NULL;
-	IPA_MHI_DBG("IPA MHI was reset, ready for re-init\n");
-
-	IPA_MHI_FUNC_EXIT();
-	return;
-fail:
-	ipa_assert();
-}
-
-static void ipa_mhi_pm_cb(void *p, enum ipa_pm_cb_event event)
-{
-	unsigned long flags;
-
-	IPA_MHI_FUNC_ENTRY();
-
-	if (event != IPA_PM_REQUEST_WAKEUP) {
-		IPA_MHI_ERR("Unexpected event %d\n", event);
-		WARN_ON(1);
-		return;
-	}
-
-	IPA_MHI_DBG("%s\n", MHI_STATE_STR(ipa_mhi_client_ctx->state));
-	spin_lock_irqsave(&ipa_mhi_client_ctx->state_lock, flags);
-	if (ipa_mhi_client_ctx->state == IPA_MHI_STATE_SUSPENDED) {
-		ipa_mhi_notify_wakeup();
-	} else if (ipa_mhi_client_ctx->state ==
-		IPA_MHI_STATE_SUSPEND_IN_PROGRESS) {
-		/* wakeup event will be trigger after suspend finishes */
-		ipa_mhi_client_ctx->trigger_wakeup = true;
-	}
-	spin_unlock_irqrestore(&ipa_mhi_client_ctx->state_lock, flags);
-	IPA_MHI_DBG("EXIT");
-}
-
-static int ipa_mhi_register_pm(void)
-{
-	int res;
-	struct ipa_pm_register_params params;
-
-	memset(&params, 0, sizeof(params));
-	params.name = "MHI";
-	params.callback = ipa_mhi_pm_cb;
-	params.group = IPA_PM_GROUP_DEFAULT;
-	res = ipa_pm_register(&params, &ipa_mhi_client_ctx->pm_hdl);
-	if (res) {
-		IPA_MHI_ERR("fail to register with PM %d\n", res);
-		return res;
-	}
-
-	res = ipa_pm_associate_ipa_cons_to_client(ipa_mhi_client_ctx->pm_hdl,
-		IPA_CLIENT_MHI_CONS);
-	if (res) {
-		IPA_MHI_ERR("fail to associate cons with PM %d\n", res);
-		goto fail_pm_cons;
-	}
-
-	res = ipa_pm_set_throughput(ipa_mhi_client_ctx->pm_hdl, 1000);
-	if (res) {
-		IPA_MHI_ERR("fail to set perf profile to PM %d\n", res);
-		goto fail_pm_cons;
-	}
-
-	/* create a modem client for clock scaling */
-	memset(&params, 0, sizeof(params));
-	params.name = "MODEM (MHI)";
-	params.group = IPA_PM_GROUP_MODEM;
-	params.skip_clk_vote = true;
-	res = ipa_pm_register(&params, &ipa_mhi_client_ctx->modem_pm_hdl);
-	if (res) {
-		IPA_MHI_ERR("fail to register with PM %d\n", res);
-		goto fail_pm_cons;
-	}
-
-	return 0;
-
-fail_pm_cons:
-	ipa_pm_deregister(ipa_mhi_client_ctx->pm_hdl);
-	ipa_mhi_client_ctx->pm_hdl = ~0;
-	return res;
-}
-
-/**
- * ipa_mhi_init() - Initialize IPA MHI driver
- * @params: initialization params
- *
- * This function is called by MHI client driver on boot to initialize IPA MHI
- * Driver. When this function returns device can move to READY state.
- * This function is doing the following:
- *	- Initialize MHI IPA internal data structures
- *	- Register with PM
- *	- Initialize debugfs
- *
- * Return codes: 0	  : success
- *		 negative : error
- */
-int ipa_mhi_init(struct ipa_mhi_init_params *params)
-{
-	int res;
-
-	IPA_MHI_FUNC_ENTRY();
-
-	if (!params) {
-		IPA_MHI_ERR("null args\n");
-		return -EINVAL;
-	}
-
-	if (!params->notify) {
-		IPA_MHI_ERR("null notify function\n");
-		return -EINVAL;
-	}
-
-	if (ipa_mhi_client_ctx) {
-		IPA_MHI_ERR("already initialized\n");
-		return -EPERM;
-	}
-
-	IPA_MHI_DBG("notify = %pS priv = %pK\n", params->notify, params->priv);
-	IPA_MHI_DBG("msi: addr_lo = 0x%x addr_hi = 0x%x\n",
-		params->msi.addr_low, params->msi.addr_hi);
-	IPA_MHI_DBG("msi: data = 0x%x mask = 0x%x\n",
-		params->msi.data, params->msi.mask);
-	IPA_MHI_DBG("mmio_addr = 0x%x\n", params->mmio_addr);
-	IPA_MHI_DBG("first_ch_idx = 0x%x\n", params->first_ch_idx);
-	IPA_MHI_DBG("first_er_idx = 0x%x\n", params->first_er_idx);
-	IPA_MHI_DBG("assert_bit40=%d\n", params->assert_bit40);
-	IPA_MHI_DBG("test_mode=%d\n", params->test_mode);
-
-	/* Initialize context */
-	ipa_mhi_client_ctx = kzalloc(sizeof(*ipa_mhi_client_ctx), GFP_KERNEL);
-	if (!ipa_mhi_client_ctx) {
-		res = -EFAULT;
-		goto fail_alloc_ctx;
-	}
-
-	ipa_mhi_client_ctx->state = IPA_MHI_STATE_INITIALIZED;
-	ipa_mhi_client_ctx->cb_notify = params->notify;
-	ipa_mhi_client_ctx->cb_priv = params->priv;
-	spin_lock_init(&ipa_mhi_client_ctx->state_lock);
-	ipa_mhi_client_ctx->msi = params->msi;
-	ipa_mhi_client_ctx->mmio_addr = params->mmio_addr;
-	ipa_mhi_client_ctx->first_ch_idx = params->first_ch_idx;
-	ipa_mhi_client_ctx->first_er_idx = params->first_er_idx;
-	ipa_mhi_client_ctx->qmi_req_id = 0;
-	ipa_mhi_client_ctx->use_ipadma = true;
-	ipa_mhi_client_ctx->assert_bit40 = !!params->assert_bit40;
-	ipa_mhi_client_ctx->test_mode = params->test_mode;
-
-	ipa_mhi_client_ctx->wq = create_singlethread_workqueue("ipa_mhi_wq");
-	if (!ipa_mhi_client_ctx->wq) {
-		IPA_MHI_ERR("failed to create workqueue\n");
-		res = -EFAULT;
-		goto fail_create_wq;
-	}
-
-	res = ipa_dma_init();
-	if (res) {
-		IPA_MHI_ERR("failed to init ipa dma %d\n", res);
-		goto fail_dma_init;
-	}
-
-	res = ipa_mhi_register_pm();
-	if (res) {
-		IPA_MHI_ERR("failed to create PM resources\n");
-		res = -EFAULT;
-		goto fail_pm;
-	}
-
-	if (ipa_get_transport_type() == IPA_TRANSPORT_TYPE_GSI) {
-		ipa_mhi_set_state(IPA_MHI_STATE_READY);
-	} else {
-		/* Initialize uC interface */
-		ipa_uc_mhi_init(ipa_mhi_uc_ready_cb,
-			ipa_mhi_uc_wakeup_request_cb);
-		if (ipa_uc_state_check() == 0)
-			ipa_mhi_set_state(IPA_MHI_STATE_READY);
-	}
-
-	ipa_register_client_callback(&ipa_mhi_set_lock_unlock, NULL,
-					IPA_CLIENT_MHI_PROD);
-
-	/* Initialize debugfs */
-	ipa_mhi_debugfs_init();
-
-	IPA_MHI_FUNC_EXIT();
-	return 0;
-
-fail_pm:
-	ipa_dma_destroy();
-fail_dma_init:
-	destroy_workqueue(ipa_mhi_client_ctx->wq);
-fail_create_wq:
-	kfree(ipa_mhi_client_ctx);
-	ipa_mhi_client_ctx = NULL;
-fail_alloc_ctx:
-	return res;
-}
-
-static void ipa_mhi_cache_dl_ul_sync_info(
-	struct ipa_config_req_msg_v01 *config_req)
-{
-	ipa_cached_dl_ul_sync_info.params.isDlUlSyncEnabled = true;
-	ipa_cached_dl_ul_sync_info.params.UlAccmVal =
-		(config_req->ul_accumulation_time_limit_valid) ?
-		config_req->ul_accumulation_time_limit : 0;
-	ipa_cached_dl_ul_sync_info.params.ulMsiEventThreshold =
-		(config_req->ul_msi_event_threshold_valid) ?
-		config_req->ul_msi_event_threshold : 0;
-	ipa_cached_dl_ul_sync_info.params.dlMsiEventThreshold =
-		(config_req->dl_msi_event_threshold_valid) ?
-		config_req->dl_msi_event_threshold : 0;
-}
-
-/**
- * ipa_mhi_handle_ipa_config_req() - hanle IPA CONFIG QMI message
- *
- * This function is called by by IPA QMI service to indicate that IPA CONFIG
- * message was sent from modem. IPA MHI will update this information to IPA uC
- * or will cache it until IPA MHI will be initialized.
- *
- * Return codes: 0	  : success
- *		 negative : error
- */
-int ipa_mhi_handle_ipa_config_req(struct ipa_config_req_msg_v01 *config_req)
-{
-	IPA_MHI_FUNC_ENTRY();
-
-	if (ipa_get_transport_type() != IPA_TRANSPORT_TYPE_GSI) {
-		ipa_mhi_cache_dl_ul_sync_info(config_req);
-		if (ipa_mhi_client_ctx &&
-				ipa_mhi_client_ctx->state !=
-						IPA_MHI_STATE_INITIALIZED)
-			ipa_uc_mhi_send_dl_ul_sync_info(
-				&ipa_cached_dl_ul_sync_info);
-	}
-
-	IPA_MHI_FUNC_EXIT();
-	return 0;
-}
-
-int ipa_mhi_is_using_dma(bool *flag)
-{
-	IPA_MHI_FUNC_ENTRY();
-
-	if (!ipa_mhi_client_ctx) {
-		IPA_MHI_ERR("not initialized\n");
-		return -EPERM;
-	}
-
-	*flag = ipa_mhi_client_ctx->use_ipadma ? true : false;
-
-	IPA_MHI_FUNC_EXIT();
-	return 0;
-}
-EXPORT_SYMBOL(ipa_mhi_is_using_dma);
-
-const char *ipa_mhi_get_state_str(int state)
-{
-	return MHI_STATE_STR(state);
-}
-EXPORT_SYMBOL(ipa_mhi_get_state_str);
-MODULE_LICENSE("GPL v2");
-MODULE_DESCRIPTION("IPA MHI client driver");

ipa/ipa_clients/ipa_uc_offload.c

@@ -1,635 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2015-2019, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/ipa_uc_offload.h>
-#include <linux/msm_ipa.h>
-#include "../ipa_common_i.h"
-#include "../ipa_v3/ipa_pm.h"
-
-#define IPA_NTN_DMA_POOL_ALIGNMENT 8
-#define OFFLOAD_DRV_NAME "ipa_uc_offload"
-#define IPA_UC_OFFLOAD_DBG(fmt, args...) \
-	do { \
-		pr_debug(OFFLOAD_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-			OFFLOAD_DRV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			OFFLOAD_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-#define IPA_UC_OFFLOAD_LOW(fmt, args...) \
-	do { \
-		pr_debug(OFFLOAD_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			OFFLOAD_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-#define IPA_UC_OFFLOAD_ERR(fmt, args...) \
-	do { \
-		pr_err(OFFLOAD_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-			OFFLOAD_DRV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			OFFLOAD_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-#define IPA_UC_OFFLOAD_INFO(fmt, args...) \
-	do { \
-		pr_info(OFFLOAD_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-			OFFLOAD_DRV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			OFFLOAD_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-enum ipa_uc_offload_state {
-	IPA_UC_OFFLOAD_STATE_INVALID,
-	IPA_UC_OFFLOAD_STATE_INITIALIZED,
-	IPA_UC_OFFLOAD_STATE_UP,
-};
-
-struct ipa_uc_offload_ctx {
-	enum ipa_uc_offload_proto proto;
-	enum ipa_uc_offload_state state;
-	void *priv;
-	u8 hdr_len;
-	u32 partial_hdr_hdl[IPA_IP_MAX];
-	char netdev_name[IPA_RESOURCE_NAME_MAX];
-	ipa_notify_cb notify;
-	struct completion ntn_completion;
-	u32 pm_hdl;
-	struct ipa_ntn_conn_in_params conn;
-};
-
-static struct ipa_uc_offload_ctx *ipa_uc_offload_ctx[IPA_UC_MAX_PROT_SIZE];
-
-
-static int ipa_commit_partial_hdr(
-	struct ipa_ioc_add_hdr *hdr,
-	const char *netdev_name,
-	struct ipa_hdr_info *hdr_info)
-{
-	int i;
-
-	if (hdr == NULL || hdr_info == NULL) {
-		IPA_UC_OFFLOAD_ERR("Invalid input\n");
-		return -EINVAL;
-	}
-
-	hdr->commit = 1;
-	hdr->num_hdrs = 2;
-
-	snprintf(hdr->hdr[0].name, sizeof(hdr->hdr[0].name),
-			 "%s_ipv4", netdev_name);
-	snprintf(hdr->hdr[1].name, sizeof(hdr->hdr[1].name),
-			 "%s_ipv6", netdev_name);
-	for (i = IPA_IP_v4; i < IPA_IP_MAX; i++) {
-		hdr->hdr[i].hdr_len = hdr_info[i].hdr_len;
-		memcpy(hdr->hdr[i].hdr, hdr_info[i].hdr, hdr->hdr[i].hdr_len);
-		hdr->hdr[i].type = hdr_info[i].hdr_type;
-		hdr->hdr[i].is_partial = 1;
-		hdr->hdr[i].is_eth2_ofst_valid = 1;
-		hdr->hdr[i].eth2_ofst = hdr_info[i].dst_mac_addr_offset;
-	}
-
-	if (ipa_add_hdr(hdr)) {
-		IPA_UC_OFFLOAD_ERR("fail to add partial headers\n");
-		return -EFAULT;
-	}
-
-	return 0;
-}
-
-static void ipa_uc_offload_ntn_pm_cb(void *p, enum ipa_pm_cb_event event)
-{
-	/* suspend/resume is not supported */
-	IPA_UC_OFFLOAD_DBG("event = %d\n", event);
-}
-
-static int ipa_uc_offload_ntn_register_pm_client(
-	struct ipa_uc_offload_ctx *ntn_ctx)
-{
-	int res;
-	struct ipa_pm_register_params params;
-
-	memset(&params, 0, sizeof(params));
-	params.name = "ETH";
-	params.callback = ipa_uc_offload_ntn_pm_cb;
-	params.user_data = ntn_ctx;
-	params.group = IPA_PM_GROUP_DEFAULT;
-	res = ipa_pm_register(&params, &ntn_ctx->pm_hdl);
-	if (res) {
-		IPA_UC_OFFLOAD_ERR("fail to register with PM %d\n", res);
-		return res;
-	}
-
-	res = ipa_pm_associate_ipa_cons_to_client(ntn_ctx->pm_hdl,
-		IPA_CLIENT_ETHERNET_CONS);
-	if (res) {
-		IPA_UC_OFFLOAD_ERR("fail to associate cons with PM %d\n", res);
-		ipa_pm_deregister(ntn_ctx->pm_hdl);
-		ntn_ctx->pm_hdl = ~0;
-		return res;
-	}
-
-	return 0;
-}
-
-static void ipa_uc_offload_ntn_deregister_pm_client(
-	struct ipa_uc_offload_ctx *ntn_ctx)
-{
-	ipa_pm_deactivate_sync(ntn_ctx->pm_hdl);
-	ipa_pm_deregister(ntn_ctx->pm_hdl);
-}
-
-static int ipa_uc_offload_ntn_reg_intf(
-	struct ipa_uc_offload_intf_params *inp,
-	struct ipa_uc_offload_out_params *outp,
-	struct ipa_uc_offload_ctx *ntn_ctx)
-{
-	struct ipa_ioc_add_hdr *hdr = NULL;
-	struct ipa_tx_intf tx;
-	struct ipa_rx_intf rx;
-	struct ipa_ioc_tx_intf_prop tx_prop[2];
-	struct ipa_ioc_rx_intf_prop rx_prop[2];
-	int ret = 0;
-	u32 len;
-
-
-	IPA_UC_OFFLOAD_DBG("register interface for netdev %s\n",
-					 inp->netdev_name);
-	ret = ipa_uc_offload_ntn_register_pm_client(ntn_ctx);
-	if (ret) {
-		IPA_UC_OFFLOAD_ERR("fail to register PM client\n");
-		return -EFAULT;
-	}
-	memcpy(ntn_ctx->netdev_name, inp->netdev_name, IPA_RESOURCE_NAME_MAX);
-	ntn_ctx->hdr_len = inp->hdr_info[0].hdr_len;
-	ntn_ctx->notify = inp->notify;
-	ntn_ctx->priv = inp->priv;
-
-	/* add partial header */
-	len = sizeof(struct ipa_ioc_add_hdr) + 2 * sizeof(struct ipa_hdr_add);
-	hdr = kzalloc(len, GFP_KERNEL);
-	if (hdr == NULL) {
-		ret = -ENOMEM;
-		goto fail_alloc;
-	}
-
-	if (ipa_commit_partial_hdr(hdr, ntn_ctx->netdev_name, inp->hdr_info)) {
-		IPA_UC_OFFLOAD_ERR("fail to commit partial headers\n");
-		ret = -EFAULT;
-		goto fail;
-	}
-
-	/* populate tx prop */
-	tx.num_props = 2;
-	tx.prop = tx_prop;
-
-	memset(tx_prop, 0, sizeof(tx_prop));
-	tx_prop[0].ip = IPA_IP_v4;
-	tx_prop[0].dst_pipe = IPA_CLIENT_ETHERNET_CONS;
-	tx_prop[0].hdr_l2_type = inp->hdr_info[0].hdr_type;
-	memcpy(tx_prop[0].hdr_name, hdr->hdr[IPA_IP_v4].name,
-		sizeof(tx_prop[0].hdr_name));
-
-	tx_prop[1].ip = IPA_IP_v6;
-	tx_prop[1].dst_pipe = IPA_CLIENT_ETHERNET_CONS;
-	tx_prop[1].hdr_l2_type = inp->hdr_info[1].hdr_type;
-	memcpy(tx_prop[1].hdr_name, hdr->hdr[IPA_IP_v6].name,
-		sizeof(tx_prop[1].hdr_name));
-
-	/* populate rx prop */
-	rx.num_props = 2;
-	rx.prop = rx_prop;
-
-	memset(rx_prop, 0, sizeof(rx_prop));
-	rx_prop[0].ip = IPA_IP_v4;
-	rx_prop[0].src_pipe = IPA_CLIENT_ETHERNET_PROD;
-	rx_prop[0].hdr_l2_type = inp->hdr_info[0].hdr_type;
-	if (inp->is_meta_data_valid) {
-		rx_prop[0].attrib.attrib_mask |= IPA_FLT_META_DATA;
-		rx_prop[0].attrib.meta_data = inp->meta_data;
-		rx_prop[0].attrib.meta_data_mask = inp->meta_data_mask;
-	}
-
-	rx_prop[1].ip = IPA_IP_v6;
-	rx_prop[1].src_pipe = IPA_CLIENT_ETHERNET_PROD;
-	rx_prop[1].hdr_l2_type = inp->hdr_info[1].hdr_type;
-	if (inp->is_meta_data_valid) {
-		rx_prop[1].attrib.attrib_mask |= IPA_FLT_META_DATA;
-		rx_prop[1].attrib.meta_data = inp->meta_data;
-		rx_prop[1].attrib.meta_data_mask = inp->meta_data_mask;
-	}
-
-	if (ipa_register_intf(inp->netdev_name, &tx, &rx)) {
-		IPA_UC_OFFLOAD_ERR("fail to add interface prop\n");
-		memset(ntn_ctx, 0, sizeof(*ntn_ctx));
-		ret = -EFAULT;
-		goto fail;
-	}
-
-	ntn_ctx->partial_hdr_hdl[IPA_IP_v4] = hdr->hdr[IPA_IP_v4].hdr_hdl;
-	ntn_ctx->partial_hdr_hdl[IPA_IP_v6] = hdr->hdr[IPA_IP_v6].hdr_hdl;
-	init_completion(&ntn_ctx->ntn_completion);
-	ntn_ctx->state = IPA_UC_OFFLOAD_STATE_INITIALIZED;
-
-	kfree(hdr);
-	return ret;
-
-fail:
-	kfree(hdr);
-fail_alloc:
-	ipa_uc_offload_ntn_deregister_pm_client(ntn_ctx);
-	return ret;
-}
-
-int ipa_uc_offload_reg_intf(
-	struct ipa_uc_offload_intf_params *inp,
-	struct ipa_uc_offload_out_params *outp)
-{
-	struct ipa_uc_offload_ctx *ctx;
-	int ret = 0;
-
-	if (inp == NULL || outp == NULL) {
-		IPA_UC_OFFLOAD_ERR("invalid params in=%pK out=%pK\n",
-			inp, outp);
-		return -EINVAL;
-	}
-
-	if (inp->proto <= IPA_UC_INVALID ||
-		inp->proto >= IPA_UC_MAX_PROT_SIZE) {
-		IPA_UC_OFFLOAD_ERR("invalid proto %d\n", inp->proto);
-		return -EINVAL;
-	}
-
-	if (!ipa_uc_offload_ctx[inp->proto]) {
-		ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
-		if (ctx == NULL) {
-			IPA_UC_OFFLOAD_ERR("fail to alloc uc offload ctx\n");
-			return -EFAULT;
-		}
-		ipa_uc_offload_ctx[inp->proto] = ctx;
-		ctx->proto = inp->proto;
-	} else
-		ctx = ipa_uc_offload_ctx[inp->proto];
-
-	if (ctx->state != IPA_UC_OFFLOAD_STATE_INVALID) {
-		IPA_UC_OFFLOAD_ERR("Already Initialized\n");
-		return -EINVAL;
-	}
-
-	if (ctx->proto == IPA_UC_NTN) {
-		ret = ipa_uc_offload_ntn_reg_intf(inp, outp, ctx);
-		if (!ret)
-			outp->clnt_hndl = IPA_UC_NTN;
-	}
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_uc_offload_reg_intf);
-
-
-static int ipa_uc_ntn_alloc_conn_smmu_info(struct ipa_ntn_setup_info *dest,
-	struct ipa_ntn_setup_info *source)
-{
-	int result;
-
-	IPA_UC_OFFLOAD_DBG("Allocating smmu info\n");
-
-	memcpy(dest, source, sizeof(struct ipa_ntn_setup_info));
-
-	dest->data_buff_list =
-		kcalloc(dest->num_buffers, sizeof(struct ntn_buff_smmu_map),
-			GFP_KERNEL);
-	if (dest->data_buff_list == NULL) {
-		IPA_UC_OFFLOAD_ERR("failed to alloc smmu info\n");
-		return -ENOMEM;
-	}
-
-	memcpy(dest->data_buff_list, source->data_buff_list,
-		sizeof(struct ntn_buff_smmu_map) * dest->num_buffers);
-
-	result = ipa_smmu_store_sgt(&dest->buff_pool_base_sgt,
-		source->buff_pool_base_sgt);
-	if (result) {
-		kfree(dest->data_buff_list);
-		return result;
-	}
-
-	result = ipa_smmu_store_sgt(&dest->ring_base_sgt,
-		source->ring_base_sgt);
-	if (result) {
-		kfree(dest->data_buff_list);
-		ipa_smmu_free_sgt(&dest->buff_pool_base_sgt);
-		return result;
-	}
-
-	return 0;
-}
-
-static void ipa_uc_ntn_free_conn_smmu_info(struct ipa_ntn_setup_info *params)
-{
-	kfree(params->data_buff_list);
-	ipa_smmu_free_sgt(&params->buff_pool_base_sgt);
-	ipa_smmu_free_sgt(&params->ring_base_sgt);
-}
-
-int ipa_uc_ntn_conn_pipes(struct ipa_ntn_conn_in_params *inp,
-			struct ipa_ntn_conn_out_params *outp,
-			struct ipa_uc_offload_ctx *ntn_ctx)
-{
-	int result = 0;
-	enum ipa_uc_offload_state prev_state;
-
-	if (ntn_ctx->conn.dl.smmu_enabled != ntn_ctx->conn.ul.smmu_enabled) {
-		IPA_UC_OFFLOAD_ERR("ul and dl smmu enablement do not match\n");
-		return -EINVAL;
-	}
-
-	prev_state = ntn_ctx->state;
-	if (inp->dl.ring_base_pa % IPA_NTN_DMA_POOL_ALIGNMENT ||
-		inp->dl.buff_pool_base_pa % IPA_NTN_DMA_POOL_ALIGNMENT) {
-		IPA_UC_OFFLOAD_ERR("alignment failure on TX\n");
-		return -EINVAL;
-	}
-	if (inp->ul.ring_base_pa % IPA_NTN_DMA_POOL_ALIGNMENT ||
-		inp->ul.buff_pool_base_pa % IPA_NTN_DMA_POOL_ALIGNMENT) {
-		IPA_UC_OFFLOAD_ERR("alignment failure on RX\n");
-		return -EINVAL;
-	}
-
-	result = ipa_pm_activate_sync(ntn_ctx->pm_hdl);
-	if (result) {
-		IPA_UC_OFFLOAD_ERR("fail to activate: %d\n", result);
-		return result;
-	}
-
-	ntn_ctx->state = IPA_UC_OFFLOAD_STATE_UP;
-	result = ipa_setup_uc_ntn_pipes(inp, ntn_ctx->notify,
-		ntn_ctx->priv, ntn_ctx->hdr_len, outp);
-	if (result) {
-		IPA_UC_OFFLOAD_ERR("fail to setup uc offload pipes: %d\n",
-				result);
-		ntn_ctx->state = prev_state;
-		result = -EFAULT;
-		goto fail;
-	}
-
-	if (ntn_ctx->conn.dl.smmu_enabled) {
-		result = ipa_uc_ntn_alloc_conn_smmu_info(&ntn_ctx->conn.dl,
-			&inp->dl);
-		if (result) {
-			IPA_UC_OFFLOAD_ERR("alloc failure on TX\n");
-			goto fail;
-		}
-		result = ipa_uc_ntn_alloc_conn_smmu_info(&ntn_ctx->conn.ul,
-			&inp->ul);
-		if (result) {
-			ipa_uc_ntn_free_conn_smmu_info(&ntn_ctx->conn.dl);
-			IPA_UC_OFFLOAD_ERR("alloc failure on RX\n");
-			goto fail;
-		}
-	}
-
-fail:
-	return result;
-}
-
-int ipa_uc_offload_conn_pipes(struct ipa_uc_offload_conn_in_params *inp,
-			struct ipa_uc_offload_conn_out_params *outp)
-{
-	int ret = 0;
-	struct ipa_uc_offload_ctx *offload_ctx;
-
-	if (!(inp && outp)) {
-		IPA_UC_OFFLOAD_ERR("bad parm. in=%pK out=%pK\n", inp, outp);
-		return -EINVAL;
-	}
-
-	if (inp->clnt_hndl <= IPA_UC_INVALID ||
-		inp->clnt_hndl >= IPA_UC_MAX_PROT_SIZE) {
-		IPA_UC_OFFLOAD_ERR("invalid client handle %d\n",
-						   inp->clnt_hndl);
-		return -EINVAL;
-	}
-
-	offload_ctx = ipa_uc_offload_ctx[inp->clnt_hndl];
-	if (!offload_ctx) {
-		IPA_UC_OFFLOAD_ERR("Invalid Handle\n");
-		return -EINVAL;
-	}
-
-	if (offload_ctx->state != IPA_UC_OFFLOAD_STATE_INITIALIZED) {
-		IPA_UC_OFFLOAD_ERR("Invalid state %d\n", offload_ctx->state);
-		return -EPERM;
-	}
-
-	switch (offload_ctx->proto) {
-	case IPA_UC_NTN:
-		ret = ipa_uc_ntn_conn_pipes(&inp->u.ntn, &outp->u.ntn,
-						offload_ctx);
-		break;
-
-	default:
-		IPA_UC_OFFLOAD_ERR("Invalid Proto :%d\n", offload_ctx->proto);
-		ret = -EINVAL;
-		break;
-	}
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_uc_offload_conn_pipes);
-
-static int ipa_uc_ntn_disconn_pipes(struct ipa_uc_offload_ctx *ntn_ctx)
-{
-	int ipa_ep_idx_ul, ipa_ep_idx_dl;
-	int ret = 0;
-
-	if (ntn_ctx->conn.dl.smmu_enabled != ntn_ctx->conn.ul.smmu_enabled) {
-		IPA_UC_OFFLOAD_ERR("ul and dl smmu enablement do not match\n");
-		return -EINVAL;
-	}
-
-	ntn_ctx->state = IPA_UC_OFFLOAD_STATE_INITIALIZED;
-	ret = ipa_pm_deactivate_sync(ntn_ctx->pm_hdl);
-	if (ret) {
-		IPA_UC_OFFLOAD_ERR("fail to deactivate res: %d\n",
-			ret);
-		return -EFAULT;
-	}
-
-	ipa_ep_idx_ul = ipa_get_ep_mapping(IPA_CLIENT_ETHERNET_PROD);
-	ipa_ep_idx_dl = ipa_get_ep_mapping(IPA_CLIENT_ETHERNET_CONS);
-	ret = ipa_tear_down_uc_offload_pipes(ipa_ep_idx_ul, ipa_ep_idx_dl,
-		&ntn_ctx->conn);
-	if (ret) {
-		IPA_UC_OFFLOAD_ERR("fail to tear down ntn offload pipes, %d\n",
-			ret);
-		return -EFAULT;
-	}
-	if (ntn_ctx->conn.dl.smmu_enabled) {
-		ipa_uc_ntn_free_conn_smmu_info(&ntn_ctx->conn.dl);
-		ipa_uc_ntn_free_conn_smmu_info(&ntn_ctx->conn.ul);
-	}
-
-	return ret;
-}
-
-int ipa_uc_offload_disconn_pipes(u32 clnt_hdl)
-{
-	struct ipa_uc_offload_ctx *offload_ctx;
-	int ret = 0;
-
-	if (clnt_hdl <= IPA_UC_INVALID ||
-		clnt_hdl >= IPA_UC_MAX_PROT_SIZE) {
-		IPA_UC_OFFLOAD_ERR("Invalid client handle %d\n", clnt_hdl);
-		return -EINVAL;
-	}
-
-	offload_ctx = ipa_uc_offload_ctx[clnt_hdl];
-	if (!offload_ctx) {
-		IPA_UC_OFFLOAD_ERR("Invalid client Handle\n");
-		return -EINVAL;
-	}
-
-	if (offload_ctx->state != IPA_UC_OFFLOAD_STATE_UP) {
-		IPA_UC_OFFLOAD_ERR("Invalid state\n");
-		return -EINVAL;
-	}
-
-	switch (offload_ctx->proto) {
-	case IPA_UC_NTN:
-		ret = ipa_uc_ntn_disconn_pipes(offload_ctx);
-		break;
-
-	default:
-		IPA_UC_OFFLOAD_ERR("Invalid Proto :%d\n", clnt_hdl);
-		ret = -EINVAL;
-		break;
-	}
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_uc_offload_disconn_pipes);
-
-static int ipa_uc_ntn_cleanup(struct ipa_uc_offload_ctx *ntn_ctx)
-{
-	int len, result = 0;
-	struct ipa_ioc_del_hdr *hdr;
-
-	ipa_uc_offload_ntn_deregister_pm_client(ntn_ctx);
-
-	len = sizeof(struct ipa_ioc_del_hdr) + 2 * sizeof(struct ipa_hdr_del);
-	hdr = kzalloc(len, GFP_KERNEL);
-	if (hdr == NULL)
-		return -ENOMEM;
-
-	hdr->commit = 1;
-	hdr->num_hdls = 2;
-	hdr->hdl[0].hdl = ntn_ctx->partial_hdr_hdl[0];
-	hdr->hdl[1].hdl = ntn_ctx->partial_hdr_hdl[1];
-
-	if (ipa_del_hdr(hdr)) {
-		IPA_UC_OFFLOAD_ERR("fail to delete partial header\n");
-		result = -EFAULT;
-		goto fail;
-	}
-
-	if (ipa_deregister_intf(ntn_ctx->netdev_name)) {
-		IPA_UC_OFFLOAD_ERR("fail to delete interface prop\n");
-		result = -EFAULT;
-		goto fail;
-	}
-
-fail:
-	kfree(hdr);
-	return result;
-}
-
-int ipa_uc_offload_cleanup(u32 clnt_hdl)
-{
-	struct ipa_uc_offload_ctx *offload_ctx;
-	int ret = 0;
-
-	if (clnt_hdl <= IPA_UC_INVALID ||
-		clnt_hdl >= IPA_UC_MAX_PROT_SIZE) {
-		IPA_UC_OFFLOAD_ERR("Invalid client handle %d\n", clnt_hdl);
-		return -EINVAL;
-	}
-
-	offload_ctx = ipa_uc_offload_ctx[clnt_hdl];
-	if (!offload_ctx) {
-		IPA_UC_OFFLOAD_ERR("Invalid client handle %d\n", clnt_hdl);
-		return -EINVAL;
-	}
-
-	if (offload_ctx->state != IPA_UC_OFFLOAD_STATE_INITIALIZED) {
-		IPA_UC_OFFLOAD_ERR("Invalid State %d\n", offload_ctx->state);
-		return -EINVAL;
-	}
-
-	switch (offload_ctx->proto) {
-	case IPA_UC_NTN:
-		ret = ipa_uc_ntn_cleanup(offload_ctx);
-		break;
-
-	default:
-		IPA_UC_OFFLOAD_ERR("Invalid Proto :%d\n", clnt_hdl);
-		ret = -EINVAL;
-		break;
-	}
-
-	if (!ret) {
-		kfree(offload_ctx);
-		offload_ctx = NULL;
-		ipa_uc_offload_ctx[clnt_hdl] = NULL;
-	}
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_uc_offload_cleanup);
-
-/**
- * ipa_uc_offload_uc_rdyCB() - To register uC ready CB if uC not
- * ready
- * @inout:	[in/out] input/output parameters
- * from/to client
- *
- * Returns:	0 on success, negative on failure
- *
- */
-int ipa_uc_offload_reg_rdyCB(struct ipa_uc_ready_params *inp)
-{
-	int ret = 0;
-
-	if (!inp) {
-		IPA_UC_OFFLOAD_ERR("Invalid input\n");
-		return -EINVAL;
-	}
-
-	if (inp->proto == IPA_UC_NTN)
-		ret = ipa_ntn_uc_reg_rdyCB(inp->notify, inp->priv);
-
-	if (ret == -EEXIST) {
-		inp->is_uC_ready = true;
-		ret = 0;
-	} else
-		inp->is_uC_ready = false;
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_uc_offload_reg_rdyCB);
-
-void ipa_uc_offload_dereg_rdyCB(enum ipa_uc_offload_proto proto)
-{
-	if (proto == IPA_UC_NTN)
-		ipa_ntn_uc_dereg_rdyCB();
-}
-EXPORT_SYMBOL(ipa_uc_offload_dereg_rdyCB);

ipa/ipa_clients/ipa_usb.c

@@ -1,2686 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2015-2019, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/mutex.h>
-#include <linux/module.h>
-#include <linux/platform_device.h>
-#include <linux/debugfs.h>
-#include <linux/ipa.h>
-#include <linux/ipa_usb.h>
-#include <linux/rndis_ipa.h>
-#include <linux/ecm_ipa.h>
-#include "../ipa_v3/ipa_i.h"
-#include "../ipa_rm_i.h"
-
-#define IPA_USB_DEV_READY_TIMEOUT_MSEC 10000
-
-#define IPA_HOLB_TMR_EN 0x1
-
-/* GSI channels weights */
-#define IPA_USB_DL_CHAN_LOW_WEIGHT 0x5
-#define IPA_USB_UL_CHAN_LOW_WEIGHT 0x4
-
-#define IPA_USB_MAX_MSG_LEN 4096
-
-#define IPA_USB_DRV_NAME "ipa_usb"
-
-#define IPA_USB_DBG(fmt, args...) \
-	do { \
-		pr_debug(IPA_USB_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-			IPA_USB_DRV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			IPA_USB_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-#define IPA_USB_DBG_LOW(fmt, args...) \
-	do { \
-		pr_debug(IPA_USB_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			IPA_USB_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-#define IPA_USB_ERR(fmt, args...) \
-	do { \
-		pr_err(IPA_USB_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-			IPA_USB_DRV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			IPA_USB_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-#define IPA_USB_INFO(fmt, args...) \
-	do { \
-		pr_info(IPA_USB_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-			IPA_USB_DRV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			IPA_USB_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-enum ipa_usb_direction {
-	IPA_USB_DIR_UL,
-	IPA_USB_DIR_DL,
-};
-
-struct ipa_usb_xdci_connect_params_internal {
-	enum ipa_usb_max_usb_packet_size max_pkt_size;
-	u32 ipa_to_usb_clnt_hdl;
-	u8 ipa_to_usb_xferrscidx;
-	bool ipa_to_usb_xferrscidx_valid;
-	u32 usb_to_ipa_clnt_hdl;
-	u8 usb_to_ipa_xferrscidx;
-	bool usb_to_ipa_xferrscidx_valid;
-	enum ipa_usb_teth_prot teth_prot;
-	struct ipa_usb_teth_prot_params teth_prot_params;
-	u32 max_supported_bandwidth_mbps;
-};
-
-enum ipa3_usb_teth_prot_state {
-	IPA_USB_TETH_PROT_INITIALIZED,
-	IPA_USB_TETH_PROT_CONNECTED,
-	IPA_USB_TETH_PROT_INVALID
-};
-
-struct ipa3_usb_teth_prot_context {
-	union {
-		struct ipa_usb_init_params rndis;
-		struct ecm_ipa_params ecm;
-		struct teth_bridge_init_params teth_bridge;
-	} teth_prot_params;
-	enum ipa3_usb_teth_prot_state state;
-	void *user_data;
-};
-
-struct ipa3_usb_pm_context {
-	struct ipa_pm_register_params reg_params;
-	struct work_struct *remote_wakeup_work;
-	u32 hdl;
-};
-
-enum ipa3_usb_state {
-	IPA_USB_INVALID,
-	IPA_USB_INITIALIZED,
-	IPA_USB_CONNECTED,
-	IPA_USB_STOPPED,
-	IPA_USB_SUSPEND_REQUESTED,
-	IPA_USB_SUSPENDED,
-	IPA_USB_SUSPENDED_NO_RWAKEUP,
-	IPA_USB_RESUME_IN_PROGRESS
-};
-
-enum ipa3_usb_transport_type {
-	IPA_USB_TRANSPORT_TETH,
-	IPA_USB_TRANSPORT_DPL,
-	IPA_USB_TRANSPORT_MAX
-};
-
-/* Get transport type from tethering protocol */
-#define IPA3_USB_GET_TTYPE(__teth_prot) \
-	(((__teth_prot) == IPA_USB_DIAG) ? \
-	IPA_USB_TRANSPORT_DPL : IPA_USB_TRANSPORT_TETH)
-
-/* Does the given transport type is DPL? */
-#define IPA3_USB_IS_TTYPE_DPL(__ttype) \
-	((__ttype) == IPA_USB_TRANSPORT_DPL)
-
-struct ipa3_usb_teth_prot_conn_params {
-	u32 usb_to_ipa_clnt_hdl;
-	u32 ipa_to_usb_clnt_hdl;
-	struct ipa_usb_teth_prot_params params;
-};
-
-/**
- * Transport type - could be either data tethering or DPL
- * Each transport has it's own PM resources and statuses
- */
-struct ipa3_usb_transport_type_ctx {
-	struct ipa3_usb_pm_context pm_ctx;
-	int (*ipa_usb_notify_cb)(enum ipa_usb_notify_event, void *user_data);
-	void *user_data;
-	enum ipa3_usb_state state;
-	bool rwakeup_pending;
-	struct ipa_usb_xdci_chan_params ul_ch_params;
-	struct ipa_usb_xdci_chan_params dl_ch_params;
-	struct ipa3_usb_teth_prot_conn_params teth_conn_params;
-};
-
-struct ipa3_usb_smmu_reg_map {
-	int cnt;
-	phys_addr_t addr;
-};
-
-/*
- * Relevant for IPA4.5 on sdx55v1 and Kona.
- */
-static const bool teth_type_switch_tbl_ipa45
-	[IPA_USB_MAX_TETH_PROT_SIZE][IPA_USB_MAX_TETH_PROT_SIZE] = {
-		[IPA_USB_RNDIS] = {true, false, true, false, false},
-		[IPA_USB_ECM] = {false, true, false, false, false},
-		[IPA_USB_RMNET] = {true, false, true, false, false},
-		[IPA_USB_MBIM] = {true, true, true, true, false},
-		[IPA_USB_DIAG] = {false, false, false, false, true},
-	};
-
-struct ipa3_usb_teth_type_switch {
-	bool valid;
-	enum ipa_usb_teth_prot teth;
-};
-
-struct ipa3_usb_context {
-	struct ipa3_usb_teth_prot_context
-		teth_prot_ctx[IPA_USB_MAX_TETH_PROT_SIZE];
-	int num_init_prot; /* without dpl */
-	struct teth_bridge_init_params teth_bridge_params;
-	struct completion dev_ready_comp;
-	u32 qmi_req_id;
-	spinlock_t state_lock;
-	bool dl_data_pending;
-	struct workqueue_struct *wq;
-	struct mutex general_mutex;
-	struct ipa3_usb_transport_type_ctx
-		ttype_ctx[IPA_USB_TRANSPORT_MAX];
-	struct dentry *dfile_state_info;
-	struct dentry *dent;
-	struct ipa3_usb_smmu_reg_map smmu_reg_map;
-	struct ipa3_usb_teth_type_switch prev_teth;
-};
-
-enum ipa3_usb_op {
-	IPA_USB_OP_INIT_TETH_PROT,
-	IPA_USB_OP_REQUEST_CHANNEL,
-	IPA_USB_OP_CONNECT,
-	IPA_USB_OP_DISCONNECT,
-	IPA_USB_OP_RELEASE_CHANNEL,
-	IPA_USB_OP_DEINIT_TETH_PROT,
-	IPA_USB_OP_SUSPEND,
-	IPA_USB_OP_SUSPEND_NO_RWAKEUP,
-	IPA_USB_OP_RESUME
-};
-
-struct ipa3_usb_status_dbg_info {
-	const char *teth_state;
-	const char *dpl_state;
-	int num_init_prot;
-	const char *inited_prots[IPA_USB_MAX_TETH_PROT_SIZE];
-	const char *teth_connected_prot;
-	const char *dpl_connected_prot;
-};
-
-static void ipa3_usb_wq_notify_remote_wakeup(struct work_struct *work);
-static void ipa3_usb_wq_dpl_notify_remote_wakeup(struct work_struct *work);
-static DECLARE_WORK(ipa3_usb_notify_remote_wakeup_work,
-	ipa3_usb_wq_notify_remote_wakeup);
-static DECLARE_WORK(ipa3_usb_dpl_notify_remote_wakeup_work,
-	ipa3_usb_wq_dpl_notify_remote_wakeup);
-
-struct ipa3_usb_context *ipa3_usb_ctx;
-
-static char *ipa3_usb_op_to_string(enum ipa3_usb_op op)
-{
-	switch (op) {
-	case IPA_USB_OP_INIT_TETH_PROT:
-		return "IPA_USB_OP_INIT_TETH_PROT";
-	case IPA_USB_OP_REQUEST_CHANNEL:
-		return "IPA_USB_OP_REQUEST_CHANNEL";
-	case IPA_USB_OP_CONNECT:
-		return "IPA_USB_OP_CONNECT";
-	case IPA_USB_OP_DISCONNECT:
-		return "IPA_USB_OP_DISCONNECT";
-	case IPA_USB_OP_RELEASE_CHANNEL:
-		return "IPA_USB_OP_RELEASE_CHANNEL";
-	case IPA_USB_OP_DEINIT_TETH_PROT:
-		return "IPA_USB_OP_DEINIT_TETH_PROT";
-	case IPA_USB_OP_SUSPEND:
-		return "IPA_USB_OP_SUSPEND";
-	case IPA_USB_OP_SUSPEND_NO_RWAKEUP:
-		return "IPA_USB_OP_SUSPEND_NO_RWAKEUP";
-	case IPA_USB_OP_RESUME:
-		return "IPA_USB_OP_RESUME";
-	}
-
-	return "UNSUPPORTED";
-}
-
-static char *ipa3_usb_state_to_string(enum ipa3_usb_state state)
-{
-	switch (state) {
-	case IPA_USB_INVALID:
-		return "IPA_USB_INVALID";
-	case IPA_USB_INITIALIZED:
-		return "IPA_USB_INITIALIZED";
-	case IPA_USB_CONNECTED:
-		return "IPA_USB_CONNECTED";
-	case IPA_USB_STOPPED:
-		return "IPA_USB_STOPPED";
-	case IPA_USB_SUSPEND_REQUESTED:
-		return "IPA_USB_SUSPEND_REQUESTED";
-	case IPA_USB_SUSPENDED:
-		return "IPA_USB_SUSPENDED";
-	case IPA_USB_SUSPENDED_NO_RWAKEUP:
-		return "IPA_USB_SUSPENDED_NO_RWAKEUP";
-	case IPA_USB_RESUME_IN_PROGRESS:
-		return "IPA_USB_RESUME_IN_PROGRESS";
-	}
-
-	return "UNSUPPORTED";
-}
-
-static char *ipa3_usb_notify_event_to_string(enum ipa_usb_notify_event event)
-{
-	switch (event) {
-	case IPA_USB_DEVICE_READY:
-		return "IPA_USB_DEVICE_READY";
-	case IPA_USB_REMOTE_WAKEUP:
-		return "IPA_USB_REMOTE_WAKEUP";
-	case IPA_USB_SUSPEND_COMPLETED:
-		return "IPA_USB_SUSPEND_COMPLETED";
-	}
-
-	return "UNSUPPORTED";
-}
-
-static bool ipa3_usb_get_teth_port_state(void)
-{
-	if (ipa3_usb_ctx == NULL)
-		return false;
-	if (ipa3_usb_ctx->ttype_ctx[IPA_USB_TRANSPORT_TETH].state ==
-					IPA_USB_CONNECTED)
-		return true;
-	else
-		return false;
-}
-
-static bool ipa3_usb_set_state(enum ipa3_usb_state new_state, bool err_permit,
-	enum ipa3_usb_transport_type ttype)
-{
-	unsigned long flags;
-	int state_legal = false;
-	enum ipa3_usb_state state;
-	bool rwakeup_pending;
-
-	spin_lock_irqsave(&ipa3_usb_ctx->state_lock, flags);
-	state = ipa3_usb_ctx->ttype_ctx[ttype].state;
-	rwakeup_pending = ipa3_usb_ctx->ttype_ctx[ttype].rwakeup_pending;
-	switch (new_state) {
-	case IPA_USB_INVALID:
-		if (state == IPA_USB_INITIALIZED)
-			state_legal = true;
-		break;
-	case IPA_USB_INITIALIZED:
-		if (state == IPA_USB_STOPPED || state == IPA_USB_INVALID ||
-			((!IPA3_USB_IS_TTYPE_DPL(ttype)) &&
-			(state == IPA_USB_INITIALIZED)))
-			state_legal = true;
-		break;
-	case IPA_USB_CONNECTED:
-		if (state == IPA_USB_INITIALIZED ||
-			state == IPA_USB_STOPPED ||
-			state == IPA_USB_RESUME_IN_PROGRESS ||
-			state == IPA_USB_SUSPENDED_NO_RWAKEUP ||
-			/*
-			 * In case of failure during suspend request
-			 * handling, state is reverted to connected.
-			 */
-			(err_permit && state == IPA_USB_SUSPEND_REQUESTED))
-			state_legal = true;
-		break;
-	case IPA_USB_STOPPED:
-		if (state == IPA_USB_CONNECTED ||
-			state == IPA_USB_SUSPENDED ||
-			state == IPA_USB_SUSPENDED_NO_RWAKEUP)
-			state_legal = true;
-		break;
-	case IPA_USB_SUSPEND_REQUESTED:
-		if (state == IPA_USB_CONNECTED)
-			state_legal = true;
-		break;
-	case IPA_USB_SUSPENDED:
-		if (state == IPA_USB_SUSPEND_REQUESTED ||
-			/*
-			 * In case of failure during resume, state is reverted
-			 * to original, which could be suspended. Allow it
-			 */
-			(err_permit && state == IPA_USB_RESUME_IN_PROGRESS)) {
-			state_legal = true;
-			rwakeup_pending = false;
-		}
-		break;
-	case IPA_USB_SUSPENDED_NO_RWAKEUP:
-		if (state == IPA_USB_CONNECTED)
-			state_legal = true;
-		break;
-	case IPA_USB_RESUME_IN_PROGRESS:
-		if (state == IPA_USB_SUSPENDED)
-			state_legal = true;
-		break;
-	default:
-		state_legal = false;
-		break;
-
-	}
-	if (state_legal) {
-		if (state != new_state) {
-			IPA_USB_DBG("ipa_usb %s state changed %s -> %s\n",
-				IPA3_USB_IS_TTYPE_DPL(ttype) ? "DPL" : "",
-				ipa3_usb_state_to_string(state),
-				ipa3_usb_state_to_string(new_state));
-			ipa3_usb_ctx->ttype_ctx[ttype].state = new_state;
-			ipa3_usb_ctx->ttype_ctx[ttype].rwakeup_pending =
-				rwakeup_pending;
-		}
-	} else {
-		IPA_USB_ERR("invalid state change %s -> %s\n",
-			ipa3_usb_state_to_string(state),
-			ipa3_usb_state_to_string(new_state));
-	}
-
-	spin_unlock_irqrestore(&ipa3_usb_ctx->state_lock, flags);
-	return state_legal;
-}
-
-static bool ipa3_usb_check_legal_op(enum ipa3_usb_op op,
-	enum ipa3_usb_transport_type ttype)
-{
-	unsigned long flags;
-	bool is_legal = false;
-	enum ipa3_usb_state state;
-	bool is_dpl;
-
-	if (ipa3_usb_ctx == NULL) {
-		IPA_USB_ERR("ipa_usb_ctx is not initialized!\n");
-		return false;
-	}
-
-	is_dpl = IPA3_USB_IS_TTYPE_DPL(ttype);
-
-	spin_lock_irqsave(&ipa3_usb_ctx->state_lock, flags);
-	state = ipa3_usb_ctx->ttype_ctx[ttype].state;
-	switch (op) {
-	case IPA_USB_OP_INIT_TETH_PROT:
-		if (state == IPA_USB_INVALID ||
-			(!is_dpl && state == IPA_USB_INITIALIZED))
-			is_legal = true;
-		break;
-	case IPA_USB_OP_REQUEST_CHANNEL:
-		if (state == IPA_USB_INITIALIZED)
-			is_legal = true;
-		break;
-	case IPA_USB_OP_CONNECT:
-		if (state == IPA_USB_INITIALIZED || state == IPA_USB_STOPPED)
-			is_legal = true;
-		break;
-	case IPA_USB_OP_DISCONNECT:
-		if  (state == IPA_USB_CONNECTED ||
-			state == IPA_USB_SUSPENDED ||
-			state == IPA_USB_SUSPENDED_NO_RWAKEUP)
-			is_legal = true;
-		break;
-	case IPA_USB_OP_RELEASE_CHANNEL:
-		/* when releasing 1st channel state will be changed already */
-		if (state == IPA_USB_STOPPED ||
-			(!is_dpl && state == IPA_USB_INITIALIZED))
-			is_legal = true;
-		break;
-	case IPA_USB_OP_DEINIT_TETH_PROT:
-		/*
-		 * For data tethering we should allow deinit an inited protocol
-		 * always. E.g. rmnet is inited and rndis is connected.
-		 * USB can deinit rmnet first and then disconnect rndis
-		 * on cable disconnect.
-		 */
-		if (!is_dpl || state == IPA_USB_INITIALIZED)
-			is_legal = true;
-		break;
-	case IPA_USB_OP_SUSPEND:
-		if (state == IPA_USB_CONNECTED)
-			is_legal = true;
-		break;
-	case IPA_USB_OP_SUSPEND_NO_RWAKEUP:
-		if (state == IPA_USB_CONNECTED)
-			is_legal = true;
-		break;
-	case IPA_USB_OP_RESUME:
-		if (state == IPA_USB_SUSPENDED ||
-			state == IPA_USB_SUSPENDED_NO_RWAKEUP)
-			is_legal = true;
-		break;
-	default:
-		is_legal = false;
-		break;
-	}
-
-	if (!is_legal) {
-		IPA_USB_ERR("Illegal %s operation: state=%s operation=%s\n",
-			is_dpl ? "DPL" : "",
-			ipa3_usb_state_to_string(state),
-			ipa3_usb_op_to_string(op));
-	}
-
-	spin_unlock_irqrestore(&ipa3_usb_ctx->state_lock, flags);
-	return is_legal;
-}
-
-static void ipa3_usb_notify_do(enum ipa3_usb_transport_type ttype,
-	enum ipa_usb_notify_event event)
-{
-	int (*cb)(enum ipa_usb_notify_event, void *user_data);
-	void *user_data;
-	int res;
-
-	IPA_USB_DBG("Trying to notify USB with %s\n",
-		ipa3_usb_notify_event_to_string(event));
-
-	cb = ipa3_usb_ctx->ttype_ctx[ttype].ipa_usb_notify_cb;
-	user_data = ipa3_usb_ctx->ttype_ctx[ttype].user_data;
-
-	if (cb) {
-		res = cb(event, user_data);
-		IPA_USB_DBG("Notified USB with %s. is_dpl=%d result=%d\n",
-			ipa3_usb_notify_event_to_string(event),
-			IPA3_USB_IS_TTYPE_DPL(ttype), res);
-	}
-}
-
-/*
- * This call-back is called from ECM or RNDIS drivers.
- * Both drivers are data tethering drivers and not DPL
- */
-void ipa3_usb_device_ready_notify_cb(void)
-{
-	IPA_USB_DBG_LOW("entry\n");
-	ipa3_usb_notify_do(IPA_USB_TRANSPORT_TETH,
-		IPA_USB_DEVICE_READY);
-	IPA_USB_DBG_LOW("exit\n");
-}
-
-static void ipa3_usb_wq_notify_remote_wakeup(struct work_struct *work)
-{
-	bool rwakeup_pending;
-	unsigned long flags;
-	enum ipa3_usb_transport_type ttype =
-		IPA_USB_TRANSPORT_TETH;
-
-
-	spin_lock_irqsave(&ipa3_usb_ctx->state_lock, flags);
-	rwakeup_pending =
-		ipa3_usb_ctx->ttype_ctx[ttype].rwakeup_pending;
-	if (!rwakeup_pending) {
-		rwakeup_pending = true;
-		ipa3_usb_notify_do(ttype,
-			IPA_USB_REMOTE_WAKEUP);
-	}
-	ipa3_usb_ctx->ttype_ctx[ttype].rwakeup_pending =
-		rwakeup_pending;
-	spin_unlock_irqrestore(&ipa3_usb_ctx->state_lock, flags);
-}
-
-static void ipa3_usb_wq_dpl_notify_remote_wakeup(struct work_struct *work)
-{
-	bool rwakeup_pending;
-	unsigned long flags;
-	enum ipa3_usb_transport_type ttype =
-		IPA_USB_TRANSPORT_DPL;
-
-	spin_lock_irqsave(&ipa3_usb_ctx->state_lock, flags);
-	rwakeup_pending =
-		ipa3_usb_ctx->ttype_ctx[ttype].rwakeup_pending;
-	if (!rwakeup_pending) {
-		rwakeup_pending = true;
-		ipa3_usb_notify_do(ttype,
-			IPA_USB_REMOTE_WAKEUP);
-	}
-	ipa3_usb_ctx->ttype_ctx[ttype].rwakeup_pending =
-		rwakeup_pending;
-	spin_unlock_irqrestore(&ipa3_usb_ctx->state_lock, flags);
-}
-
-
-static void ipa3_usb_pm_cb(void *p, enum ipa_pm_cb_event event)
-{
-	struct ipa3_usb_transport_type_ctx *ttype_ctx =
-		(struct ipa3_usb_transport_type_ctx *)p;
-	unsigned long flags;
-
-	IPA_USB_DBG_LOW("entry\n");
-
-	if (event != IPA_PM_REQUEST_WAKEUP) {
-		IPA_USB_ERR("Unexpected event %d\n", event);
-		WARN_ON(1);
-		return;
-	}
-
-	spin_lock_irqsave(&ipa3_usb_ctx->state_lock, flags);
-	IPA_USB_DBG("state is %s\n",
-		ipa3_usb_state_to_string(ttype_ctx->state));
-	if (ttype_ctx->state == IPA_USB_SUSPENDED)
-		queue_work(ipa3_usb_ctx->wq,
-			ttype_ctx->pm_ctx.remote_wakeup_work);
-	spin_unlock_irqrestore(&ipa3_usb_ctx->state_lock, flags);
-	IPA_USB_DBG_LOW("exit\n");
-}
-
-static char *ipa3_usb_teth_prot_to_string(enum ipa_usb_teth_prot teth_prot)
-{
-	switch (teth_prot) {
-	case IPA_USB_RNDIS:
-		return "rndis_ipa";
-	case IPA_USB_ECM:
-		return "ecm_ipa";
-	case IPA_USB_RMNET:
-	case IPA_USB_MBIM:
-		return "teth_bridge";
-	case IPA_USB_DIAG:
-		return "dpl";
-	default:
-		break;
-	}
-
-	return "unsupported";
-}
-
-static char *ipa3_usb_teth_bridge_prot_to_string(
-	enum ipa_usb_teth_prot teth_prot)
-{
-	switch (teth_prot) {
-	case IPA_USB_RMNET:
-		return "rmnet";
-	case IPA_USB_MBIM:
-		return "mbim";
-	default:
-		break;
-	}
-
-	return "unsupported";
-}
-
-static int ipa3_usb_init_teth_bridge(void)
-{
-	int result;
-
-	result = teth_bridge_init(&ipa3_usb_ctx->teth_bridge_params);
-	if (result) {
-		IPA_USB_ERR("Failed to initialize teth_bridge\n");
-		return result;
-	}
-
-	return 0;
-}
-
-static int ipa3_usb_register_pm(enum ipa3_usb_transport_type ttype)
-{
-	struct ipa3_usb_transport_type_ctx *ttype_ctx =
-		&ipa3_usb_ctx->ttype_ctx[ttype];
-	int result;
-
-	/* there is one PM resource for teth and one for DPL */
-	if (!IPA3_USB_IS_TTYPE_DPL(ttype) && ipa3_usb_ctx->num_init_prot > 0)
-		return 0;
-
-	memset(&ttype_ctx->pm_ctx.reg_params, 0,
-		sizeof(ttype_ctx->pm_ctx.reg_params));
-	ttype_ctx->pm_ctx.reg_params.name = (ttype == IPA_USB_TRANSPORT_DPL) ?
-				"USB DPL" : "USB";
-	ttype_ctx->pm_ctx.reg_params.callback = ipa3_usb_pm_cb;
-	ttype_ctx->pm_ctx.reg_params.user_data = ttype_ctx;
-	ttype_ctx->pm_ctx.reg_params.group = IPA_PM_GROUP_DEFAULT;
-
-	result = ipa_pm_register(&ttype_ctx->pm_ctx.reg_params,
-		&ttype_ctx->pm_ctx.hdl);
-	if (result) {
-		IPA_USB_ERR("fail to register with PM %d\n", result);
-		goto fail_pm_reg;
-	}
-
-	result = ipa_pm_associate_ipa_cons_to_client(ttype_ctx->pm_ctx.hdl,
-		(ttype == IPA_USB_TRANSPORT_DPL) ?
-		IPA_CLIENT_USB_DPL_CONS : IPA_CLIENT_USB_CONS);
-	if (result) {
-		IPA_USB_ERR("fail to associate cons with PM %d\n", result);
-		goto fail_pm_cons;
-	}
-
-	return 0;
-
-fail_pm_cons:
-	ipa_pm_deregister(ttype_ctx->pm_ctx.hdl);
-fail_pm_reg:
-	memset(&ttype_ctx->pm_ctx.reg_params, 0,
-		sizeof(ttype_ctx->pm_ctx.reg_params));
-	return result;
-}
-
-static int ipa3_usb_deregister_pm(enum ipa3_usb_transport_type ttype)
-{
-	struct ipa3_usb_pm_context *pm_ctx =
-		&ipa3_usb_ctx->ttype_ctx[ttype].pm_ctx;
-	int result;
-
-	result = ipa_pm_deregister(pm_ctx->hdl);
-	if (result)
-		return result;
-
-	memset(&pm_ctx->reg_params, 0, sizeof(pm_ctx->reg_params));
-	return 0;
-}
-
-static bool ipa3_usb_is_teth_switch_valid(enum ipa_usb_teth_prot new_teth)
-{
-	enum ipa_usb_teth_prot old_teth;
-	u32 ipa_r_rev;
-
-	IPA_USB_DBG("Start new_teth=%s\n",
-		ipa3_usb_teth_prot_to_string(new_teth));
-
-	if (IPA3_USB_IS_TTYPE_DPL(IPA3_USB_GET_TTYPE(new_teth)))
-		return true;
-
-	if (ipa3_ctx->ipa_hw_type != IPA_HW_v4_5)
-		return true;
-
-	ipa_r_rev = ipa3_get_r_rev_version();
-	IPA_USB_DBG("ipa_r_rev=%u\n", ipa_r_rev);
-
-	/* issue relevant for IPA4.5v1 */
-	if (ipa_r_rev != 10 && ipa_r_rev != 13)
-		return true;
-
-	if (ipa3_usb_ctx == NULL) {
-		IPA_USB_ERR("Invalid context");
-		return false;
-	}
-
-	if (new_teth < 0 || new_teth >= IPA_USB_MAX_TETH_PROT_SIZE) {
-		IPA_USB_ERR("Invalid new_teth %d\n", new_teth);
-		return false;
-	}
-
-	if (!ipa3_usb_ctx->prev_teth.valid) {
-		ipa3_usb_ctx->prev_teth.teth = new_teth;
-		ipa3_usb_ctx->prev_teth.valid = true;
-		return true;
-	}
-
-	old_teth = ipa3_usb_ctx->prev_teth.teth;
-	if (teth_type_switch_tbl_ipa45[old_teth][new_teth]) {
-		ipa3_usb_ctx->prev_teth.teth = new_teth;
-		return true;
-	}
-
-	IPA_USB_DBG("Invalid teth switch %s -> %s\n",
-		ipa3_usb_teth_prot_to_string(old_teth),
-		ipa3_usb_teth_prot_to_string(new_teth));
-	return false;
-}
-
-
-static int ipa_usb_set_lock_unlock(bool is_lock)
-{
-	IPA_USB_DBG("entry\n");
-	if (is_lock)
-		mutex_lock(&ipa3_usb_ctx->general_mutex);
-	else
-		mutex_unlock(&ipa3_usb_ctx->general_mutex);
-	IPA_USB_DBG("exit\n");
-
-	return 0;
-}
-
-int ipa_usb_init_teth_prot(enum ipa_usb_teth_prot teth_prot,
-			   struct ipa_usb_teth_params *teth_params,
-			   int (*ipa_usb_notify_cb)(enum ipa_usb_notify_event,
-			   void *),
-			   void *user_data)
-{
-	int result = -EFAULT;
-	enum ipa3_usb_transport_type ttype;
-	struct ipa3_usb_teth_prot_context *teth_prot_ptr;
-
-	mutex_lock(&ipa3_usb_ctx->general_mutex);
-	IPA_USB_DBG_LOW("entry\n");
-	if (teth_prot < 0 || teth_prot >= IPA_USB_MAX_TETH_PROT_SIZE ||
-		((teth_prot == IPA_USB_RNDIS || teth_prot == IPA_USB_ECM) &&
-		teth_params == NULL) || ipa_usb_notify_cb == NULL ||
-		user_data == NULL) {
-		IPA_USB_ERR("bad parameters\n");
-		result = -EINVAL;
-		goto bad_params;
-	}
-
-	ttype = IPA3_USB_GET_TTYPE(teth_prot);
-
-	if (!ipa3_usb_check_legal_op(IPA_USB_OP_INIT_TETH_PROT, ttype)) {
-		IPA_USB_ERR("Illegal operation\n");
-		result = -EPERM;
-		goto bad_params;
-	}
-
-	/* Register with IPA PM */
-	teth_prot_ptr = &ipa3_usb_ctx->teth_prot_ctx[teth_prot];
-	result = ipa3_usb_register_pm(ttype);
-	if (result) {
-		IPA_USB_ERR("Failed registering IPA PM\n");
-		goto bad_params;
-	}
-
-	if (!ipa3_usb_ctx->ttype_ctx[ttype].ipa_usb_notify_cb) {
-		ipa3_usb_ctx->ttype_ctx[ttype].ipa_usb_notify_cb =
-			ipa_usb_notify_cb;
-	} else if (!IPA3_USB_IS_TTYPE_DPL(ttype)) {
-		if (ipa3_usb_ctx->ttype_ctx[ttype].ipa_usb_notify_cb !=
-			ipa_usb_notify_cb) {
-			IPA_USB_ERR("Got different notify_cb\n");
-			result = -EINVAL;
-			goto bad_params;
-		}
-	} else {
-		IPA_USB_ERR("Already has dpl_notify_cb\n");
-		result = -EINVAL;
-		goto bad_params;
-	}
-
-	/* Initialize tethering protocol */
-	switch (teth_prot) {
-	case IPA_USB_RNDIS:
-	case IPA_USB_ECM:
-		if (ipa3_usb_ctx->teth_prot_ctx[teth_prot].state !=
-			IPA_USB_TETH_PROT_INVALID) {
-			IPA_USB_DBG("%s already initialized\n",
-				ipa3_usb_teth_prot_to_string(teth_prot));
-			result = -EPERM;
-			goto bad_params;
-		}
-		ipa3_usb_ctx->teth_prot_ctx[teth_prot].user_data = user_data;
-		if (teth_prot == IPA_USB_RNDIS) {
-			struct ipa_usb_init_params *rndis_ptr =
-				&teth_prot_ptr->teth_prot_params.rndis;
-
-			rndis_ptr->device_ready_notify =
-				ipa3_usb_device_ready_notify_cb;
-			memcpy(rndis_ptr->host_ethaddr,
-				teth_params->host_ethaddr,
-				sizeof(teth_params->host_ethaddr));
-			memcpy(rndis_ptr->device_ethaddr,
-				teth_params->device_ethaddr,
-				sizeof(teth_params->device_ethaddr));
-
-			result = rndis_ipa_init(rndis_ptr);
-			if (result) {
-				IPA_USB_ERR("Failed to initialize %s\n",
-					ipa3_usb_teth_prot_to_string(
-					teth_prot));
-				goto teth_prot_init_fail;
-			}
-		} else {
-			struct ecm_ipa_params *ecm_ptr =
-				&teth_prot_ptr->teth_prot_params.ecm;
-
-			ecm_ptr->device_ready_notify =
-				ipa3_usb_device_ready_notify_cb;
-			memcpy(ecm_ptr->host_ethaddr,
-				teth_params->host_ethaddr,
-				sizeof(teth_params->host_ethaddr));
-			memcpy(ecm_ptr->device_ethaddr,
-				teth_params->device_ethaddr,
-				sizeof(teth_params->device_ethaddr));
-
-			result = ecm_ipa_init(ecm_ptr);
-			if (result) {
-				IPA_USB_ERR("Failed to initialize %s\n",
-					ipa3_usb_teth_prot_to_string(
-					teth_prot));
-				goto teth_prot_init_fail;
-			}
-		}
-		teth_prot_ptr->state =
-			IPA_USB_TETH_PROT_INITIALIZED;
-		ipa3_usb_ctx->num_init_prot++;
-		IPA_USB_DBG("initialized %s\n",
-			ipa3_usb_teth_prot_to_string(teth_prot));
-		break;
-	case IPA_USB_RMNET:
-	case IPA_USB_MBIM:
-		if (ipa3_usb_ctx->teth_prot_ctx[teth_prot].state !=
-			IPA_USB_TETH_PROT_INVALID) {
-			IPA_USB_DBG("%s already initialized\n",
-				ipa3_usb_teth_prot_to_string(teth_prot));
-			result = -EPERM;
-			goto bad_params;
-		}
-		ipa3_usb_ctx->teth_prot_ctx[teth_prot].user_data = user_data;
-		result = ipa3_usb_init_teth_bridge();
-		if (result)
-			goto teth_prot_init_fail;
-		ipa3_usb_ctx->teth_prot_ctx[teth_prot].state =
-			IPA_USB_TETH_PROT_INITIALIZED;
-		ipa3_usb_ctx->num_init_prot++;
-		IPA_USB_DBG("initialized %s %s\n",
-			ipa3_usb_teth_prot_to_string(teth_prot),
-			ipa3_usb_teth_bridge_prot_to_string(teth_prot));
-		/*
-		 * Register for xdci lock/unlock callback with ipa core driver.
-		 * As per use case, only register for IPA_CONS end point now.
-		 * If needed we can include the same for IPA_PROD ep.
-		 * For IPA_USB_DIAG/DPL config there will not be any UL ep.
-		 */
-		ipa3_register_client_callback(&ipa_usb_set_lock_unlock,
-			&ipa3_usb_get_teth_port_state, IPA_CLIENT_USB_PROD);
-		break;
-	case IPA_USB_DIAG:
-		if (ipa3_usb_ctx->teth_prot_ctx[teth_prot].state !=
-			IPA_USB_TETH_PROT_INVALID) {
-			IPA_USB_DBG("DPL already initialized\n");
-			result = -EPERM;
-			goto bad_params;
-		}
-		ipa3_usb_ctx->teth_prot_ctx[teth_prot].user_data = user_data;
-		ipa3_usb_ctx->teth_prot_ctx[teth_prot].state =
-			IPA_USB_TETH_PROT_INITIALIZED;
-		IPA_USB_DBG("initialized DPL\n");
-		break;
-	default:
-		IPA_USB_ERR("unexpected tethering protocol\n");
-		result = -EINVAL;
-		goto bad_params;
-	}
-
-	if (!ipa3_usb_set_state(IPA_USB_INITIALIZED, false, ttype))
-		IPA_USB_ERR("failed to change state to initialized\n");
-
-	IPA_USB_DBG_LOW("exit\n");
-	mutex_unlock(&ipa3_usb_ctx->general_mutex);
-	return 0;
-
-teth_prot_init_fail:
-	if ((IPA3_USB_IS_TTYPE_DPL(ttype))
-		|| (ipa3_usb_ctx->num_init_prot == 0)) {
-		ipa3_usb_deregister_pm(ttype);
-	}
-bad_params:
-	mutex_unlock(&ipa3_usb_ctx->general_mutex);
-	return result;
-}
-EXPORT_SYMBOL(ipa_usb_init_teth_prot);
-
-void ipa3_usb_gsi_evt_err_cb(struct gsi_evt_err_notify *notify)
-{
-	IPA_USB_DBG_LOW("entry\n");
-	if (!notify)
-		return;
-	IPA_USB_ERR("Received event error %d, description: %d\n",
-		notify->evt_id, notify->err_desc);
-	IPA_USB_DBG_LOW("exit\n");
-}
-
-void ipa3_usb_gsi_chan_err_cb(struct gsi_chan_err_notify *notify)
-{
-	IPA_USB_DBG_LOW("entry\n");
-	if (!notify)
-		return;
-	IPA_USB_ERR("Received channel error %d, description: %d\n",
-		notify->evt_id, notify->err_desc);
-	IPA_USB_DBG_LOW("exit\n");
-}
-
-static bool ipa3_usb_check_chan_params(struct ipa_usb_xdci_chan_params *params)
-{
-	IPA_USB_DBG_LOW("gevntcount_low_addr = %x\n",
-			params->gevntcount_low_addr);
-	IPA_USB_DBG_LOW("gevntcount_hi_addr = %x\n",
-			params->gevntcount_hi_addr);
-	IPA_USB_DBG_LOW("dir = %d\n", params->dir);
-	IPA_USB_DBG_LOW("xfer_ring_len = %d\n", params->xfer_ring_len);
-	IPA_USB_DBG_LOW("last_trb_addr_iova = %x\n",
-		params->xfer_scratch.last_trb_addr_iova);
-	IPA_USB_DBG_LOW("const_buffer_size = %d\n",
-		params->xfer_scratch.const_buffer_size);
-	IPA_USB_DBG_LOW("depcmd_low_addr = %x\n",
-		params->xfer_scratch.depcmd_low_addr);
-	IPA_USB_DBG_LOW("depcmd_hi_addr = %x\n",
-		params->xfer_scratch.depcmd_hi_addr);
-
-	if (params->client >= IPA_CLIENT_MAX  ||
-		params->teth_prot < 0 ||
-		params->teth_prot >= IPA_USB_MAX_TETH_PROT_SIZE ||
-		params->xfer_ring_len % GSI_CHAN_RE_SIZE_16B ||
-		params->xfer_scratch.const_buffer_size < 1 ||
-		params->xfer_scratch.const_buffer_size > 31) {
-		IPA_USB_ERR("Invalid params\n");
-		return false;
-	}
-	switch (params->teth_prot) {
-	case IPA_USB_DIAG:
-		if (!IPA_CLIENT_IS_CONS(params->client)) {
-			IPA_USB_ERR("DPL supports only DL channel\n");
-			return false;
-		}
-	case IPA_USB_RNDIS:
-	case IPA_USB_ECM:
-		if (ipa3_usb_ctx->teth_prot_ctx[params->teth_prot].state ==
-			IPA_USB_TETH_PROT_INVALID) {
-			IPA_USB_ERR("%s is not initialized\n",
-				ipa3_usb_teth_prot_to_string(
-				params->teth_prot));
-			return false;
-		}
-		break;
-	case IPA_USB_RMNET:
-	case IPA_USB_MBIM:
-		if (ipa3_usb_ctx->teth_prot_ctx[params->teth_prot].state ==
-			IPA_USB_TETH_PROT_INVALID) {
-			IPA_USB_ERR("%s is not initialized\n",
-				ipa3_usb_teth_bridge_prot_to_string(
-				params->teth_prot));
-			return false;
-		}
-		break;
-	default:
-		IPA_USB_ERR("Unknown tethering protocol (%d)\n",
-			params->teth_prot);
-		return false;
-	}
-	return true;
-}
-
-static int ipa3_usb_smmu_map_xdci_channel(
-	struct ipa_usb_xdci_chan_params *params, bool map)
-{
-	int result;
-	u32 gevntcount_r = rounddown(params->gevntcount_low_addr, PAGE_SIZE);
-	u32 xfer_scratch_r =
-		rounddown(params->xfer_scratch.depcmd_low_addr, PAGE_SIZE);
-
-	if (gevntcount_r != xfer_scratch_r) {
-		IPA_USB_ERR("No support more than 1 page map for USB regs\n");
-		WARN_ON(1);
-		return -EINVAL;
-	}
-
-	if (map) {
-		if (ipa3_usb_ctx->smmu_reg_map.cnt == 0) {
-			ipa3_usb_ctx->smmu_reg_map.addr = gevntcount_r;
-			result = ipa3_smmu_map_peer_reg(
-				ipa3_usb_ctx->smmu_reg_map.addr, true,
-				IPA_SMMU_CB_AP);
-			if (result) {
-				IPA_USB_ERR("failed to map USB regs %d\n",
-					result);
-				return result;
-			}
-		} else {
-			if (gevntcount_r != ipa3_usb_ctx->smmu_reg_map.addr) {
-				IPA_USB_ERR(
-					"No support for map different reg\n");
-				return -EINVAL;
-			}
-		}
-		ipa3_usb_ctx->smmu_reg_map.cnt++;
-	} else {
-		if (gevntcount_r != ipa3_usb_ctx->smmu_reg_map.addr) {
-			IPA_USB_ERR(
-				"No support for map different reg\n");
-			return -EINVAL;
-		}
-
-		if (ipa3_usb_ctx->smmu_reg_map.cnt == 1) {
-			result = ipa3_smmu_map_peer_reg(
-				ipa3_usb_ctx->smmu_reg_map.addr, false,
-				IPA_SMMU_CB_AP);
-			if (result) {
-				IPA_USB_ERR("failed to unmap USB regs %d\n",
-					result);
-				return result;
-			}
-		}
-		ipa3_usb_ctx->smmu_reg_map.cnt--;
-	}
-
-
-	result = ipa3_smmu_map_peer_buff(params->xfer_ring_base_addr_iova,
-		params->xfer_ring_len, map, params->sgt_xfer_rings,
-		IPA_SMMU_CB_AP);
-	if (result) {
-		IPA_USB_ERR("failed to map Xfer ring %d\n", result);
-		return result;
-	}
-
-	result = ipa3_smmu_map_peer_buff(params->data_buff_base_addr_iova,
-		params->data_buff_base_len, map, params->sgt_data_buff,
-		IPA_SMMU_CB_AP);
-	if (result) {
-		IPA_USB_ERR("failed to map TRBs buff %d\n", result);
-		return result;
-	}
-
-	return 0;
-}
-
-static int ipa3_usb_request_xdci_channel(
-	struct ipa_usb_xdci_chan_params *params,
-	enum ipa_usb_direction dir,
-	struct ipa_req_chan_out_params *out_params)
-{
-	int result = -EFAULT;
-	struct ipa_request_gsi_channel_params chan_params;
-	enum ipa3_usb_transport_type ttype;
-	enum ipa_usb_teth_prot teth_prot;
-	struct ipa_usb_init_params *rndis_ptr;
-	struct ecm_ipa_params *ecm_ptr;
-	struct ipa_usb_xdci_chan_params *xdci_ch_params;
-
-	IPA_USB_DBG_LOW("entry\n");
-	if (params == NULL || out_params == NULL ||
-		!ipa3_usb_check_chan_params(params)) {
-		IPA_USB_ERR("bad parameters\n");
-		return -EINVAL;
-	}
-
-	ttype = IPA3_USB_GET_TTYPE(params->teth_prot);
-	teth_prot = params->teth_prot;
-
-	if (!ipa3_usb_check_legal_op(IPA_USB_OP_REQUEST_CHANNEL, ttype)) {
-		IPA_USB_ERR("Illegal operation\n");
-		return -EPERM;
-	}
-
-	rndis_ptr =
-		&ipa3_usb_ctx->teth_prot_ctx[teth_prot].teth_prot_params.rndis;
-	ecm_ptr =
-		&ipa3_usb_ctx->teth_prot_ctx[teth_prot].teth_prot_params.ecm;
-
-	memset(&chan_params, 0, sizeof(struct ipa_request_gsi_channel_params));
-	memcpy(&chan_params.ipa_ep_cfg, &params->ipa_ep_cfg,
-		sizeof(struct ipa_ep_cfg));
-	chan_params.client = params->client;
-	switch (params->teth_prot) {
-	case IPA_USB_RNDIS:
-		chan_params.priv = rndis_ptr->private;
-		if (params->dir == GSI_CHAN_DIR_FROM_GSI)
-			chan_params.notify = rndis_ptr->ipa_tx_notify;
-		else
-			chan_params.notify = rndis_ptr->ipa_rx_notify;
-		chan_params.skip_ep_cfg = rndis_ptr->skip_ep_cfg;
-		break;
-	case IPA_USB_ECM:
-		chan_params.priv = ecm_ptr->private;
-		if (params->dir == GSI_CHAN_DIR_FROM_GSI)
-			chan_params.notify = ecm_ptr->ecm_ipa_tx_dp_notify;
-		else
-			chan_params.notify = ecm_ptr->ecm_ipa_rx_dp_notify;
-		chan_params.skip_ep_cfg = ecm_ptr->skip_ep_cfg;
-		break;
-	case IPA_USB_RMNET:
-	case IPA_USB_MBIM:
-		chan_params.priv =
-			ipa3_usb_ctx->teth_bridge_params.private_data;
-		chan_params.notify =
-			ipa3_usb_ctx->teth_bridge_params.usb_notify_cb;
-		chan_params.skip_ep_cfg =
-			ipa3_usb_ctx->teth_bridge_params.skip_ep_cfg;
-		break;
-	case IPA_USB_DIAG:
-		chan_params.priv = NULL;
-		chan_params.notify = NULL;
-		chan_params.skip_ep_cfg = true;
-		break;
-	default:
-		break;
-	}
-
-	result = ipa3_usb_smmu_map_xdci_channel(params, true);
-	if (result) {
-		IPA_USB_ERR("failed to smmu map %d\n", result);
-		return result;
-	}
-
-	/* store channel params for SMMU unmap */
-	if (dir == IPA_USB_DIR_UL)
-		xdci_ch_params = &ipa3_usb_ctx->ttype_ctx[ttype].ul_ch_params;
-	else
-		xdci_ch_params = &ipa3_usb_ctx->ttype_ctx[ttype].dl_ch_params;
-
-	*xdci_ch_params = *params;
-	result = ipa_smmu_store_sgt(
-		&xdci_ch_params->sgt_xfer_rings,
-		params->sgt_xfer_rings);
-	if (result)
-		return result;
-
-	result = ipa_smmu_store_sgt(
-		&xdci_ch_params->sgt_data_buff,
-		params->sgt_data_buff);
-	if (result) {
-		ipa_smmu_free_sgt(&xdci_ch_params->sgt_xfer_rings);
-		return result;
-	}
-	chan_params.keep_ipa_awake = params->keep_ipa_awake;
-	chan_params.evt_ring_params.intf = GSI_EVT_CHTYPE_XDCI_EV;
-	chan_params.evt_ring_params.intr = GSI_INTR_IRQ;
-	chan_params.evt_ring_params.re_size = GSI_EVT_RING_RE_SIZE_16B;
-	chan_params.evt_ring_params.ring_len = params->xfer_ring_len -
-		chan_params.evt_ring_params.re_size;
-	chan_params.evt_ring_params.ring_base_addr =
-		params->xfer_ring_base_addr_iova;
-	chan_params.evt_ring_params.ring_base_vaddr = NULL;
-	chan_params.evt_ring_params.int_modt = 0;
-	chan_params.evt_ring_params.int_modt = 0;
-	chan_params.evt_ring_params.intvec = 0;
-	chan_params.evt_ring_params.msi_addr = 0;
-	chan_params.evt_ring_params.rp_update_addr = 0;
-	chan_params.evt_ring_params.exclusive = true;
-	chan_params.evt_ring_params.err_cb = ipa3_usb_gsi_evt_err_cb;
-	chan_params.evt_ring_params.user_data = NULL;
-	chan_params.evt_scratch.xdci.gevntcount_low_addr =
-		params->gevntcount_low_addr;
-	chan_params.evt_scratch.xdci.gevntcount_hi_addr =
-		params->gevntcount_hi_addr;
-	chan_params.chan_params.prot = GSI_CHAN_PROT_XDCI;
-	chan_params.chan_params.dir = params->dir;
-	/* chan_id is set in ipa3_request_gsi_channel() */
-	chan_params.chan_params.re_size = GSI_CHAN_RE_SIZE_16B;
-	chan_params.chan_params.ring_len = params->xfer_ring_len;
-	chan_params.chan_params.ring_base_addr =
-		params->xfer_ring_base_addr_iova;
-	chan_params.chan_params.ring_base_vaddr = NULL;
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0)
-		chan_params.chan_params.use_db_eng = GSI_CHAN_DIRECT_MODE;
-	else
-		chan_params.chan_params.use_db_eng = GSI_CHAN_DB_MODE;
-	chan_params.chan_params.max_prefetch = GSI_ONE_PREFETCH_SEG;
-	if (params->dir == GSI_CHAN_DIR_FROM_GSI)
-		chan_params.chan_params.low_weight =
-			IPA_USB_DL_CHAN_LOW_WEIGHT;
-	else
-		chan_params.chan_params.low_weight =
-			IPA_USB_UL_CHAN_LOW_WEIGHT;
-	chan_params.chan_params.xfer_cb = NULL;
-	chan_params.chan_params.err_cb = ipa3_usb_gsi_chan_err_cb;
-	chan_params.chan_params.chan_user_data = NULL;
-	chan_params.chan_scratch.xdci.last_trb_addr =
-		params->xfer_scratch.last_trb_addr_iova;
-	/* xferrscidx will be updated later */
-	chan_params.chan_scratch.xdci.xferrscidx = 0;
-	chan_params.chan_scratch.xdci.const_buffer_size =
-		params->xfer_scratch.const_buffer_size;
-	chan_params.chan_scratch.xdci.depcmd_low_addr =
-		params->xfer_scratch.depcmd_low_addr;
-	chan_params.chan_scratch.xdci.depcmd_hi_addr =
-		params->xfer_scratch.depcmd_hi_addr;
-
-	/*
-	 * Update scratch for MCS smart prefetch:
-	 * Starting IPA4.5, smart prefetch implemented by H/W.
-	 * At IPA 4.0/4.1/4.2, we do not use MCS smart prefetch
-	 *  so keep the fields zero.
-	 */
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_0) {
-		chan_params.chan_scratch.xdci.outstanding_threshold =
-		((params->teth_prot == IPA_USB_MBIM) ? 1 : 2) *
-		chan_params.chan_params.re_size;
-	}
-	/* max_outstanding_tre is set in ipa3_request_gsi_channel() */
-
-	result = ipa3_request_gsi_channel(&chan_params, out_params);
-	if (result) {
-		IPA_USB_ERR("failed to allocate GSI channel\n");
-		ipa3_usb_smmu_map_xdci_channel(params, false);
-		return result;
-	}
-
-	IPA_USB_DBG_LOW("exit\n");
-	return 0;
-}
-
-static int ipa3_usb_release_xdci_channel(u32 clnt_hdl,
-	enum ipa_usb_direction dir,
-	enum ipa3_usb_transport_type ttype)
-{
-	int result = 0;
-	struct ipa_usb_xdci_chan_params *xdci_ch_params;
-
-	IPA_USB_DBG_LOW("entry\n");
-	if (ttype < 0 || ttype >= IPA_USB_TRANSPORT_MAX) {
-		IPA_USB_ERR("bad parameter\n");
-		return -EINVAL;
-	}
-
-	if (!ipa3_usb_check_legal_op(IPA_USB_OP_RELEASE_CHANNEL, ttype)) {
-		IPA_USB_ERR("Illegal operation\n");
-		return -EPERM;
-	}
-
-	/* Release channel */
-	result = ipa3_release_gsi_channel(clnt_hdl);
-	if (result) {
-		IPA_USB_ERR("failed to deallocate channel\n");
-		return result;
-	}
-
-	if (dir == IPA_USB_DIR_UL)
-		xdci_ch_params = &ipa3_usb_ctx->ttype_ctx[ttype].ul_ch_params;
-	else
-		xdci_ch_params = &ipa3_usb_ctx->ttype_ctx[ttype].dl_ch_params;
-
-	result = ipa3_usb_smmu_map_xdci_channel(xdci_ch_params, false);
-
-	if (xdci_ch_params->sgt_xfer_rings != NULL)
-		ipa_smmu_free_sgt(&xdci_ch_params->sgt_xfer_rings);
-	if (xdci_ch_params->sgt_data_buff != NULL)
-		ipa_smmu_free_sgt(&xdci_ch_params->sgt_data_buff);
-
-	/* Change ipa_usb state to INITIALIZED */
-	if (!ipa3_usb_set_state(IPA_USB_INITIALIZED, false, ttype))
-		IPA_USB_ERR("failed to change state to initialized\n");
-
-	IPA_USB_DBG_LOW("exit\n");
-	return 0;
-}
-
-
-static bool ipa3_usb_check_connect_params(
-	struct ipa_usb_xdci_connect_params_internal *params)
-{
-	IPA_USB_DBG_LOW("ul xferrscidx = %d\n", params->usb_to_ipa_xferrscidx);
-	IPA_USB_DBG_LOW("dl xferrscidx = %d\n", params->ipa_to_usb_xferrscidx);
-	IPA_USB_DBG_LOW("max_supported_bandwidth_mbps = %d\n",
-		params->max_supported_bandwidth_mbps);
-
-	if (params->max_pkt_size < IPA_USB_FULL_SPEED_64B  ||
-		params->max_pkt_size > IPA_USB_SUPER_SPEED_1024B  ||
-		params->ipa_to_usb_xferrscidx > 127 ||
-		(params->teth_prot != IPA_USB_DIAG &&
-		(params->usb_to_ipa_xferrscidx > 127)) ||
-		params->teth_prot < 0 ||
-		params->teth_prot >= IPA_USB_MAX_TETH_PROT_SIZE) {
-		IPA_USB_ERR("Invalid params\n");
-		return false;
-	}
-
-	if (ipa3_usb_ctx->teth_prot_ctx[params->teth_prot].state ==
-		IPA_USB_TETH_PROT_INVALID) {
-		IPA_USB_ERR("%s is not initialized\n",
-			ipa3_usb_teth_prot_to_string(
-			params->teth_prot));
-		return false;
-	}
-
-	return true;
-}
-
-static int ipa3_usb_connect_teth_bridge(
-	struct teth_bridge_connect_params *params)
-{
-	int result;
-
-	result = teth_bridge_connect(params);
-	if (result) {
-		IPA_USB_ERR("failed to connect teth_bridge (%s)\n",
-			params->tethering_mode == TETH_TETHERING_MODE_RMNET ?
-			"rmnet" : "mbim");
-		return result;
-	}
-
-	return 0;
-}
-
-
-static int ipa3_usb_connect_teth_prot(enum ipa_usb_teth_prot teth_prot)
-{
-	int result;
-	struct teth_bridge_connect_params teth_bridge_params;
-	struct ipa3_usb_teth_prot_conn_params *teth_conn_params;
-	enum ipa3_usb_transport_type ttype;
-	struct ipa3_usb_teth_prot_context *teth_prot_ptr =
-		&ipa3_usb_ctx->teth_prot_ctx[teth_prot];
-
-	IPA_USB_DBG("connecting protocol = %s\n",
-		ipa3_usb_teth_prot_to_string(teth_prot));
-
-	ttype = IPA3_USB_GET_TTYPE(teth_prot);
-
-	teth_conn_params = &(ipa3_usb_ctx->ttype_ctx[ttype].teth_conn_params);
-
-	switch (teth_prot) {
-	case IPA_USB_RNDIS:
-		if (teth_prot_ptr->state ==
-			IPA_USB_TETH_PROT_CONNECTED) {
-			IPA_USB_DBG("%s is already connected\n",
-				ipa3_usb_teth_prot_to_string(teth_prot));
-			break;
-		}
-		ipa3_usb_ctx->ttype_ctx[ttype].user_data =
-			teth_prot_ptr->user_data;
-		result = rndis_ipa_pipe_connect_notify(
-			teth_conn_params->usb_to_ipa_clnt_hdl,
-			teth_conn_params->ipa_to_usb_clnt_hdl,
-			teth_conn_params->params.max_xfer_size_bytes_to_dev,
-			teth_conn_params->params.max_packet_number_to_dev,
-			teth_conn_params->params.max_xfer_size_bytes_to_host,
-			teth_prot_ptr->teth_prot_params.rndis.private);
-		if (result) {
-			IPA_USB_ERR("failed to connect %s\n",
-				ipa3_usb_teth_prot_to_string(teth_prot));
-			ipa3_usb_ctx->ttype_ctx[ttype].user_data = NULL;
-			return result;
-		}
-		teth_prot_ptr->state =
-			IPA_USB_TETH_PROT_CONNECTED;
-		IPA_USB_DBG("%s is connected\n",
-			ipa3_usb_teth_prot_to_string(teth_prot));
-		break;
-	case IPA_USB_ECM:
-		if (teth_prot_ptr->state ==
-			IPA_USB_TETH_PROT_CONNECTED) {
-			IPA_USB_DBG("%s is already connected\n",
-				ipa3_usb_teth_prot_to_string(teth_prot));
-			break;
-		}
-		ipa3_usb_ctx->ttype_ctx[ttype].user_data =
-			teth_prot_ptr->user_data;
-		result = ecm_ipa_connect(teth_conn_params->usb_to_ipa_clnt_hdl,
-			teth_conn_params->ipa_to_usb_clnt_hdl,
-			teth_prot_ptr->teth_prot_params.ecm.private);
-		if (result) {
-			IPA_USB_ERR("failed to connect %s\n",
-				ipa3_usb_teth_prot_to_string(teth_prot));
-			ipa3_usb_ctx->ttype_ctx[ttype].user_data = NULL;
-			return result;
-		}
-		teth_prot_ptr->state =
-			IPA_USB_TETH_PROT_CONNECTED;
-		IPA_USB_DBG("%s is connected\n",
-			ipa3_usb_teth_prot_to_string(teth_prot));
-		break;
-	case IPA_USB_RMNET:
-	case IPA_USB_MBIM:
-		if (teth_prot_ptr->state ==
-			IPA_USB_TETH_PROT_CONNECTED) {
-			IPA_USB_DBG("%s is already connected\n",
-				ipa3_usb_teth_prot_to_string(teth_prot));
-			break;
-		}
-		result = ipa3_usb_init_teth_bridge();
-		if (result)
-			return result;
-
-		ipa3_usb_ctx->ttype_ctx[ttype].user_data =
-			teth_prot_ptr->user_data;
-		teth_bridge_params.ipa_usb_pipe_hdl =
-			teth_conn_params->ipa_to_usb_clnt_hdl;
-		teth_bridge_params.usb_ipa_pipe_hdl =
-			teth_conn_params->usb_to_ipa_clnt_hdl;
-		teth_bridge_params.tethering_mode =
-			(teth_prot == IPA_USB_RMNET) ?
-			(TETH_TETHERING_MODE_RMNET):(TETH_TETHERING_MODE_MBIM);
-		teth_bridge_params.client_type = IPA_CLIENT_USB_PROD;
-		result = ipa3_usb_connect_teth_bridge(&teth_bridge_params);
-		if (result) {
-			ipa3_usb_ctx->ttype_ctx[ttype].user_data = NULL;
-			return result;
-		}
-		ipa3_usb_ctx->teth_prot_ctx[teth_prot].state =
-			IPA_USB_TETH_PROT_CONNECTED;
-		ipa3_usb_notify_do(ttype, IPA_USB_DEVICE_READY);
-		IPA_USB_DBG("%s (%s) is connected\n",
-			ipa3_usb_teth_prot_to_string(teth_prot),
-			ipa3_usb_teth_bridge_prot_to_string(teth_prot));
-		break;
-	case IPA_USB_DIAG:
-		if (ipa3_usb_ctx->teth_prot_ctx[IPA_USB_DIAG].state ==
-			IPA_USB_TETH_PROT_CONNECTED) {
-			IPA_USB_DBG("%s is already connected\n",
-				ipa3_usb_teth_prot_to_string(teth_prot));
-			break;
-		}
-
-		ipa3_usb_ctx->ttype_ctx[ttype].user_data =
-			ipa3_usb_ctx->teth_prot_ctx[teth_prot].user_data;
-		ipa3_usb_ctx->teth_prot_ctx[IPA_USB_DIAG].state =
-			IPA_USB_TETH_PROT_CONNECTED;
-		ipa3_usb_notify_do(ttype, IPA_USB_DEVICE_READY);
-		IPA_USB_DBG("%s is connected\n",
-			ipa3_usb_teth_prot_to_string(teth_prot));
-		break;
-	default:
-		IPA_USB_ERR("Invalid tethering protocol\n");
-		return -EFAULT;
-	}
-
-	return 0;
-}
-
-static int ipa3_usb_disconnect_teth_bridge(void)
-{
-	int result;
-
-	result = teth_bridge_disconnect(IPA_CLIENT_USB_PROD);
-	if (result) {
-		IPA_USB_ERR("failed to disconnect teth_bridge\n");
-		return result;
-	}
-
-	return 0;
-}
-
-static int ipa3_usb_disconnect_teth_prot(enum ipa_usb_teth_prot teth_prot)
-{
-	int result = 0;
-	enum ipa3_usb_transport_type ttype;
-	struct ipa3_usb_teth_prot_context *teth_prot_ptr =
-		&ipa3_usb_ctx->teth_prot_ctx[teth_prot];
-
-	ttype = IPA3_USB_GET_TTYPE(teth_prot);
-
-	switch (teth_prot) {
-	case IPA_USB_RNDIS:
-	case IPA_USB_ECM:
-		if (ipa3_usb_ctx->teth_prot_ctx[teth_prot].state !=
-			IPA_USB_TETH_PROT_CONNECTED) {
-			IPA_USB_DBG("%s is not connected\n",
-				ipa3_usb_teth_prot_to_string(teth_prot));
-			return -EPERM;
-		}
-		if (teth_prot == IPA_USB_RNDIS) {
-			result = rndis_ipa_pipe_disconnect_notify(
-				teth_prot_ptr->teth_prot_params.rndis.private);
-		} else {
-			result = ecm_ipa_disconnect(
-				teth_prot_ptr->teth_prot_params.ecm.private);
-		}
-		if (result) {
-			IPA_USB_ERR("failed to disconnect %s\n",
-				ipa3_usb_teth_prot_to_string(teth_prot));
-			break;
-		}
-		teth_prot_ptr->state = IPA_USB_TETH_PROT_INITIALIZED;
-		IPA_USB_DBG("disconnected %s\n",
-			ipa3_usb_teth_prot_to_string(teth_prot));
-		break;
-	case IPA_USB_RMNET:
-	case IPA_USB_MBIM:
-		if (teth_prot_ptr->state != IPA_USB_TETH_PROT_CONNECTED) {
-			IPA_USB_DBG("%s (%s) is not connected\n",
-				ipa3_usb_teth_prot_to_string(teth_prot),
-				ipa3_usb_teth_bridge_prot_to_string(teth_prot));
-			return -EPERM;
-		}
-		result = ipa3_usb_disconnect_teth_bridge();
-		if (result)
-			break;
-
-		teth_prot_ptr->state = IPA_USB_TETH_PROT_INITIALIZED;
-		IPA_USB_DBG("disconnected %s (%s)\n",
-			ipa3_usb_teth_prot_to_string(teth_prot),
-			ipa3_usb_teth_bridge_prot_to_string(teth_prot));
-		break;
-	case IPA_USB_DIAG:
-		if (teth_prot_ptr->state != IPA_USB_TETH_PROT_CONNECTED) {
-			IPA_USB_DBG("%s is not connected\n",
-				ipa3_usb_teth_prot_to_string(teth_prot));
-			return -EPERM;
-		}
-		teth_prot_ptr->state = IPA_USB_TETH_PROT_INITIALIZED;
-		IPA_USB_DBG("disconnected %s\n",
-			ipa3_usb_teth_prot_to_string(teth_prot));
-		break;
-	default:
-		break;
-	}
-
-	ipa3_usb_ctx->ttype_ctx[ttype].user_data = NULL;
-	return result;
-}
-
-static int ipa3_usb_xdci_connect_internal(
-	struct ipa_usb_xdci_connect_params_internal *params)
-{
-	int result = -EFAULT;
-	enum ipa3_usb_transport_type ttype;
-	struct ipa3_usb_teth_prot_conn_params *teth_prot_ptr;
-
-	IPA_USB_DBG_LOW("entry\n");
-	if (params == NULL || !ipa3_usb_check_connect_params(params)) {
-		IPA_USB_ERR("bad parameters\n");
-		return -EINVAL;
-	}
-
-	ttype = (params->teth_prot == IPA_USB_DIAG) ? IPA_USB_TRANSPORT_DPL :
-		IPA_USB_TRANSPORT_TETH;
-
-	if (!ipa3_usb_check_legal_op(IPA_USB_OP_CONNECT, ttype)) {
-		IPA_USB_ERR("Illegal operation\n");
-		return -EPERM;
-	}
-
-	teth_prot_ptr = &ipa3_usb_ctx->ttype_ctx[ttype].teth_conn_params;
-	teth_prot_ptr->ipa_to_usb_clnt_hdl = params->ipa_to_usb_clnt_hdl;
-
-	if (!IPA3_USB_IS_TTYPE_DPL(ttype))
-		teth_prot_ptr->usb_to_ipa_clnt_hdl =
-		params->usb_to_ipa_clnt_hdl;
-	teth_prot_ptr->params = params->teth_prot_params;
-
-	/* Set EE xDCI specific scratch */
-	result = ipa3_set_usb_max_packet_size(params->max_pkt_size);
-	if (result) {
-		IPA_USB_ERR("failed setting xDCI EE scratch field\n");
-		return result;
-	}
-	/* perf profile is not set on  USB DPL pipe */
-	if (ttype != IPA_USB_TRANSPORT_DPL) {
-		result = ipa_pm_set_throughput(
-			ipa3_usb_ctx->ttype_ctx[ttype].pm_ctx.hdl,
-			params->max_supported_bandwidth_mbps);
-		if (result) {
-			IPA_USB_ERR("failed to set pm throughput\n");
-			return result;
-		}
-	}
-
-	result = ipa_pm_activate_sync(
-		ipa3_usb_ctx->ttype_ctx[ttype].pm_ctx.hdl);
-	if (result) {
-		IPA_USB_ERR("failed to activate pm\n");
-		return result;
-	}
-
-	if (params->teth_prot != IPA_USB_DIAG) {
-		/* Start UL channel */
-		result = ipa3_xdci_start(params->usb_to_ipa_clnt_hdl,
-			params->usb_to_ipa_xferrscidx,
-			params->usb_to_ipa_xferrscidx_valid);
-		if (result) {
-			IPA_USB_ERR("failed to connect UL channel\n");
-			goto connect_ul_fail;
-		}
-	}
-
-	/* Start DL/DPL channel */
-	result = ipa3_xdci_start(params->ipa_to_usb_clnt_hdl,
-		params->ipa_to_usb_xferrscidx,
-		params->ipa_to_usb_xferrscidx_valid);
-	if (result) {
-		IPA_USB_ERR("failed to connect DL/DPL channel\n");
-		goto connect_dl_fail;
-	}
-
-	/* MHIP pipe enablement */
-	if (ipa3_is_mhip_offload_enabled()) {
-		result = ipa_mpm_mhip_xdci_pipe_enable(params->teth_prot);
-		if (result) {
-			IPA_USB_ERR("failed to enable MHIP channel\n");
-			goto connect_teth_prot_fail;
-		}
-	}
-
-	/* Connect tethering protocol */
-	result = ipa3_usb_connect_teth_prot(params->teth_prot);
-	if (result) {
-		IPA_USB_ERR("failed to connect teth protocol\n");
-		goto connect_mhip_prot_fail;
-	}
-
-	if (!ipa3_usb_set_state(IPA_USB_CONNECTED, false, ttype)) {
-		IPA_USB_ERR(
-			"failed to change state to connected\n");
-		goto state_change_connected_fail;
-	}
-
-	IPA_USB_DBG_LOW("exit\n");
-	return 0;
-
-state_change_connected_fail:
-	ipa3_usb_disconnect_teth_prot(params->teth_prot);
-connect_mhip_prot_fail:
-	if (ipa3_is_mhip_offload_enabled())
-		ipa_mpm_mhip_xdci_pipe_disable(params->teth_prot);
-connect_teth_prot_fail:
-	ipa3_xdci_disconnect(params->ipa_to_usb_clnt_hdl, false, -1);
-	ipa3_reset_gsi_channel(params->ipa_to_usb_clnt_hdl);
-	ipa3_reset_gsi_event_ring(params->ipa_to_usb_clnt_hdl);
-connect_dl_fail:
-	if (params->teth_prot != IPA_USB_DIAG) {
-		ipa3_xdci_disconnect(params->usb_to_ipa_clnt_hdl, false, -1);
-		ipa3_reset_gsi_channel(params->usb_to_ipa_clnt_hdl);
-		ipa3_reset_gsi_event_ring(params->usb_to_ipa_clnt_hdl);
-	}
-connect_ul_fail:
-		ipa_pm_deactivate_sync(
-			ipa3_usb_ctx->ttype_ctx[ttype].pm_ctx.hdl);
-	return result;
-}
-
-#ifdef CONFIG_DEBUG_FS
-static char dbg_buff[IPA_USB_MAX_MSG_LEN];
-
-static int ipa3_usb_get_status_dbg_info(struct ipa3_usb_status_dbg_info *status)
-{
-	int res;
-	int i;
-	unsigned long flags;
-
-	IPA_USB_DBG_LOW("entry\n");
-
-	if (ipa3_usb_ctx == NULL) {
-		IPA_USB_ERR("IPA USB was not inited yet\n");
-		return -EFAULT;
-	}
-
-	mutex_lock(&ipa3_usb_ctx->general_mutex);
-
-	if (!status) {
-		IPA_USB_ERR("Invalid input\n");
-		res = -EINVAL;
-		goto bail;
-	}
-
-	memset(status, 0, sizeof(struct ipa3_usb_status_dbg_info));
-
-	spin_lock_irqsave(&ipa3_usb_ctx->state_lock, flags);
-	status->teth_state = ipa3_usb_state_to_string(
-		ipa3_usb_ctx->ttype_ctx[IPA_USB_TRANSPORT_TETH].state);
-	status->dpl_state = ipa3_usb_state_to_string(
-		ipa3_usb_ctx->ttype_ctx[IPA_USB_TRANSPORT_DPL].state);
-	spin_unlock_irqrestore(&ipa3_usb_ctx->state_lock, flags);
-
-	for (i = 0 ; i < IPA_USB_MAX_TETH_PROT_SIZE ; i++) {
-		if (ipa3_usb_ctx->teth_prot_ctx[i].state ==
-			IPA_USB_TETH_PROT_INITIALIZED) {
-			if ((i == IPA_USB_RMNET) || (i == IPA_USB_MBIM))
-				status->inited_prots[status->num_init_prot++] =
-					ipa3_usb_teth_bridge_prot_to_string(i);
-			else
-				status->inited_prots[status->num_init_prot++] =
-					ipa3_usb_teth_prot_to_string(i);
-		} else if (ipa3_usb_ctx->teth_prot_ctx[i].state ==
-			IPA_USB_TETH_PROT_CONNECTED) {
-			switch (i) {
-			case IPA_USB_RMNET:
-			case IPA_USB_MBIM:
-				status->teth_connected_prot =
-					ipa3_usb_teth_bridge_prot_to_string(i);
-				break;
-			case IPA_USB_DIAG:
-				status->dpl_connected_prot =
-					ipa3_usb_teth_prot_to_string(i);
-				break;
-			default:
-				status->teth_connected_prot =
-					ipa3_usb_teth_prot_to_string(i);
-			}
-		}
-	}
-
-	res = 0;
-	IPA_USB_DBG_LOW("exit\n");
-bail:
-	mutex_unlock(&ipa3_usb_ctx->general_mutex);
-	return res;
-}
-
-static ssize_t ipa3_read_usb_state_info(struct file *file, char __user *ubuf,
-		size_t count, loff_t *ppos)
-{
-	struct ipa3_usb_status_dbg_info status;
-	int result;
-	int nbytes;
-	int cnt = 0;
-	int i;
-
-	result = ipa3_usb_get_status_dbg_info(&status);
-	if (result) {
-		nbytes = scnprintf(dbg_buff, IPA_USB_MAX_MSG_LEN,
-				"Fail to read IPA USB status\n");
-		cnt += nbytes;
-	} else {
-		nbytes = scnprintf(dbg_buff, IPA_USB_MAX_MSG_LEN,
-			"Tethering Data State: %s\n"
-			"DPL State: %s\n"
-			"Protocols in Initialized State: ",
-			status.teth_state,
-			status.dpl_state);
-		cnt += nbytes;
-
-		for (i = 0 ; i < status.num_init_prot ; i++) {
-			nbytes = scnprintf(dbg_buff + cnt,
-					IPA_USB_MAX_MSG_LEN - cnt,
-					"%s ", status.inited_prots[i]);
-			cnt += nbytes;
-		}
-		nbytes = scnprintf(dbg_buff + cnt, IPA_USB_MAX_MSG_LEN - cnt,
-				status.num_init_prot ? "\n" : "None\n");
-		cnt += nbytes;
-
-		nbytes = scnprintf(dbg_buff + cnt, IPA_USB_MAX_MSG_LEN - cnt,
-				"Protocols in Connected State: ");
-		cnt += nbytes;
-		if (status.teth_connected_prot) {
-			nbytes = scnprintf(dbg_buff + cnt,
-				IPA_USB_MAX_MSG_LEN - cnt,
-				"%s ", status.teth_connected_prot);
-			cnt += nbytes;
-		}
-		if (status.dpl_connected_prot) {
-			nbytes = scnprintf(dbg_buff + cnt,
-				IPA_USB_MAX_MSG_LEN - cnt,
-				"%s ", status.dpl_connected_prot);
-			cnt += nbytes;
-		}
-		nbytes = scnprintf(dbg_buff + cnt, IPA_USB_MAX_MSG_LEN - cnt,
-				(status.teth_connected_prot ||
-				status.dpl_connected_prot) ? "\n" : "None\n");
-		cnt += nbytes;
-	}
-
-	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, cnt);
-}
-
-const struct file_operations ipa3_ipa_usb_ops = {
-	.read = ipa3_read_usb_state_info,
-};
-
-static void ipa_usb_debugfs_init(void)
-{
-	const mode_t read_only_mode = 0444;
-
-	ipa3_usb_ctx->dent = debugfs_create_dir("ipa_usb", 0);
-	if (IS_ERR(ipa3_usb_ctx->dent)) {
-		pr_err("fail to create folder in debug_fs\n");
-		return;
-	}
-
-	ipa3_usb_ctx->dfile_state_info = debugfs_create_file("state_info",
-			read_only_mode, ipa3_usb_ctx->dent, 0,
-			&ipa3_ipa_usb_ops);
-	if (!ipa3_usb_ctx->dfile_state_info ||
-		IS_ERR(ipa3_usb_ctx->dfile_state_info)) {
-		pr_err("failed to create file for state_info\n");
-		goto fail;
-	}
-
-	return;
-
-fail:
-	debugfs_remove_recursive(ipa3_usb_ctx->dent);
-	ipa3_usb_ctx->dent = NULL;
-}
-
-static void ipa_usb_debugfs_remove(void)
-{
-	if (IS_ERR(ipa3_usb_ctx->dent)) {
-		IPA_USB_ERR("ipa_usb debugfs folder was not created\n");
-		return;
-	}
-
-	debugfs_remove_recursive(ipa3_usb_ctx->dent);
-}
-#else /* CONFIG_DEBUG_FS */
-static void ipa_usb_debugfs_init(void){}
-static void ipa_usb_debugfs_remove(void){}
-#endif /* CONFIG_DEBUG_FS */
-
-int ipa_usb_xdci_connect(struct ipa_usb_xdci_chan_params *ul_chan_params,
-			 struct ipa_usb_xdci_chan_params *dl_chan_params,
-			 struct ipa_req_chan_out_params *ul_out_params,
-			 struct ipa_req_chan_out_params *dl_out_params,
-			 struct ipa_usb_xdci_connect_params *connect_params)
-{
-	int result = -EFAULT;
-	struct ipa_usb_xdci_connect_params_internal conn_params;
-
-	mutex_lock(&ipa3_usb_ctx->general_mutex);
-	IPA_USB_DBG_LOW("entry\n");
-	if (connect_params == NULL || dl_chan_params == NULL ||
-		dl_out_params == NULL ||
-		(connect_params->teth_prot != IPA_USB_DIAG &&
-		(ul_chan_params == NULL || ul_out_params == NULL))) {
-		IPA_USB_ERR("bad parameters\n");
-		result = -EINVAL;
-		goto bad_params;
-	}
-
-	if (!ipa3_usb_is_teth_switch_valid(connect_params->teth_prot)) {
-		IPA_USB_ERR("Invalid teth type switch\n");
-		goto bad_params;
-	}
-
-	if (connect_params->teth_prot != IPA_USB_DIAG) {
-		result = ipa3_usb_request_xdci_channel(ul_chan_params,
-			IPA_USB_DIR_UL, ul_out_params);
-		if (result) {
-			IPA_USB_ERR("failed to allocate UL channel\n");
-			goto bad_params;
-		}
-	}
-
-	result = ipa3_usb_request_xdci_channel(dl_chan_params, IPA_USB_DIR_DL,
-		dl_out_params);
-	if (result) {
-		IPA_USB_ERR("failed to allocate DL/DPL channel\n");
-		goto alloc_dl_chan_fail;
-	}
-
-	memset(&conn_params, 0,
-		sizeof(struct ipa_usb_xdci_connect_params_internal));
-	conn_params.max_pkt_size = connect_params->max_pkt_size;
-	conn_params.ipa_to_usb_clnt_hdl = dl_out_params->clnt_hdl;
-	conn_params.ipa_to_usb_xferrscidx =
-		connect_params->ipa_to_usb_xferrscidx;
-	conn_params.ipa_to_usb_xferrscidx_valid =
-		connect_params->ipa_to_usb_xferrscidx_valid;
-	if (connect_params->teth_prot != IPA_USB_DIAG) {
-		conn_params.usb_to_ipa_clnt_hdl = ul_out_params->clnt_hdl;
-		conn_params.usb_to_ipa_xferrscidx =
-			connect_params->usb_to_ipa_xferrscidx;
-		conn_params.usb_to_ipa_xferrscidx_valid =
-			connect_params->usb_to_ipa_xferrscidx_valid;
-	}
-	conn_params.teth_prot = connect_params->teth_prot;
-	conn_params.teth_prot_params = connect_params->teth_prot_params;
-	conn_params.max_supported_bandwidth_mbps =
-		connect_params->max_supported_bandwidth_mbps;
-	result = ipa3_usb_xdci_connect_internal(&conn_params);
-	if (result) {
-		IPA_USB_ERR("failed to connect\n");
-		goto connect_fail;
-	}
-
-	IPA_USB_DBG_LOW("exit\n");
-	mutex_unlock(&ipa3_usb_ctx->general_mutex);
-	return 0;
-
-connect_fail:
-	ipa3_usb_release_xdci_channel(dl_out_params->clnt_hdl, IPA_USB_DIR_DL,
-		IPA3_USB_GET_TTYPE(dl_chan_params->teth_prot));
-alloc_dl_chan_fail:
-	if (connect_params->teth_prot != IPA_USB_DIAG)
-		ipa3_usb_release_xdci_channel(ul_out_params->clnt_hdl,
-			IPA_USB_DIR_UL,
-			IPA3_USB_GET_TTYPE(ul_chan_params->teth_prot));
-bad_params:
-	mutex_unlock(&ipa3_usb_ctx->general_mutex);
-	return result;
-}
-EXPORT_SYMBOL(ipa_usb_xdci_connect);
-
-static int ipa3_usb_check_disconnect_prot(enum ipa_usb_teth_prot teth_prot)
-{
-	if (teth_prot < 0 || teth_prot >= IPA_USB_MAX_TETH_PROT_SIZE) {
-		IPA_USB_ERR("bad parameter\n");
-		return -EFAULT;
-	}
-
-	if (ipa3_usb_ctx->teth_prot_ctx[teth_prot].state !=
-		IPA_USB_TETH_PROT_CONNECTED) {
-		IPA_USB_ERR("%s is not connected\n",
-			ipa3_usb_teth_prot_to_string(teth_prot));
-		return -EFAULT;
-	}
-
-	return 0;
-}
-
-/* Assumes lock already acquired */
-static int ipa_usb_xdci_dismiss_channels(u32 ul_clnt_hdl, u32 dl_clnt_hdl,
-				enum ipa_usb_teth_prot teth_prot)
-{
-	int result = 0;
-	enum ipa3_usb_transport_type ttype;
-
-	ttype = IPA3_USB_GET_TTYPE(teth_prot);
-
-	IPA_USB_DBG_LOW("entry\n");
-
-	/* Reset DL channel */
-	result = ipa3_reset_gsi_channel(dl_clnt_hdl);
-	if (result) {
-		IPA_USB_ERR("failed to reset DL channel\n");
-		return result;
-	}
-
-	/* Reset DL event ring */
-	result = ipa3_reset_gsi_event_ring(dl_clnt_hdl);
-	if (result) {
-		IPA_USB_ERR("failed to reset DL event ring\n");
-		return result;
-	}
-
-	if (!IPA3_USB_IS_TTYPE_DPL(ttype)) {
-		ipa3_xdci_ep_delay_rm(ul_clnt_hdl); /* Remove ep_delay if set */
-		/* Reset UL channel */
-		result = ipa3_reset_gsi_channel(ul_clnt_hdl);
-		if (result) {
-			IPA_USB_ERR("failed to reset UL channel\n");
-			return result;
-		}
-
-		/* Reset UL event ring */
-		result = ipa3_reset_gsi_event_ring(ul_clnt_hdl);
-		if (result) {
-			IPA_USB_ERR("failed to reset UL event ring\n");
-			return result;
-		}
-	}
-
-	/* Change state to STOPPED */
-	if (!ipa3_usb_set_state(IPA_USB_STOPPED, false, ttype))
-		IPA_USB_ERR("failed to change state to stopped\n");
-
-	if (!IPA3_USB_IS_TTYPE_DPL(ttype)) {
-		result = ipa3_usb_release_xdci_channel(ul_clnt_hdl,
-			IPA_USB_DIR_UL, ttype);
-		if (result) {
-			IPA_USB_ERR("failed to release UL channel\n");
-			return result;
-		}
-	}
-
-	result = ipa3_usb_release_xdci_channel(dl_clnt_hdl,
-		IPA_USB_DIR_DL, ttype);
-	if (result) {
-		IPA_USB_ERR("failed to release DL channel\n");
-		return result;
-	}
-
-	IPA_USB_DBG_LOW("exit\n");
-
-	return 0;
-}
-
-int ipa_usb_xdci_disconnect(u32 ul_clnt_hdl, u32 dl_clnt_hdl,
-			    enum ipa_usb_teth_prot teth_prot)
-{
-	int result = 0;
-	struct ipa_ep_cfg_holb holb_cfg;
-	unsigned long flags;
-	enum ipa3_usb_state orig_state;
-	enum ipa3_usb_transport_type ttype;
-
-	mutex_lock(&ipa3_usb_ctx->general_mutex);
-	IPA_USB_DBG_LOW("entry\n");
-
-	ttype = IPA3_USB_GET_TTYPE(teth_prot);
-
-	if (!ipa3_usb_check_legal_op(IPA_USB_OP_DISCONNECT, ttype)) {
-		IPA_USB_ERR("Illegal operation\n");
-		result = -EPERM;
-		goto bad_params;
-	}
-
-	spin_lock_irqsave(&ipa3_usb_ctx->state_lock, flags);
-	if (ipa3_usb_ctx->ttype_ctx[ttype].state ==
-		IPA_USB_SUSPENDED_NO_RWAKEUP) {
-		spin_unlock_irqrestore(&ipa3_usb_ctx->state_lock, flags);
-		result = ipa_usb_xdci_dismiss_channels(ul_clnt_hdl, dl_clnt_hdl,
-			teth_prot);
-		mutex_unlock(&ipa3_usb_ctx->general_mutex);
-		return result;
-	}
-
-	if (ipa3_usb_check_disconnect_prot(teth_prot)) {
-		spin_unlock_irqrestore(&ipa3_usb_ctx->state_lock, flags);
-		result = -EINVAL;
-		goto bad_params;
-	}
-
-	if (ipa3_usb_ctx->ttype_ctx[ttype].state != IPA_USB_SUSPENDED) {
-		spin_unlock_irqrestore(&ipa3_usb_ctx->state_lock, flags);
-		/* Stop DL/DPL channel */
-		result = ipa3_xdci_disconnect(dl_clnt_hdl, false, -1);
-		if (result) {
-			IPA_USB_ERR("failed to disconnect DL/DPL channel\n");
-			goto bad_params;
-		}
-	} else {
-		spin_unlock_irqrestore(&ipa3_usb_ctx->state_lock, flags);
-		memset(&holb_cfg, 0, sizeof(holb_cfg));
-		holb_cfg.en = IPA_HOLB_TMR_EN;
-		holb_cfg.tmr_val = 0;
-		ipa3_cfg_ep_holb(dl_clnt_hdl, &holb_cfg);
-	}
-
-	spin_lock_irqsave(&ipa3_usb_ctx->state_lock, flags);
-	orig_state = ipa3_usb_ctx->ttype_ctx[ttype].state;
-	if (!IPA3_USB_IS_TTYPE_DPL(ttype)) {
-		if (orig_state != IPA_USB_SUSPENDED) {
-			spin_unlock_irqrestore(&ipa3_usb_ctx->state_lock,
-				flags);
-			/* Stop UL channel */
-			result = ipa3_xdci_disconnect(ul_clnt_hdl,
-				true,
-				ipa3_usb_ctx->qmi_req_id);
-			if (result) {
-				IPA_USB_ERR("failed disconnect UL channel\n");
-				goto bad_params;
-			}
-			ipa3_usb_ctx->qmi_req_id++;
-		} else
-			spin_unlock_irqrestore(&ipa3_usb_ctx->state_lock,
-				flags);
-	} else
-		spin_unlock_irqrestore(&ipa3_usb_ctx->state_lock, flags);
-
-	/* Stop UL/DL MHIP channels */
-	if (ipa3_is_mhip_offload_enabled()) {
-		result = ipa_mpm_mhip_xdci_pipe_disable(teth_prot);
-		if (result) {
-			IPA_USB_ERR("failed to disconnect MHIP pipe\n");
-			goto bad_params;
-		}
-	}
-
-	result = ipa_usb_xdci_dismiss_channels(ul_clnt_hdl, dl_clnt_hdl,
-			teth_prot);
-	if (result)
-		goto bad_params;
-
-	/* Disconnect tethering protocol */
-	result = ipa3_usb_disconnect_teth_prot(teth_prot);
-	if (result)
-		goto bad_params;
-
-	if (orig_state != IPA_USB_SUSPENDED) {
-		result = ipa_pm_deactivate_sync(
-			ipa3_usb_ctx->ttype_ctx[ttype].pm_ctx.hdl);
-		if (result) {
-			IPA_USB_ERR("failed to deactivate PM\n");
-			goto bad_params;
-		}
-	}
-
-	IPA_USB_DBG_LOW("exit\n");
-	mutex_unlock(&ipa3_usb_ctx->general_mutex);
-	return 0;
-
-bad_params:
-	mutex_unlock(&ipa3_usb_ctx->general_mutex);
-	return result;
-
-}
-EXPORT_SYMBOL(ipa_usb_xdci_disconnect);
-
-int ipa_usb_deinit_teth_prot(enum ipa_usb_teth_prot teth_prot)
-{
-	int result = -EFAULT;
-	enum ipa3_usb_transport_type ttype;
-	struct ipa3_usb_teth_prot_context *teth_prot_ptr;
-
-	mutex_lock(&ipa3_usb_ctx->general_mutex);
-	IPA_USB_DBG_LOW("entry\n");
-	if (teth_prot < 0 || teth_prot >= IPA_USB_MAX_TETH_PROT_SIZE) {
-		IPA_USB_ERR("bad parameters\n");
-		result = -EINVAL;
-		goto bad_params;
-	}
-
-	ttype = IPA3_USB_GET_TTYPE(teth_prot);
-
-	if (!ipa3_usb_check_legal_op(IPA_USB_OP_DEINIT_TETH_PROT, ttype)) {
-		IPA_USB_ERR("Illegal operation\n");
-		result = -EPERM;
-		goto bad_params;
-	}
-
-	/* Clean-up tethering protocol */
-	teth_prot_ptr = &ipa3_usb_ctx->teth_prot_ctx[teth_prot];
-
-	switch (teth_prot) {
-	case IPA_USB_RNDIS:
-	case IPA_USB_ECM:
-		if (teth_prot_ptr->state !=
-			IPA_USB_TETH_PROT_INITIALIZED) {
-			IPA_USB_ERR("%s is not initialized\n",
-				ipa3_usb_teth_prot_to_string(teth_prot));
-			result = -EINVAL;
-			goto bad_params;
-		}
-		if (teth_prot == IPA_USB_RNDIS)
-			rndis_ipa_cleanup(
-				teth_prot_ptr->teth_prot_params.rndis.private);
-		else
-			ecm_ipa_cleanup(
-				teth_prot_ptr->teth_prot_params.ecm.private);
-		teth_prot_ptr->user_data = NULL;
-		teth_prot_ptr->state = IPA_USB_TETH_PROT_INVALID;
-		ipa3_usb_ctx->num_init_prot--;
-		IPA_USB_DBG("deinitialized %s\n",
-			ipa3_usb_teth_prot_to_string(teth_prot));
-		break;
-	case IPA_USB_RMNET:
-	case IPA_USB_MBIM:
-		if (teth_prot_ptr->state !=
-			IPA_USB_TETH_PROT_INITIALIZED) {
-			IPA_USB_ERR("%s (%s) is not initialized\n",
-				ipa3_usb_teth_prot_to_string(teth_prot),
-				ipa3_usb_teth_bridge_prot_to_string(teth_prot));
-			result = -EINVAL;
-			goto bad_params;
-		}
-
-		teth_prot_ptr->user_data = NULL;
-		teth_prot_ptr->state = IPA_USB_TETH_PROT_INVALID;
-		ipa3_usb_ctx->num_init_prot--;
-		IPA_USB_DBG("deinitialized %s (%s)\n",
-			ipa3_usb_teth_prot_to_string(teth_prot),
-			ipa3_usb_teth_bridge_prot_to_string(teth_prot));
-		break;
-	case IPA_USB_DIAG:
-		if (teth_prot_ptr->state !=
-			IPA_USB_TETH_PROT_INITIALIZED) {
-			IPA_USB_ERR("%s is not initialized\n",
-				ipa3_usb_teth_prot_to_string(teth_prot));
-			result = -EINVAL;
-			goto bad_params;
-		}
-		teth_prot_ptr->user_data = NULL;
-		teth_prot_ptr->state = IPA_USB_TETH_PROT_INVALID;
-		IPA_USB_DBG("deinitialized %s\n",
-			ipa3_usb_teth_prot_to_string(teth_prot));
-		break;
-	default:
-		IPA_USB_ERR("unexpected tethering protocol\n");
-		result = -EINVAL;
-		goto bad_params;
-	}
-
-	if (IPA3_USB_IS_TTYPE_DPL(ttype) ||
-		(ipa3_usb_ctx->num_init_prot == 0)) {
-		if (!ipa3_usb_set_state(IPA_USB_INVALID, false, ttype))
-			IPA_USB_ERR(
-				"failed to change state to invalid\n");
-		ipa3_usb_deregister_pm(ttype);
-		ipa3_usb_ctx->ttype_ctx[ttype].ipa_usb_notify_cb = NULL;
-	}
-
-	IPA_USB_DBG_LOW("exit\n");
-	mutex_unlock(&ipa3_usb_ctx->general_mutex);
-	return 0;
-
-bad_params:
-	mutex_unlock(&ipa3_usb_ctx->general_mutex);
-	return result;
-}
-EXPORT_SYMBOL(ipa_usb_deinit_teth_prot);
-
-/* Assumes lock already acquired */
-static int ipa3_usb_suspend_no_remote_wakeup(u32 ul_clnt_hdl, u32 dl_clnt_hdl,
-	enum ipa_usb_teth_prot teth_prot)
-{
-	int result = 0;
-	enum ipa3_usb_transport_type ttype;
-
-	ttype = IPA3_USB_GET_TTYPE(teth_prot);
-
-	if (!ipa3_usb_check_legal_op(IPA_USB_OP_SUSPEND_NO_RWAKEUP, ttype)) {
-		IPA_USB_ERR("Illegal operation\n");
-		result = -EPERM;
-		goto fail_exit;
-	}
-
-	IPA_USB_DBG("Start suspend with no remote wakeup sequence: %s\n",
-		IPA3_USB_IS_TTYPE_DPL(ttype) ?
-		"DPL channel":"Data Tethering channels");
-
-	if (ipa3_usb_check_disconnect_prot(teth_prot)) {
-		result = -EINVAL;
-		goto fail_exit;
-	}
-
-	/* Stop DL/DPL channel */
-	result = ipa3_xdci_disconnect(dl_clnt_hdl, false, -1);
-	if (result) {
-		IPA_USB_ERR("failed to disconnect DL/DPL channel\n");
-		goto fail_exit;
-	}
-
-	if (!IPA3_USB_IS_TTYPE_DPL(ttype)) {
-		/* Stop UL channel */
-		result = ipa3_xdci_disconnect(ul_clnt_hdl, true,
-			ipa3_usb_ctx->qmi_req_id);
-		if (result) {
-			IPA_USB_ERR("failed disconnect UL channel\n");
-			goto start_dl;
-		}
-		ipa3_usb_ctx->qmi_req_id++;
-	}
-
-	/* Stop MHIP channel */
-	if (ipa3_is_mhip_offload_enabled()) {
-		result = ipa_mpm_mhip_xdci_pipe_disable(teth_prot);
-		if (result) {
-			IPA_USB_ERR("failed to disconnect MHIP pipe\n");
-			goto start_ul;
-		}
-	}
-
-	/* Disconnect tethering protocol */
-	result = ipa3_usb_disconnect_teth_prot(teth_prot);
-	if (result)
-		goto enable_mhip;
-
-	result = ipa_pm_deactivate_sync(
-		ipa3_usb_ctx->ttype_ctx[ttype].pm_ctx.hdl);
-	if (result) {
-		IPA_USB_ERR("failed to deactivate PM\n");
-		goto connect_teth;
-	}
-
-	/* Change ipa_usb state to SUSPENDED_NO_RWAKEUP */
-	if (!ipa3_usb_set_state(IPA_USB_SUSPENDED_NO_RWAKEUP, false, ttype))
-		IPA_USB_ERR("failed to change state to suspend no rwakeup\n");
-
-	IPA_USB_DBG_LOW("exit\n");
-	return 0;
-
-connect_teth:
-	(void)ipa3_usb_connect_teth_prot(teth_prot);
-enable_mhip:
-	if (ipa3_is_mhip_offload_enabled())
-		(void)ipa_mpm_mhip_xdci_pipe_enable(teth_prot);
-start_ul:
-	if (!IPA3_USB_IS_TTYPE_DPL(ttype))
-		(void)ipa3_xdci_connect(ul_clnt_hdl);
-start_dl:
-	(void)ipa3_xdci_connect(dl_clnt_hdl);
-fail_exit:
-	return result;
-}
-
-int ipa_usb_xdci_suspend(u32 ul_clnt_hdl, u32 dl_clnt_hdl,
-	enum ipa_usb_teth_prot teth_prot, bool with_remote_wakeup)
-{
-	int result = 0;
-	enum ipa3_usb_transport_type ttype;
-
-	mutex_lock(&ipa3_usb_ctx->general_mutex);
-	IPA_USB_DBG_LOW("entry\n");
-
-	if (teth_prot < 0 || teth_prot >= IPA_USB_MAX_TETH_PROT_SIZE) {
-		IPA_USB_ERR("bad parameters\n");
-		result = -EINVAL;
-		goto bad_params;
-	}
-
-	if (!with_remote_wakeup) {
-		result = ipa3_usb_suspend_no_remote_wakeup(ul_clnt_hdl,
-			dl_clnt_hdl, teth_prot);
-		mutex_unlock(&ipa3_usb_ctx->general_mutex);
-		return result;
-	}
-
-	ttype = IPA3_USB_GET_TTYPE(teth_prot);
-
-	if (!ipa3_usb_check_legal_op(IPA_USB_OP_SUSPEND, ttype)) {
-		IPA_USB_ERR("Illegal operation\n");
-		result = -EPERM;
-		goto bad_params;
-	}
-
-	IPA_USB_DBG("Start suspend sequence: %s\n",
-		IPA3_USB_IS_TTYPE_DPL(ttype) ?
-		"DPL channel":"Data Tethering channels");
-
-	/* Change state to SUSPEND_REQUESTED */
-	if (!ipa3_usb_set_state(IPA_USB_SUSPEND_REQUESTED, false, ttype)) {
-		IPA_USB_ERR(
-			"fail changing state to suspend_req\n");
-		result = -EFAULT;
-		goto bad_params;
-	}
-
-	/* Stop UL channel & suspend DL/DPL EP */
-	result = ipa3_xdci_suspend(ul_clnt_hdl, dl_clnt_hdl,
-		true,
-		ipa3_usb_ctx->qmi_req_id, IPA3_USB_IS_TTYPE_DPL(ttype));
-	if (result) {
-		IPA_USB_ERR("failed to suspend\n");
-		goto suspend_fail;
-	}
-	ipa3_usb_ctx->qmi_req_id++;
-
-	result = ipa_pm_deactivate_sync(
-		ipa3_usb_ctx->ttype_ctx[ttype].pm_ctx.hdl);
-	if (result) {
-		IPA_USB_ERR("failed to deactivate PM IPA client\n");
-		goto pm_deactivate_fail;
-	}
-
-	/* Change state to SUSPENDED */
-	if (!ipa3_usb_set_state(IPA_USB_SUSPENDED, false, ttype))
-		IPA_USB_ERR("failed to change state to suspended\n");
-
-
-	IPA_USB_DBG_LOW("exit\n");
-	mutex_unlock(&ipa3_usb_ctx->general_mutex);
-	return 0;
-
-pm_deactivate_fail:
-	ipa3_xdci_resume(ul_clnt_hdl, dl_clnt_hdl,
-		IPA3_USB_IS_TTYPE_DPL(ttype));
-suspend_fail:
-	/* Change state back to CONNECTED */
-	if (!ipa3_usb_set_state(IPA_USB_CONNECTED, true, ttype))
-		IPA_USB_ERR("failed to change state back to connected\n");
-bad_params:
-	mutex_unlock(&ipa3_usb_ctx->general_mutex);
-	return result;
-}
-EXPORT_SYMBOL(ipa_usb_xdci_suspend);
-
-/* Assumes lock already acquired */
-static int ipa3_usb_resume_no_remote_wakeup(u32 ul_clnt_hdl, u32 dl_clnt_hdl,
-	enum ipa_usb_teth_prot teth_prot)
-{
-	int result = -EFAULT;
-	enum ipa3_usb_transport_type ttype;
-
-	ttype = IPA3_USB_GET_TTYPE(teth_prot);
-
-	IPA_USB_DBG("Start resume with no remote wakeup sequence: %s\n",
-		IPA3_USB_IS_TTYPE_DPL(ttype) ?
-		"DPL channel":"Data Tethering channels");
-
-	/* Activate PM */
-	result = ipa_pm_activate_sync(
-		ipa3_usb_ctx->ttype_ctx[ttype].pm_ctx.hdl);
-	if (result)
-		goto fail_exit;
-
-	/* Connect tethering protocol */
-	result = ipa3_usb_connect_teth_prot(teth_prot);
-	if (result) {
-		IPA_USB_ERR("failed to connect teth protocol\n");
-		goto release_prod;
-	}
-
-	if (!IPA3_USB_IS_TTYPE_DPL(ttype)) {
-		/* Start UL channel */
-		result = ipa3_xdci_connect(ul_clnt_hdl);
-		if (result) {
-			IPA_USB_ERR("failed to start UL channel\n");
-			goto disconn_teth;
-		}
-	}
-
-	/* Start DL/DPL channel */
-	result = ipa3_xdci_connect(dl_clnt_hdl);
-	if (result) {
-		IPA_USB_ERR("failed to start DL/DPL channel\n");
-		goto stop_ul;
-	}
-
-	/* Start MHIP channel */
-	if (ipa3_is_mhip_offload_enabled()) {
-		result = ipa_mpm_mhip_xdci_pipe_enable(teth_prot);
-		if (result) {
-			IPA_USB_ERR("failed to enable MHIP pipe\n");
-			goto stop_dl;
-		}
-	}
-	/* Change state to CONNECTED */
-	if (!ipa3_usb_set_state(IPA_USB_CONNECTED, false, ttype)) {
-		IPA_USB_ERR("failed to change state to connected\n");
-		result = -EFAULT;
-		goto stop_mhip;
-	}
-
-	return 0;
-stop_mhip:
-	if (ipa3_is_mhip_offload_enabled())
-		(void)ipa_mpm_mhip_xdci_pipe_disable(teth_prot);
-stop_dl:
-	(void)ipa3_xdci_disconnect(dl_clnt_hdl, false, -1);
-stop_ul:
-	if (!IPA3_USB_IS_TTYPE_DPL(ttype)) {
-		(void)ipa3_xdci_disconnect(ul_clnt_hdl, true,
-			ipa3_usb_ctx->qmi_req_id);
-		ipa3_usb_ctx->qmi_req_id++;
-	}
-disconn_teth:
-	(void)ipa3_usb_disconnect_teth_prot(teth_prot);
-release_prod:
-	(void)ipa_pm_deactivate_sync(
-		ipa3_usb_ctx->ttype_ctx[ttype].pm_ctx.hdl);
-fail_exit:
-	return result;
-}
-
-int ipa_usb_xdci_resume(u32 ul_clnt_hdl, u32 dl_clnt_hdl,
-	enum ipa_usb_teth_prot teth_prot)
-{
-	int result = -EFAULT;
-	enum ipa3_usb_state prev_state;
-	unsigned long flags;
-	enum ipa3_usb_transport_type ttype;
-
-	mutex_lock(&ipa3_usb_ctx->general_mutex);
-	IPA_USB_DBG_LOW("entry\n");
-
-	if (teth_prot < 0 || teth_prot >= IPA_USB_MAX_TETH_PROT_SIZE) {
-		IPA_USB_ERR("bad parameters\n");
-		result = -EINVAL;
-		goto bad_params;
-	}
-
-	ttype = IPA3_USB_GET_TTYPE(teth_prot);
-
-	if (!ipa3_usb_check_legal_op(IPA_USB_OP_RESUME, ttype)) {
-		IPA_USB_ERR("Illegal operation\n");
-		result = -EPERM;
-		goto bad_params;
-	}
-
-	spin_lock_irqsave(&ipa3_usb_ctx->state_lock, flags);
-	prev_state = ipa3_usb_ctx->ttype_ctx[ttype].state;
-	spin_unlock_irqrestore(&ipa3_usb_ctx->state_lock, flags);
-	if (prev_state == IPA_USB_SUSPENDED_NO_RWAKEUP) {
-		result = ipa3_usb_resume_no_remote_wakeup(ul_clnt_hdl,
-			dl_clnt_hdl, teth_prot);
-		goto bad_params;
-	}
-
-	IPA_USB_DBG("Start resume sequence: %s\n",
-		IPA3_USB_IS_TTYPE_DPL(ttype) ?
-		"DPL channel" : "Data Tethering channels");
-
-	/* Change state to RESUME_IN_PROGRESS */
-	if (!ipa3_usb_set_state(IPA_USB_RESUME_IN_PROGRESS, false, ttype)) {
-		IPA_USB_ERR("failed to change state to resume_in_progress\n");
-		result = -EFAULT;
-		goto bad_params;
-	}
-
-	/* Activate PM */
-	result = ipa_pm_activate_sync(
-		ipa3_usb_ctx->ttype_ctx[ttype].pm_ctx.hdl);
-	if (result)
-		goto activate_pm_fail;
-
-	if (!IPA3_USB_IS_TTYPE_DPL(ttype)) {
-		/* Start UL channel */
-		result = ipa3_start_gsi_channel(ul_clnt_hdl);
-		if (result) {
-			IPA_USB_ERR("failed to start UL channel\n");
-			goto start_ul_fail;
-		}
-	}
-
-	/* Start DL/DPL channel */
-	result = ipa3_start_gsi_channel(dl_clnt_hdl);
-	if (result) {
-		IPA_USB_ERR("failed to start DL/DPL channel\n");
-		goto start_dl_fail;
-	}
-
-	/* Change state to CONNECTED */
-	if (!ipa3_usb_set_state(IPA_USB_CONNECTED, false, ttype)) {
-		IPA_USB_ERR("failed to change state to connected\n");
-		result = -EFAULT;
-		goto state_change_connected_fail;
-	}
-
-	IPA_USB_DBG_LOW("exit\n");
-	mutex_unlock(&ipa3_usb_ctx->general_mutex);
-	return 0;
-
-state_change_connected_fail:
-	result = ipa3_stop_gsi_channel(dl_clnt_hdl);
-	if (result)
-		IPA_USB_ERR("Error stopping DL/DPL channel: %d\n",
-			result);
-start_dl_fail:
-	if (!IPA3_USB_IS_TTYPE_DPL(ttype)) {
-		result = ipa3_stop_gsi_channel(ul_clnt_hdl);
-		if (result)
-			IPA_USB_ERR("Error stopping UL channel: %d\n", result);
-	}
-start_ul_fail:
-	ipa_pm_deactivate_sync(
-		ipa3_usb_ctx->ttype_ctx[ttype].pm_ctx.hdl);
-activate_pm_fail:
-	/* Change state back to prev_state */
-	if (!ipa3_usb_set_state(prev_state, true, ttype))
-		IPA_USB_ERR("failed to change state back to %s\n",
-			ipa3_usb_state_to_string(prev_state));
-bad_params:
-	mutex_unlock(&ipa3_usb_ctx->general_mutex);
-	return result;
-}
-EXPORT_SYMBOL(ipa_usb_xdci_resume);
-
-static int __init ipa3_usb_init(void)
-{
-	int i;
-	unsigned long flags;
-	int res;
-	struct ipa3_usb_pm_context *pm_ctx;
-
-	pr_debug("entry\n");
-	ipa3_usb_ctx = kzalloc(sizeof(struct ipa3_usb_context), GFP_KERNEL);
-	if (ipa3_usb_ctx == NULL) {
-		pr_err(":ipa_usb init failed\n");
-		return -ENOMEM;
-	}
-	memset(ipa3_usb_ctx, 0, sizeof(struct ipa3_usb_context));
-
-	for (i = 0; i < IPA_USB_MAX_TETH_PROT_SIZE; i++)
-		ipa3_usb_ctx->teth_prot_ctx[i].state =
-			IPA_USB_TETH_PROT_INVALID;
-	ipa3_usb_ctx->num_init_prot = 0;
-	init_completion(&ipa3_usb_ctx->dev_ready_comp);
-	ipa3_usb_ctx->qmi_req_id = 0;
-	spin_lock_init(&ipa3_usb_ctx->state_lock);
-	ipa3_usb_ctx->dl_data_pending = false;
-	mutex_init(&ipa3_usb_ctx->general_mutex);
-
-	/* init PM related members */
-	pm_ctx = &ipa3_usb_ctx->ttype_ctx[IPA_USB_TRANSPORT_TETH].pm_ctx;
-	pm_ctx->hdl = ~0;
-	pm_ctx->remote_wakeup_work = &ipa3_usb_notify_remote_wakeup_work;
-	pm_ctx = &ipa3_usb_ctx->ttype_ctx[IPA_USB_TRANSPORT_DPL].pm_ctx;
-	pm_ctx->hdl = ~0;
-	pm_ctx->remote_wakeup_work = &ipa3_usb_dpl_notify_remote_wakeup_work;
-
-	for (i = 0; i < IPA_USB_TRANSPORT_MAX; i++) {
-		ipa3_usb_ctx->ttype_ctx[i].user_data = NULL;
-	}
-
-	spin_lock_irqsave(&ipa3_usb_ctx->state_lock, flags);
-	for (i = 0; i < IPA_USB_TRANSPORT_MAX; i++) {
-		ipa3_usb_ctx->ttype_ctx[i].state = IPA_USB_INVALID;
-	}
-	spin_unlock_irqrestore(&ipa3_usb_ctx->state_lock, flags);
-
-	ipa3_usb_ctx->wq = create_singlethread_workqueue("ipa_usb_wq");
-	if (!ipa3_usb_ctx->wq) {
-		pr_err("failed to create workqueue\n");
-		res = -EFAULT;
-		goto ipa_usb_workqueue_fail;
-	}
-
-	ipa_usb_debugfs_init();
-
-	pr_info("exit: IPA_USB init success!\n");
-
-	return 0;
-
-ipa_usb_workqueue_fail:
-	pr_err("init failed (%d)\n", -res);
-	kfree(ipa3_usb_ctx);
-	return res;
-}
-
-static void ipa3_usb_exit(void)
-{
-	IPA_USB_DBG_LOW("IPA_USB exit\n");
-
-	/*
-	 * Deregister for xdci lock/unlock callback from ipa core driver.
-	 * As per use case, only deregister for IPA_CONS end point for now.
-	 * If needed we can include the same for IPA_PROD ep.
-	 * For IPA_USB_DIAG/DPL config there will not be any UL config.
-	 */
-	ipa3_deregister_client_callback(IPA_CLIENT_USB_PROD);
-
-	ipa_usb_debugfs_remove();
-	kfree(ipa3_usb_ctx);
-}
-
-/**
- * ipa3_get_usb_gsi_stats() - Query USB gsi stats from uc
- * @stats:	[inout] stats blob from client populated by driver
- *
- * Returns:	0 on success, negative on failure
- *
- * @note Cannot be called from atomic context
- *
- */
-int ipa3_get_usb_gsi_stats(struct ipa3_uc_dbg_ring_stats *stats)
-{
-	int i;
-
-	if (!ipa3_ctx->usb_ctx.dbg_stats.uc_dbg_stats_mmio) {
-		IPAERR("bad parms NULL usb_gsi_stats_mmio\n");
-		return -EINVAL;
-	}
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-	for (i = 0; i < MAX_USB_CHANNELS; i++) {
-		stats->ring[i].ringFull = ioread32(
-			ipa3_ctx->usb_ctx.dbg_stats.uc_dbg_stats_mmio
-			+ i * IPA3_UC_DEBUG_STATS_OFF +
-			IPA3_UC_DEBUG_STATS_RINGFULL_OFF);
-		stats->ring[i].ringEmpty = ioread32(
-			ipa3_ctx->usb_ctx.dbg_stats.uc_dbg_stats_mmio
-			+ i * IPA3_UC_DEBUG_STATS_OFF +
-			IPA3_UC_DEBUG_STATS_RINGEMPTY_OFF);
-		stats->ring[i].ringUsageHigh = ioread32(
-			ipa3_ctx->usb_ctx.dbg_stats.uc_dbg_stats_mmio
-			+ i * IPA3_UC_DEBUG_STATS_OFF +
-			IPA3_UC_DEBUG_STATS_RINGUSAGEHIGH_OFF);
-		stats->ring[i].ringUsageLow = ioread32(
-			ipa3_ctx->usb_ctx.dbg_stats.uc_dbg_stats_mmio
-			+ i * IPA3_UC_DEBUG_STATS_OFF +
-			IPA3_UC_DEBUG_STATS_RINGUSAGELOW_OFF);
-		stats->ring[i].RingUtilCount = ioread32(
-			ipa3_ctx->usb_ctx.dbg_stats.uc_dbg_stats_mmio
-			+ i * IPA3_UC_DEBUG_STATS_OFF +
-			IPA3_UC_DEBUG_STATS_RINGUTILCOUNT_OFF);
-	}
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-
-
-	return 0;
-}
-
-
-arch_initcall(ipa3_usb_init);
-module_exit(ipa3_usb_exit);
-
-MODULE_LICENSE("GPL v2");
-MODULE_DESCRIPTION("IPA USB client driver");

ipa/ipa_clients/ipa_wdi3.c

@@ -1,762 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2017-2019, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/ipa_wdi3.h>
-#include <linux/msm_ipa.h>
-#include <linux/string.h>
-#include "../ipa_common_i.h"
-#include "../ipa_v3/ipa_pm.h"
-#include "../ipa_v3/ipa_i.h"
-
-#define OFFLOAD_DRV_NAME "ipa_wdi"
-#define IPA_WDI_DBG(fmt, args...) \
-	do { \
-		pr_debug(OFFLOAD_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-			OFFLOAD_DRV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			OFFLOAD_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-#define IPA_WDI_DBG_LOW(fmt, args...) \
-	do { \
-		pr_debug(OFFLOAD_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			OFFLOAD_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-#define IPA_WDI_ERR(fmt, args...) \
-	do { \
-		pr_err(OFFLOAD_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-			OFFLOAD_DRV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			OFFLOAD_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-struct ipa_wdi_intf_info {
-	char netdev_name[IPA_RESOURCE_NAME_MAX];
-	u8 hdr_len;
-	u32 partial_hdr_hdl[IPA_IP_MAX];
-	struct list_head link;
-};
-
-struct ipa_wdi_context {
-	struct list_head head_intf_list;
-	struct completion wdi_completion;
-	struct mutex lock;
-	enum ipa_wdi_version wdi_version;
-	u8 is_smmu_enabled;
-	u32 tx_pipe_hdl;
-	u32 rx_pipe_hdl;
-	u8 num_sys_pipe_needed;
-	u32 sys_pipe_hdl[IPA_WDI_MAX_SUPPORTED_SYS_PIPE];
-	u32 ipa_pm_hdl;
-#ifdef IPA_WAN_MSG_IPv6_ADDR_GW_LEN
-	ipa_wdi_meter_notifier_cb wdi_notify;
-#endif
-};
-
-static struct ipa_wdi_context *ipa_wdi_ctx;
-
-int ipa_wdi_init(struct ipa_wdi_init_in_params *in,
-	struct ipa_wdi_init_out_params *out)
-{
-	struct ipa_wdi_uc_ready_params uc_ready_params;
-	struct ipa_smmu_in_params smmu_in;
-	struct ipa_smmu_out_params smmu_out;
-
-	if (ipa_wdi_ctx) {
-		IPA_WDI_ERR("ipa_wdi_ctx was initialized before\n");
-		return -EFAULT;
-	}
-
-	if (in->wdi_version > IPA_WDI_3 || in->wdi_version < IPA_WDI_1) {
-		IPA_WDI_ERR("wrong wdi version: %d\n", in->wdi_version);
-		return -EFAULT;
-	}
-
-	ipa_wdi_ctx = kzalloc(sizeof(*ipa_wdi_ctx), GFP_KERNEL);
-	if (ipa_wdi_ctx == NULL) {
-		IPA_WDI_ERR("fail to alloc wdi ctx\n");
-		return -ENOMEM;
-	}
-	mutex_init(&ipa_wdi_ctx->lock);
-	init_completion(&ipa_wdi_ctx->wdi_completion);
-	INIT_LIST_HEAD(&ipa_wdi_ctx->head_intf_list);
-
-	ipa_wdi_ctx->wdi_version = in->wdi_version;
-	uc_ready_params.notify = in->notify;
-	uc_ready_params.priv = in->priv;
-#ifdef IPA_WAN_MSG_IPv6_ADDR_GW_LEN
-	ipa_wdi_ctx->wdi_notify = in->wdi_notify;
-#endif
-
-	if (ipa_uc_reg_rdyCB(&uc_ready_params) != 0) {
-		mutex_destroy(&ipa_wdi_ctx->lock);
-		kfree(ipa_wdi_ctx);
-		ipa_wdi_ctx = NULL;
-		return -EFAULT;
-	}
-
-	out->is_uC_ready = uc_ready_params.is_uC_ready;
-
-	smmu_in.smmu_client = IPA_SMMU_WLAN_CLIENT;
-	if (ipa_get_smmu_params(&smmu_in, &smmu_out))
-		out->is_smmu_enabled = false;
-	else
-		out->is_smmu_enabled = smmu_out.smmu_enable;
-
-	ipa_wdi_ctx->is_smmu_enabled = out->is_smmu_enabled;
-
-	if (ipa3_ctx->ipa_wdi3_over_gsi)
-		out->is_over_gsi = true;
-	else
-		out->is_over_gsi = false;
-	return 0;
-}
-EXPORT_SYMBOL(ipa_wdi_init);
-
-int ipa_wdi_cleanup(void)
-{
-	struct ipa_wdi_intf_info *entry;
-	struct ipa_wdi_intf_info *next;
-
-	/* clear interface list */
-	list_for_each_entry_safe(entry, next,
-		&ipa_wdi_ctx->head_intf_list, link) {
-		list_del(&entry->link);
-		kfree(entry);
-	}
-	mutex_destroy(&ipa_wdi_ctx->lock);
-	kfree(ipa_wdi_ctx);
-	ipa_wdi_ctx = NULL;
-	return 0;
-}
-EXPORT_SYMBOL(ipa_wdi_cleanup);
-
-static int ipa_wdi_commit_partial_hdr(
-	struct ipa_ioc_add_hdr *hdr,
-	const char *netdev_name,
-	struct ipa_wdi_hdr_info *hdr_info)
-{
-	int i;
-
-	if (!hdr || !hdr_info || !netdev_name) {
-		IPA_WDI_ERR("Invalid input\n");
-		return -EINVAL;
-	}
-
-	hdr->commit = 1;
-	hdr->num_hdrs = 2;
-
-	snprintf(hdr->hdr[0].name, sizeof(hdr->hdr[0].name),
-			 "%s_ipv4", netdev_name);
-	snprintf(hdr->hdr[1].name, sizeof(hdr->hdr[1].name),
-			 "%s_ipv6", netdev_name);
-	for (i = IPA_IP_v4; i < IPA_IP_MAX; i++) {
-		hdr->hdr[i].hdr_len = hdr_info[i].hdr_len;
-		memcpy(hdr->hdr[i].hdr, hdr_info[i].hdr, hdr->hdr[i].hdr_len);
-		hdr->hdr[i].type = hdr_info[i].hdr_type;
-		hdr->hdr[i].is_partial = 1;
-		hdr->hdr[i].is_eth2_ofst_valid = 1;
-		hdr->hdr[i].eth2_ofst = hdr_info[i].dst_mac_addr_offset;
-	}
-
-	if (ipa_add_hdr(hdr)) {
-		IPA_WDI_ERR("fail to add partial headers\n");
-		return -EFAULT;
-	}
-
-	return 0;
-}
-
-int ipa_wdi_reg_intf(struct ipa_wdi_reg_intf_in_params *in)
-{
-	struct ipa_ioc_add_hdr *hdr;
-	struct ipa_wdi_intf_info *new_intf;
-	struct ipa_wdi_intf_info *entry;
-	struct ipa_tx_intf tx;
-	struct ipa_rx_intf rx;
-	struct ipa_ioc_tx_intf_prop tx_prop[2];
-	struct ipa_ioc_rx_intf_prop rx_prop[2];
-	u32 len;
-	int ret = 0;
-
-	if (in == NULL) {
-		IPA_WDI_ERR("invalid params in=%pK\n", in);
-		return -EINVAL;
-	}
-
-	if (!ipa_wdi_ctx) {
-		IPA_WDI_ERR("wdi ctx is not initialized\n");
-		return -EPERM;
-	}
-
-	IPA_WDI_DBG("register interface for netdev %s\n",
-		in->netdev_name);
-
-	mutex_lock(&ipa_wdi_ctx->lock);
-	list_for_each_entry(entry, &ipa_wdi_ctx->head_intf_list, link)
-		if (strcmp(entry->netdev_name, in->netdev_name) == 0) {
-			IPA_WDI_DBG("intf was added before.\n");
-			mutex_unlock(&ipa_wdi_ctx->lock);
-			return 0;
-		}
-
-	IPA_WDI_DBG("intf was not added before, proceed.\n");
-	new_intf = kzalloc(sizeof(*new_intf), GFP_KERNEL);
-	if (new_intf == NULL) {
-		IPA_WDI_ERR("fail to alloc new intf\n");
-		mutex_unlock(&ipa_wdi_ctx->lock);
-		return -ENOMEM;
-	}
-
-	INIT_LIST_HEAD(&new_intf->link);
-	strlcpy(new_intf->netdev_name, in->netdev_name,
-		sizeof(new_intf->netdev_name));
-	new_intf->hdr_len = in->hdr_info[0].hdr_len;
-
-	/* add partial header */
-	len = sizeof(struct ipa_ioc_add_hdr) + 2 * sizeof(struct ipa_hdr_add);
-	hdr = kzalloc(len, GFP_KERNEL);
-	if (hdr == NULL) {
-		IPA_WDI_ERR("fail to alloc %d bytes\n", len);
-		ret = -EFAULT;
-		goto fail_alloc_hdr;
-	}
-
-	if (ipa_wdi_commit_partial_hdr(hdr, in->netdev_name, in->hdr_info)) {
-		IPA_WDI_ERR("fail to commit partial headers\n");
-		ret = -EFAULT;
-		goto fail_commit_hdr;
-	}
-
-	new_intf->partial_hdr_hdl[IPA_IP_v4] = hdr->hdr[IPA_IP_v4].hdr_hdl;
-	new_intf->partial_hdr_hdl[IPA_IP_v6] = hdr->hdr[IPA_IP_v6].hdr_hdl;
-	IPA_WDI_DBG("IPv4 hdr hdl: %d IPv6 hdr hdl: %d\n",
-		hdr->hdr[IPA_IP_v4].hdr_hdl, hdr->hdr[IPA_IP_v6].hdr_hdl);
-
-	/* populate tx prop */
-	tx.num_props = 2;
-	tx.prop = tx_prop;
-
-	memset(tx_prop, 0, sizeof(tx_prop));
-	tx_prop[0].ip = IPA_IP_v4;
-	if (!ipa3_ctx->ipa_wdi3_over_gsi)
-		tx_prop[0].dst_pipe = IPA_CLIENT_WLAN1_CONS;
-	else
-		tx_prop[0].dst_pipe = IPA_CLIENT_WLAN2_CONS;
-	tx_prop[0].alt_dst_pipe = in->alt_dst_pipe;
-	tx_prop[0].hdr_l2_type = in->hdr_info[0].hdr_type;
-	strlcpy(tx_prop[0].hdr_name, hdr->hdr[IPA_IP_v4].name,
-		sizeof(tx_prop[0].hdr_name));
-
-	tx_prop[1].ip = IPA_IP_v6;
-	if (!ipa3_ctx->ipa_wdi3_over_gsi)
-		tx_prop[1].dst_pipe = IPA_CLIENT_WLAN1_CONS;
-	else
-		tx_prop[1].dst_pipe = IPA_CLIENT_WLAN2_CONS;
-	tx_prop[1].alt_dst_pipe = in->alt_dst_pipe;
-	tx_prop[1].hdr_l2_type = in->hdr_info[1].hdr_type;
-	strlcpy(tx_prop[1].hdr_name, hdr->hdr[IPA_IP_v6].name,
-		sizeof(tx_prop[1].hdr_name));
-
-	/* populate rx prop */
-	rx.num_props = 2;
-	rx.prop = rx_prop;
-	memset(rx_prop, 0, sizeof(rx_prop));
-	rx_prop[0].ip = IPA_IP_v4;
-	if (!ipa3_ctx->ipa_wdi3_over_gsi)
-		rx_prop[0].src_pipe = IPA_CLIENT_WLAN1_PROD;
-	else
-		rx_prop[0].src_pipe = IPA_CLIENT_WLAN2_PROD;
-	rx_prop[0].hdr_l2_type = in->hdr_info[0].hdr_type;
-	if (in->is_meta_data_valid) {
-		rx_prop[0].attrib.attrib_mask |= IPA_FLT_META_DATA;
-		rx_prop[0].attrib.meta_data = in->meta_data;
-		rx_prop[0].attrib.meta_data_mask = in->meta_data_mask;
-	}
-
-	rx_prop[1].ip = IPA_IP_v6;
-	if (!ipa3_ctx->ipa_wdi3_over_gsi)
-		rx_prop[1].src_pipe = IPA_CLIENT_WLAN1_PROD;
-	else
-		rx_prop[1].src_pipe = IPA_CLIENT_WLAN2_PROD;
-	rx_prop[1].hdr_l2_type = in->hdr_info[1].hdr_type;
-	if (in->is_meta_data_valid) {
-		rx_prop[1].attrib.attrib_mask |= IPA_FLT_META_DATA;
-		rx_prop[1].attrib.meta_data = in->meta_data;
-		rx_prop[1].attrib.meta_data_mask = in->meta_data_mask;
-	}
-
-	if (ipa_register_intf(in->netdev_name, &tx, &rx)) {
-		IPA_WDI_ERR("fail to add interface prop\n");
-		ret = -EFAULT;
-		goto fail_commit_hdr;
-	}
-
-	list_add(&new_intf->link, &ipa_wdi_ctx->head_intf_list);
-	init_completion(&ipa_wdi_ctx->wdi_completion);
-
-	kfree(hdr);
-	mutex_unlock(&ipa_wdi_ctx->lock);
-	return 0;
-
-fail_commit_hdr:
-	kfree(hdr);
-fail_alloc_hdr:
-	kfree(new_intf);
-	mutex_unlock(&ipa_wdi_ctx->lock);
-	return ret;
-}
-EXPORT_SYMBOL(ipa_wdi_reg_intf);
-
-int ipa_wdi_dereg_intf(const char *netdev_name)
-{
-	int len, ret = 0;
-	struct ipa_ioc_del_hdr *hdr = NULL;
-	struct ipa_wdi_intf_info *entry;
-	struct ipa_wdi_intf_info *next;
-
-	if (!netdev_name) {
-		IPA_WDI_ERR("no netdev name.\n");
-		return -EINVAL;
-	}
-
-	if (!ipa_wdi_ctx) {
-		IPA_WDI_ERR("wdi ctx is not initialized.\n");
-		return -EPERM;
-	}
-
-	mutex_lock(&ipa_wdi_ctx->lock);
-	list_for_each_entry_safe(entry, next, &ipa_wdi_ctx->head_intf_list,
-		link)
-		if (strcmp(entry->netdev_name, netdev_name) == 0) {
-			len = sizeof(struct ipa_ioc_del_hdr) +
-				2 * sizeof(struct ipa_hdr_del);
-			hdr = kzalloc(len, GFP_KERNEL);
-			if (hdr == NULL) {
-				IPA_WDI_ERR("fail to alloc %d bytes\n", len);
-				mutex_unlock(&ipa_wdi_ctx->lock);
-				return -ENOMEM;
-			}
-
-			hdr->commit = 1;
-			hdr->num_hdls = 2;
-			hdr->hdl[0].hdl = entry->partial_hdr_hdl[0];
-			hdr->hdl[1].hdl = entry->partial_hdr_hdl[1];
-			IPA_WDI_DBG("IPv4 hdr hdl: %d IPv6 hdr hdl: %d\n",
-				hdr->hdl[0].hdl, hdr->hdl[1].hdl);
-
-			if (ipa_del_hdr(hdr)) {
-				IPA_WDI_ERR("fail to delete partial header\n");
-				ret = -EFAULT;
-				goto fail;
-			}
-
-			if (ipa_deregister_intf(entry->netdev_name)) {
-				IPA_WDI_ERR("fail to del interface props\n");
-				ret = -EFAULT;
-				goto fail;
-			}
-
-			list_del(&entry->link);
-			kfree(entry);
-
-			break;
-		}
-
-fail:
-	kfree(hdr);
-	mutex_unlock(&ipa_wdi_ctx->lock);
-	return ret;
-}
-EXPORT_SYMBOL(ipa_wdi_dereg_intf);
-
-
-static void ipa_wdi_pm_cb(void *p, enum ipa_pm_cb_event event)
-{
-	IPA_WDI_DBG("received pm event %d\n", event);
-}
-
-int ipa_wdi_conn_pipes(struct ipa_wdi_conn_in_params *in,
-			struct ipa_wdi_conn_out_params *out)
-{
-	int i, j, ret = 0;
-	struct ipa_pm_register_params pm_params;
-	struct ipa_wdi_in_params in_tx;
-	struct ipa_wdi_in_params in_rx;
-	struct ipa_wdi_out_params out_tx;
-	struct ipa_wdi_out_params out_rx;
-
-	if (!(in && out)) {
-		IPA_WDI_ERR("empty parameters. in=%pK out=%pK\n", in, out);
-		return -EINVAL;
-	}
-
-	if (!ipa_wdi_ctx) {
-		IPA_WDI_ERR("wdi ctx is not initialized\n");
-		return -EPERM;
-	}
-
-	if (in->num_sys_pipe_needed > IPA_WDI_MAX_SUPPORTED_SYS_PIPE) {
-		IPA_WDI_ERR("ipa can only support up to %d sys pipe\n",
-			IPA_WDI_MAX_SUPPORTED_SYS_PIPE);
-		return -EINVAL;
-	}
-	ipa_wdi_ctx->num_sys_pipe_needed = in->num_sys_pipe_needed;
-	IPA_WDI_DBG("number of sys pipe %d\n", in->num_sys_pipe_needed);
-
-	/* setup sys pipe when needed */
-	for (i = 0; i < ipa_wdi_ctx->num_sys_pipe_needed; i++) {
-		ret = ipa_setup_sys_pipe(&in->sys_in[i],
-			&ipa_wdi_ctx->sys_pipe_hdl[i]);
-		if (ret) {
-			IPA_WDI_ERR("fail to setup sys pipe %d\n", i);
-			ret = -EFAULT;
-			goto fail_setup_sys_pipe;
-		}
-	}
-
-	memset(&pm_params, 0, sizeof(pm_params));
-	pm_params.name = "wdi";
-	pm_params.callback = ipa_wdi_pm_cb;
-	pm_params.user_data = NULL;
-	pm_params.group = IPA_PM_GROUP_DEFAULT;
-	if (ipa_pm_register(&pm_params, &ipa_wdi_ctx->ipa_pm_hdl)) {
-		IPA_WDI_ERR("fail to register ipa pm\n");
-		ret = -EFAULT;
-		goto fail_setup_sys_pipe;
-	}
-
-	if (ipa_wdi_ctx->wdi_version == IPA_WDI_3) {
-		if (ipa_conn_wdi_pipes(in, out, ipa_wdi_ctx->wdi_notify)) {
-			IPA_WDI_ERR("fail to setup wdi pipes\n");
-			ret = -EFAULT;
-			goto fail_connect_pipe;
-		}
-	} else {
-		memset(&in_tx, 0, sizeof(in_tx));
-		memset(&in_rx, 0, sizeof(in_rx));
-		memset(&out_tx, 0, sizeof(out_tx));
-		memset(&out_rx, 0, sizeof(out_rx));
-#ifdef IPA_WAN_MSG_IPv6_ADDR_GW_LEN
-		in_rx.wdi_notify = ipa_wdi_ctx->wdi_notify;
-#endif
-		if (in->is_smmu_enabled == false) {
-			/* firsr setup rx pipe */
-			in_rx.sys.ipa_ep_cfg = in->u_rx.rx.ipa_ep_cfg;
-			in_rx.sys.client = in->u_rx.rx.client;
-			in_rx.sys.notify = in->notify;
-			in_rx.sys.priv = in->priv;
-			in_rx.smmu_enabled = in->is_smmu_enabled;
-			in_rx.u.ul.rdy_ring_base_pa =
-				in->u_rx.rx.transfer_ring_base_pa;
-			in_rx.u.ul.rdy_ring_size =
-				in->u_rx.rx.transfer_ring_size;
-			in_rx.u.ul.rdy_ring_rp_pa =
-				in->u_rx.rx.transfer_ring_doorbell_pa;
-			in_rx.u.ul.rdy_comp_ring_base_pa =
-				in->u_rx.rx.event_ring_base_pa;
-			in_rx.u.ul.rdy_comp_ring_wp_pa =
-				in->u_rx.rx.event_ring_doorbell_pa;
-			in_rx.u.ul.rdy_comp_ring_size =
-				in->u_rx.rx.event_ring_size;
-			if (ipa_connect_wdi_pipe(&in_rx, &out_rx)) {
-				IPA_WDI_ERR("fail to setup rx pipe\n");
-				ret = -EFAULT;
-				goto fail_connect_pipe;
-			}
-			ipa_wdi_ctx->rx_pipe_hdl = out_rx.clnt_hdl;
-			out->rx_uc_db_pa = out_rx.uc_door_bell_pa;
-			IPA_WDI_DBG("rx uc db pa: 0x%pad\n", &out->rx_uc_db_pa);
-
-			/* then setup tx pipe */
-			in_tx.sys.ipa_ep_cfg = in->u_tx.tx.ipa_ep_cfg;
-			in_tx.sys.client = in->u_tx.tx.client;
-			in_tx.smmu_enabled = in->is_smmu_enabled;
-			in_tx.u.dl.comp_ring_base_pa =
-				in->u_tx.tx.transfer_ring_base_pa;
-			in_tx.u.dl.comp_ring_size =
-				in->u_tx.tx.transfer_ring_size;
-			in_tx.u.dl.ce_ring_base_pa =
-				in->u_tx.tx.event_ring_base_pa;
-			in_tx.u.dl.ce_door_bell_pa =
-				in->u_tx.tx.event_ring_doorbell_pa;
-			in_tx.u.dl.ce_ring_size =
-				in->u_tx.tx.event_ring_size;
-			in_tx.u.dl.num_tx_buffers =
-				in->u_tx.tx.num_pkt_buffers;
-			if (ipa_connect_wdi_pipe(&in_tx, &out_tx)) {
-				IPA_WDI_ERR("fail to setup tx pipe\n");
-				ret = -EFAULT;
-				goto fail;
-			}
-			ipa_wdi_ctx->tx_pipe_hdl = out_tx.clnt_hdl;
-			out->tx_uc_db_pa = out_tx.uc_door_bell_pa;
-			IPA_WDI_DBG("tx uc db pa: 0x%pad\n", &out->tx_uc_db_pa);
-		} else { /* smmu is enabled */
-			/* firsr setup rx pipe */
-			in_rx.sys.ipa_ep_cfg = in->u_rx.rx_smmu.ipa_ep_cfg;
-			in_rx.sys.client = in->u_rx.rx_smmu.client;
-			in_rx.sys.notify = in->notify;
-			in_rx.sys.priv = in->priv;
-			in_rx.smmu_enabled = in->is_smmu_enabled;
-			in_rx.u.ul_smmu.rdy_ring =
-				in->u_rx.rx_smmu.transfer_ring_base;
-			in_rx.u.ul_smmu.rdy_ring_size =
-				in->u_rx.rx_smmu.transfer_ring_size;
-			in_rx.u.ul_smmu.rdy_ring_rp_pa =
-				in->u_rx.rx_smmu.transfer_ring_doorbell_pa;
-			in_rx.u.ul_smmu.rdy_comp_ring =
-				in->u_rx.rx_smmu.event_ring_base;
-			in_rx.u.ul_smmu.rdy_comp_ring_wp_pa =
-				in->u_rx.rx_smmu.event_ring_doorbell_pa;
-			in_rx.u.ul_smmu.rdy_comp_ring_size =
-				in->u_rx.rx_smmu.event_ring_size;
-			if (ipa_connect_wdi_pipe(&in_rx, &out_rx)) {
-				IPA_WDI_ERR("fail to setup rx pipe\n");
-				ret = -EFAULT;
-				goto fail_connect_pipe;
-			}
-			ipa_wdi_ctx->rx_pipe_hdl = out_rx.clnt_hdl;
-			out->rx_uc_db_pa = out_rx.uc_door_bell_pa;
-			IPA_WDI_DBG("rx uc db pa: 0x%pad\n", &out->rx_uc_db_pa);
-
-			/* then setup tx pipe */
-			in_tx.sys.ipa_ep_cfg = in->u_tx.tx_smmu.ipa_ep_cfg;
-			in_tx.sys.client = in->u_tx.tx_smmu.client;
-			in_tx.smmu_enabled = in->is_smmu_enabled;
-			in_tx.u.dl_smmu.comp_ring =
-				in->u_tx.tx_smmu.transfer_ring_base;
-			in_tx.u.dl_smmu.comp_ring_size =
-				in->u_tx.tx_smmu.transfer_ring_size;
-			in_tx.u.dl_smmu.ce_ring =
-				in->u_tx.tx_smmu.event_ring_base;
-			in_tx.u.dl_smmu.ce_door_bell_pa =
-				in->u_tx.tx_smmu.event_ring_doorbell_pa;
-			in_tx.u.dl_smmu.ce_ring_size =
-				in->u_tx.tx_smmu.event_ring_size;
-			in_tx.u.dl_smmu.num_tx_buffers =
-				in->u_tx.tx_smmu.num_pkt_buffers;
-			if (ipa_connect_wdi_pipe(&in_tx, &out_tx)) {
-				IPA_WDI_ERR("fail to setup tx pipe\n");
-				ret = -EFAULT;
-				goto fail;
-			}
-			ipa_wdi_ctx->tx_pipe_hdl = out_tx.clnt_hdl;
-			out->tx_uc_db_pa = out_tx.uc_door_bell_pa;
-			IPA_WDI_DBG("tx uc db pa: 0x%pad\n", &out->tx_uc_db_pa);
-		}
-	}
-
-	return 0;
-
-fail:
-	ipa_disconnect_wdi_pipe(ipa_wdi_ctx->rx_pipe_hdl);
-fail_connect_pipe:
-	ipa_pm_deregister(ipa_wdi_ctx->ipa_pm_hdl);
-
-fail_setup_sys_pipe:
-	for (j = 0; j < i; j++)
-		ipa_teardown_sys_pipe(ipa_wdi_ctx->sys_pipe_hdl[j]);
-	return ret;
-}
-EXPORT_SYMBOL(ipa_wdi_conn_pipes);
-
-int ipa_wdi_disconn_pipes(void)
-{
-	int i, ipa_ep_idx_rx, ipa_ep_idx_tx;
-
-	if (!ipa_wdi_ctx) {
-		IPA_WDI_ERR("wdi ctx is not initialized\n");
-		return -EPERM;
-	}
-
-	/* tear down sys pipe if needed */
-	for (i = 0; i < ipa_wdi_ctx->num_sys_pipe_needed; i++) {
-		if (ipa_teardown_sys_pipe(ipa_wdi_ctx->sys_pipe_hdl[i])) {
-			IPA_WDI_ERR("fail to tear down sys pipe %d\n", i);
-			return -EFAULT;
-		}
-	}
-
-	if (!ipa3_ctx->ipa_wdi3_over_gsi) {
-		ipa_ep_idx_rx = ipa_get_ep_mapping(IPA_CLIENT_WLAN1_PROD);
-		ipa_ep_idx_tx = ipa_get_ep_mapping(IPA_CLIENT_WLAN1_CONS);
-	} else {
-		ipa_ep_idx_rx = ipa_get_ep_mapping(IPA_CLIENT_WLAN2_PROD);
-		ipa_ep_idx_tx = ipa_get_ep_mapping(IPA_CLIENT_WLAN2_CONS);
-	}
-
-	if (ipa_wdi_ctx->wdi_version == IPA_WDI_3) {
-		if (ipa_disconn_wdi_pipes(ipa_ep_idx_rx, ipa_ep_idx_tx)) {
-			IPA_WDI_ERR("fail to tear down wdi pipes\n");
-			return -EFAULT;
-		}
-	} else {
-		if (ipa_disconnect_wdi_pipe(ipa_wdi_ctx->tx_pipe_hdl)) {
-			IPA_WDI_ERR("fail to tear down wdi tx pipes\n");
-			return -EFAULT;
-		}
-		if (ipa_disconnect_wdi_pipe(ipa_wdi_ctx->rx_pipe_hdl)) {
-			IPA_WDI_ERR("fail to tear down wdi rx pipes\n");
-			return -EFAULT;
-		}
-	}
-
-	if (ipa_pm_deregister(ipa_wdi_ctx->ipa_pm_hdl)) {
-		IPA_WDI_ERR("fail to deregister ipa pm\n");
-		return -EFAULT;
-	}
-
-	return 0;
-}
-EXPORT_SYMBOL(ipa_wdi_disconn_pipes);
-
-int ipa_wdi_enable_pipes(void)
-{
-	int ret;
-	int ipa_ep_idx_tx, ipa_ep_idx_rx;
-
-	if (!ipa_wdi_ctx) {
-		IPA_WDI_ERR("wdi ctx is not initialized.\n");
-		return -EPERM;
-	}
-
-	if (!ipa3_ctx->ipa_wdi3_over_gsi) {
-		ipa_ep_idx_rx = ipa_get_ep_mapping(IPA_CLIENT_WLAN1_PROD);
-		ipa_ep_idx_tx = ipa_get_ep_mapping(IPA_CLIENT_WLAN1_CONS);
-	} else {
-		ipa_ep_idx_rx = ipa_get_ep_mapping(IPA_CLIENT_WLAN2_PROD);
-		ipa_ep_idx_tx = ipa_get_ep_mapping(IPA_CLIENT_WLAN2_CONS);
-	}
-
-	if (ipa_wdi_ctx->wdi_version == IPA_WDI_3) {
-		if (ipa_enable_wdi_pipes(ipa_ep_idx_tx, ipa_ep_idx_rx)) {
-			IPA_WDI_ERR("fail to enable wdi pipes\n");
-			return -EFAULT;
-		}
-	} else {
-		if (ipa_enable_wdi_pipe(ipa_wdi_ctx->tx_pipe_hdl)) {
-			IPA_WDI_ERR("fail to enable wdi tx pipe\n");
-			return -EFAULT;
-		}
-		if (ipa_resume_wdi_pipe(ipa_wdi_ctx->tx_pipe_hdl)) {
-			IPA_WDI_ERR("fail to resume wdi tx pipe\n");
-			return -EFAULT;
-		}
-		if (ipa_enable_wdi_pipe(ipa_wdi_ctx->rx_pipe_hdl)) {
-			IPA_WDI_ERR("fail to enable wdi rx pipe\n");
-			return -EFAULT;
-		}
-		if (ipa_resume_wdi_pipe(ipa_wdi_ctx->rx_pipe_hdl)) {
-			IPA_WDI_ERR("fail to resume wdi rx pipe\n");
-			return -EFAULT;
-		}
-	}
-
-	ret = ipa_pm_activate_sync(ipa_wdi_ctx->ipa_pm_hdl);
-	if (ret) {
-		IPA_WDI_ERR("fail to activate ipa pm\n");
-		return -EFAULT;
-	}
-
-	return 0;
-}
-EXPORT_SYMBOL(ipa_wdi_enable_pipes);
-
-int ipa_wdi_disable_pipes(void)
-{
-	int ret;
-	int ipa_ep_idx_tx, ipa_ep_idx_rx;
-
-	if (!ipa_wdi_ctx) {
-		IPA_WDI_ERR("wdi ctx is not initialized.\n");
-		return -EPERM;
-	}
-
-	if (!ipa3_ctx->ipa_wdi3_over_gsi) {
-		ipa_ep_idx_rx = ipa_get_ep_mapping(IPA_CLIENT_WLAN1_PROD);
-		ipa_ep_idx_tx = ipa_get_ep_mapping(IPA_CLIENT_WLAN1_CONS);
-	} else {
-		ipa_ep_idx_rx = ipa_get_ep_mapping(IPA_CLIENT_WLAN2_PROD);
-		ipa_ep_idx_tx = ipa_get_ep_mapping(IPA_CLIENT_WLAN2_CONS);
-	}
-
-	if (ipa_wdi_ctx->wdi_version == IPA_WDI_3) {
-		if (ipa_disable_wdi_pipes(ipa_ep_idx_tx, ipa_ep_idx_rx)) {
-			IPA_WDI_ERR("fail to disable wdi pipes\n");
-			return -EFAULT;
-		}
-	} else {
-		if (ipa_suspend_wdi_pipe(ipa_wdi_ctx->tx_pipe_hdl)) {
-			IPA_WDI_ERR("fail to suspend wdi tx pipe\n");
-			return -EFAULT;
-		}
-		if (ipa_disable_wdi_pipe(ipa_wdi_ctx->tx_pipe_hdl)) {
-			IPA_WDI_ERR("fail to disable wdi tx pipe\n");
-			return -EFAULT;
-		}
-		if (ipa_suspend_wdi_pipe(ipa_wdi_ctx->rx_pipe_hdl)) {
-			IPA_WDI_ERR("fail to suspend wdi rx pipe\n");
-			return -EFAULT;
-		}
-		if (ipa_disable_wdi_pipe(ipa_wdi_ctx->rx_pipe_hdl)) {
-			IPA_WDI_ERR("fail to disable wdi rx pipe\n");
-			return -EFAULT;
-		}
-	}
-
-	ret = ipa_pm_deactivate_sync(ipa_wdi_ctx->ipa_pm_hdl);
-	if (ret) {
-		IPA_WDI_ERR("fail to deactivate ipa pm\n");
-		return -EFAULT;
-	}
-
-	return 0;
-}
-EXPORT_SYMBOL(ipa_wdi_disable_pipes);
-
-int ipa_wdi_set_perf_profile(struct ipa_wdi_perf_profile *profile)
-{
-	if (profile == NULL) {
-		IPA_WDI_ERR("Invalid input\n");
-		return -EINVAL;
-	}
-
-	if (ipa_pm_set_throughput(ipa_wdi_ctx->ipa_pm_hdl,
-		profile->max_supported_bw_mbps)) {
-		IPA_WDI_ERR("fail to set pm throughput\n");
-		return -EFAULT;
-	}
-
-	return 0;
-}
-EXPORT_SYMBOL(ipa_wdi_set_perf_profile);
-
-int ipa_wdi_create_smmu_mapping(u32 num_buffers,
-	struct ipa_wdi_buffer_info *info)
-{
-	return ipa_create_wdi_mapping(num_buffers, info);
-}
-EXPORT_SYMBOL(ipa_wdi_create_smmu_mapping);
-
-int ipa_wdi_release_smmu_mapping(u32 num_buffers,
-	struct ipa_wdi_buffer_info *info)
-{
-	return ipa_release_wdi_mapping(num_buffers, info);
-}
-EXPORT_SYMBOL(ipa_wdi_release_smmu_mapping);
-
-int ipa_wdi_get_stats(struct IpaHwStatsWDIInfoData_t *stats)
-{
-	return ipa_get_wdi_stats(stats);
-}
-EXPORT_SYMBOL(ipa_wdi_get_stats);

ipa/ipa_clients/ipa_wigig.c

@@ -1,2090 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/ipa_wigig.h>
-#include <linux/debugfs.h>
-#include <linux/string.h>
-#include "../ipa_common_i.h"
-#include "../ipa_v3/ipa_pm.h"
-
-#define OFFLOAD_DRV_NAME "ipa_wigig"
-#define IPA_WIGIG_DBG(fmt, args...) \
-	do { \
-		pr_debug(OFFLOAD_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-			OFFLOAD_DRV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			OFFLOAD_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-#define IPA_WIGIG_DBG_LOW(fmt, args...) \
-	do { \
-		pr_debug(OFFLOAD_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			OFFLOAD_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-#define IPA_WIGIG_ERR(fmt, args...) \
-	do { \
-		pr_err(OFFLOAD_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-			OFFLOAD_DRV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			OFFLOAD_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-#define IPA_WIGIG_ERR_RL(fmt, args...) \
-	do { \
-		pr_err_ratelimited_ipa( \
-		OFFLOAD_DRV_NAME " %s:%d " fmt, __func__,\
-		__LINE__, ## args);\
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			OFFLOAD_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-
-#define IPA_WIGIG_TX_PIPE_NUM	4
-
-enum ipa_wigig_pipes_idx {
-	IPA_CLIENT_WIGIG_PROD_IDX = 0,
-	IPA_CLIENT_WIGIG1_CONS_IDX = 1,
-	IPA_CLIENT_WIGIG2_CONS_IDX = 2,
-	IPA_CLIENT_WIGIG3_CONS_IDX = 3,
-	IPA_CLIENT_WIGIG4_CONS_IDX = 4,
-	IPA_WIGIG_MAX_PIPES
-};
-
-struct ipa_wigig_intf_info {
-	char netdev_name[IPA_RESOURCE_NAME_MAX];
-	u8 netdev_mac[IPA_MAC_ADDR_SIZE];
-	u8 hdr_len;
-	u32 partial_hdr_hdl[IPA_IP_MAX];
-	struct list_head link;
-};
-
-struct ipa_wigig_pipe_values {
-	uint8_t dir;
-	uint8_t tx_ring_id;
-	uint32_t desc_ring_HWHEAD;
-	uint32_t desc_ring_HWTAIL;
-	uint32_t status_ring_HWHEAD;
-	uint32_t status_ring_HWTAIL;
-};
-
-struct ipa_wigig_regs_save {
-	struct ipa_wigig_pipe_values pipes_val[IPA_WIGIG_MAX_PIPES];
-	u32 int_gen_tx_val;
-	u32 int_gen_rx_val;
-};
-
-struct ipa_wigig_context {
-	struct list_head head_intf_list;
-	struct mutex lock;
-	u32 ipa_pm_hdl;
-	phys_addr_t periph_baddr_pa;
-	phys_addr_t pseudo_cause_pa;
-	phys_addr_t int_gen_tx_pa;
-	phys_addr_t int_gen_rx_pa;
-	phys_addr_t dma_ep_misc_pa;
-	ipa_notify_cb tx_notify;
-	void *priv;
-	union pipes {
-		struct ipa_wigig_pipe_setup_info flat[IPA_WIGIG_MAX_PIPES];
-		struct ipa_wigig_pipe_setup_info_smmu
-			smmu[IPA_WIGIG_MAX_PIPES];
-	} pipes;
-	struct ipa_wigig_rx_pipe_data_buffer_info_smmu rx_buff_smmu;
-	struct ipa_wigig_tx_pipe_data_buffer_info_smmu
-		tx_buff_smmu[IPA_WIGIG_TX_PIPE_NUM];
-	char clients_mac[IPA_WIGIG_TX_PIPE_NUM][IPA_MAC_ADDR_SIZE];
-	struct ipa_wigig_regs_save regs_save;
-	bool smmu_en;
-	bool shared_cb;
-	u8 conn_pipes;
-	struct dentry *parent;
-	struct dentry *dent_conn_clients;
-	struct dentry *dent_smmu;
-};
-
-static struct ipa_wigig_context *ipa_wigig_ctx;
-
-#ifdef CONFIG_DEBUG_FS
-static int ipa_wigig_init_debugfs(struct dentry *parent);
-static inline void ipa_wigig_deinit_debugfs(void);
-#else
-static int ipa_wigig_init_debugfs(struct dentry *parent) { return 0; }
-static inline void ipa_wigig_deinit_debugfs(void) { }
-#endif
-
-int ipa_wigig_init(struct ipa_wigig_init_in_params *in,
-	struct ipa_wigig_init_out_params *out)
-{
-	struct ipa_wdi_uc_ready_params inout;
-
-	if (!in || !out) {
-		IPA_WIGIG_ERR("invalid params in=%pK, out %pK\n", in, out);
-		return -EINVAL;
-	}
-
-	IPA_WIGIG_DBG("\n");
-	if (ipa_wigig_ctx) {
-		IPA_WIGIG_ERR("ipa_wigig_ctx was initialized before\n");
-		return -EINVAL;
-	}
-
-	ipa_wigig_ctx = kzalloc(sizeof(*ipa_wigig_ctx), GFP_KERNEL);
-	if (ipa_wigig_ctx == NULL)
-		return -ENOMEM;
-
-	mutex_init(&ipa_wigig_ctx->lock);
-	INIT_LIST_HEAD(&ipa_wigig_ctx->head_intf_list);
-
-	ipa_wigig_ctx->pseudo_cause_pa = in->pseudo_cause_pa;
-	ipa_wigig_ctx->int_gen_tx_pa = in->int_gen_tx_pa;
-	ipa_wigig_ctx->int_gen_rx_pa = in->int_gen_rx_pa;
-	ipa_wigig_ctx->dma_ep_misc_pa = in->dma_ep_misc_pa;
-	ipa_wigig_ctx->periph_baddr_pa = in->periph_baddr_pa;
-
-	IPA_WIGIG_DBG(
-		"periph_baddr_pa 0x%pa pseudo_cause_pa 0x%pa, int_gen_tx_pa 0x%pa, int_gen_rx_pa 0x%pa, dma_ep_misc_pa 0x%pa"
-		, &ipa_wigig_ctx->periph_baddr_pa,
-		&ipa_wigig_ctx->pseudo_cause_pa,
-		&ipa_wigig_ctx->int_gen_tx_pa,
-		&ipa_wigig_ctx->int_gen_rx_pa,
-		&ipa_wigig_ctx->dma_ep_misc_pa);
-
-	inout.notify = in->notify;
-	inout.priv = in->priv;
-	if (ipa_wigig_internal_init(&inout, in->int_notify,
-		&out->uc_db_pa)) {
-		kfree(ipa_wigig_ctx);
-		ipa_wigig_ctx = NULL;
-		return -EFAULT;
-	}
-
-	IPA_WIGIG_DBG("uc_db_pa 0x%pa\n", &out->uc_db_pa);
-
-	out->is_uc_ready = inout.is_uC_ready;
-
-	IPA_WIGIG_DBG("exit\n");
-
-	return 0;
-}
-EXPORT_SYMBOL(ipa_wigig_init);
-
-int ipa_wigig_cleanup(void)
-{
-	struct ipa_wigig_intf_info *entry;
-	struct ipa_wigig_intf_info *next;
-
-	IPA_WIGIG_DBG("\n");
-
-	if (!ipa_wigig_ctx)
-		return -ENODEV;
-
-	/* clear interface list */
-	list_for_each_entry_safe(entry, next,
-		&ipa_wigig_ctx->head_intf_list, link) {
-		list_del(&entry->link);
-		kfree(entry);
-	}
-
-	mutex_destroy(&ipa_wigig_ctx->lock);
-
-	ipa_wigig_deinit_debugfs();
-
-	kfree(ipa_wigig_ctx);
-	ipa_wigig_ctx = NULL;
-
-	IPA_WIGIG_DBG("exit\n");
-	return 0;
-}
-EXPORT_SYMBOL(ipa_wigig_cleanup);
-
-bool ipa_wigig_is_smmu_enabled(void)
-{
-	struct ipa_smmu_in_params in;
-	struct ipa_smmu_out_params out;
-
-	IPA_WIGIG_DBG("\n");
-
-	in.smmu_client = IPA_SMMU_WIGIG_CLIENT;
-	ipa_get_smmu_params(&in, &out);
-
-	IPA_WIGIG_DBG("exit (%d)\n", out.smmu_enable);
-
-	return out.smmu_enable;
-}
-EXPORT_SYMBOL(ipa_wigig_is_smmu_enabled);
-
-static int ipa_wigig_init_smmu_params(void)
-{
-	struct ipa_smmu_in_params in;
-	struct ipa_smmu_out_params out;
-	int ret;
-
-	IPA_WIGIG_DBG("\n");
-
-	in.smmu_client = IPA_SMMU_WIGIG_CLIENT;
-	ret = ipa_get_smmu_params(&in, &out);
-	if (ret) {
-		IPA_WIGIG_ERR("couldn't get SMMU params %d\n", ret);
-		return ret;
-	}
-	ipa_wigig_ctx->smmu_en = out.smmu_enable;
-	ipa_wigig_ctx->shared_cb = out.shared_cb;
-	IPA_WIGIG_DBG("SMMU (%s), 11ad CB (%s)\n",
-		out.smmu_enable ? "enabled" : "disabled",
-		out.shared_cb ? "shared" : "not shared");
-
-	return 0;
-}
-
-static int ipa_wigig_commit_partial_hdr(
-	struct ipa_ioc_add_hdr *hdr,
-	const char *netdev_name,
-	struct ipa_wigig_hdr_info *hdr_info)
-{
-	int i;
-
-	IPA_WIGIG_DBG("\n");
-
-	if (!netdev_name) {
-		IPA_WIGIG_ERR("Invalid input\n");
-		return -EINVAL;
-	}
-
-	IPA_WIGIG_DBG("dst_mac_addr_offset %d hdr_len %d hdr_type %d\n",
-		hdr_info->dst_mac_addr_offset,
-		hdr_info->hdr_len,
-		hdr_info->hdr_type);
-
-	hdr->commit = 1;
-	hdr->num_hdrs = 2;
-
-	snprintf(hdr->hdr[0].name, sizeof(hdr->hdr[0].name),
-		"%s_ipv4", netdev_name);
-	snprintf(hdr->hdr[1].name, sizeof(hdr->hdr[1].name),
-		"%s_ipv6", netdev_name);
-	for (i = IPA_IP_v4; i < IPA_IP_MAX; i++) {
-		hdr->hdr[i].hdr_len = hdr_info[i].hdr_len;
-		memcpy(hdr->hdr[i].hdr, hdr_info[i].hdr, hdr->hdr[i].hdr_len);
-		hdr->hdr[i].type = hdr_info[i].hdr_type;
-		hdr->hdr[i].is_partial = 1;
-		hdr->hdr[i].is_eth2_ofst_valid = 1;
-		hdr->hdr[i].eth2_ofst = hdr_info[i].dst_mac_addr_offset;
-	}
-
-	if (ipa_add_hdr(hdr)) {
-		IPA_WIGIG_ERR("fail to add partial headers\n");
-		return -EFAULT;
-	}
-
-	IPA_WIGIG_DBG("exit\n");
-
-	return 0;
-}
-
-static void ipa_wigig_free_msg(void *msg, uint32_t len, uint32_t type)
-{
-	IPA_WIGIG_DBG("free msg type:%d, len:%d, buff %pK", type, len, msg);
-	kfree(msg);
-	IPA_WIGIG_DBG("exit\n");
-}
-
-static int ipa_wigig_send_wlan_msg(enum ipa_wlan_event msg_type,
-	const char *netdev_name, u8 *mac)
-{
-	struct ipa_msg_meta msg_meta;
-	struct ipa_wlan_msg *wlan_msg;
-	int ret;
-
-	IPA_WIGIG_DBG("%d\n", msg_type);
-
-	wlan_msg = kzalloc(sizeof(*wlan_msg), GFP_KERNEL);
-	if (wlan_msg == NULL)
-		return -ENOMEM;
-	strlcpy(wlan_msg->name, netdev_name, IPA_RESOURCE_NAME_MAX);
-	memcpy(wlan_msg->mac_addr, mac, IPA_MAC_ADDR_SIZE);
-	msg_meta.msg_len = sizeof(struct ipa_wlan_msg);
-	msg_meta.msg_type = msg_type;
-
-	IPA_WIGIG_DBG("send msg type:%d, len:%d, buff %pK", msg_meta.msg_type,
-		msg_meta.msg_len, wlan_msg);
-	ret = ipa_send_msg(&msg_meta, wlan_msg, ipa_wigig_free_msg);
-
-	IPA_WIGIG_DBG("exit\n");
-
-	return ret;
-}
-
-int ipa_wigig_send_msg(int msg_type,
-	const char *netdev_name, u8 *mac,
-	enum ipa_client_type client, bool to_wigig)
-{
-	struct ipa_msg_meta msg_meta;
-	struct ipa_wigig_msg *wigig_msg;
-	int ret;
-
-	IPA_WIGIG_DBG("\n");
-
-	wigig_msg = kzalloc(sizeof(struct ipa_wigig_msg), GFP_KERNEL);
-	if (wigig_msg == NULL)
-		return -ENOMEM;
-	strlcpy(wigig_msg->name, netdev_name, IPA_RESOURCE_NAME_MAX);
-	memcpy(wigig_msg->client_mac_addr, mac, IPA_MAC_ADDR_SIZE);
-	if (msg_type == WIGIG_CLIENT_CONNECT)
-		wigig_msg->u.ipa_client = client;
-	else
-		wigig_msg->u.to_wigig = to_wigig;
-
-	msg_meta.msg_type = msg_type;
-	msg_meta.msg_len = sizeof(struct ipa_wigig_msg);
-
-	IPA_WIGIG_DBG("send msg type:%d, len:%d, buff %pK", msg_meta.msg_type,
-		msg_meta.msg_len, wigig_msg);
-	ret = ipa_send_msg(&msg_meta, wigig_msg, ipa_wigig_free_msg);
-
-	IPA_WIGIG_DBG("exit\n");
-
-	return ret;
-}
-
-static int ipa_wigig_get_devname(char *netdev_name)
-{
-	struct ipa_wigig_intf_info *entry;
-
-	mutex_lock(&ipa_wigig_ctx->lock);
-
-	if (!list_is_singular(&ipa_wigig_ctx->head_intf_list)) {
-		IPA_WIGIG_DBG("list is not singular, was an IF registered?\n");
-		mutex_unlock(&ipa_wigig_ctx->lock);
-		return -EFAULT;
-	}
-	entry = list_first_entry(&ipa_wigig_ctx->head_intf_list,
-		struct ipa_wigig_intf_info,
-		link);
-	strlcpy(netdev_name, entry->netdev_name, IPA_RESOURCE_NAME_MAX);
-
-	mutex_unlock(&ipa_wigig_ctx->lock);
-
-	return 0;
-}
-
-int ipa_wigig_reg_intf(
-	struct ipa_wigig_reg_intf_in_params *in)
-{
-	struct ipa_wigig_intf_info *new_intf;
-	struct ipa_wigig_intf_info *entry;
-	struct ipa_tx_intf tx;
-	struct ipa_rx_intf rx;
-	struct ipa_ioc_tx_intf_prop tx_prop[2];
-	struct ipa_ioc_rx_intf_prop rx_prop[2];
-	struct ipa_ioc_add_hdr *hdr;
-	struct ipa_ioc_del_hdr *del_hdr = NULL;
-	u32 len;
-	int ret = 0;
-
-	IPA_WIGIG_DBG("\n");
-
-	if (in == NULL) {
-		IPA_WIGIG_ERR("invalid params in=%pK\n", in);
-		return -EINVAL;
-	}
-
-	if (!ipa_wigig_ctx) {
-		IPA_WIGIG_ERR("wigig ctx is not initialized\n");
-		return -EPERM;
-	}
-
-	IPA_WIGIG_DBG(
-		"register interface for netdev %s, MAC 0x[%X][%X][%X][%X][%X][%X]\n"
-		, in->netdev_name,
-		in->netdev_mac[0], in->netdev_mac[1], in->netdev_mac[2],
-		in->netdev_mac[3], in->netdev_mac[4], in->netdev_mac[5]);
-
-	mutex_lock(&ipa_wigig_ctx->lock);
-	list_for_each_entry(entry, &ipa_wigig_ctx->head_intf_list, link)
-		if (strcmp(entry->netdev_name, in->netdev_name) == 0) {
-			IPA_WIGIG_DBG("intf was added before.\n");
-			mutex_unlock(&ipa_wigig_ctx->lock);
-			return 0;
-		}
-
-	IPA_WIGIG_DBG("intf was not added before, proceed.\n");
-	new_intf = kzalloc(sizeof(*new_intf), GFP_KERNEL);
-	if (new_intf == NULL) {
-		ret = -ENOMEM;
-		goto fail;
-	}
-
-	INIT_LIST_HEAD(&new_intf->link);
-	strlcpy(new_intf->netdev_name, in->netdev_name,
-		sizeof(new_intf->netdev_name));
-	new_intf->hdr_len = in->hdr_info[0].hdr_len;
-	memcpy(new_intf->netdev_mac, in->netdev_mac, IPA_MAC_ADDR_SIZE);
-
-	/* add partial header */
-	len = sizeof(struct ipa_ioc_add_hdr) + 2 * sizeof(struct ipa_hdr_add);
-	hdr = kzalloc(len, GFP_KERNEL);
-	if (hdr == NULL) {
-		ret = -EFAULT;
-		goto fail_alloc_hdr;
-	}
-
-	if (ipa_wigig_commit_partial_hdr(hdr,
-		in->netdev_name,
-		in->hdr_info)) {
-		IPA_WIGIG_ERR("fail to commit partial headers\n");
-		ret = -EFAULT;
-		goto fail_commit_hdr;
-	}
-
-	new_intf->partial_hdr_hdl[IPA_IP_v4] = hdr->hdr[IPA_IP_v4].hdr_hdl;
-	new_intf->partial_hdr_hdl[IPA_IP_v6] = hdr->hdr[IPA_IP_v6].hdr_hdl;
-	IPA_WIGIG_DBG("IPv4 hdr hdl: %d IPv6 hdr hdl: %d\n",
-		hdr->hdr[IPA_IP_v4].hdr_hdl, hdr->hdr[IPA_IP_v6].hdr_hdl);
-
-	/* populate tx prop */
-	tx.num_props = 2;
-	tx.prop = tx_prop;
-
-	memset(tx_prop, 0, sizeof(tx_prop));
-	tx_prop[0].ip = IPA_IP_v4;
-	/*
-	 * for consumers, we register a default pipe, but IPACM will determine
-	 * the actual pipe according to the relevant client MAC
-	 */
-	tx_prop[0].dst_pipe = IPA_CLIENT_WIGIG1_CONS;
-	tx_prop[0].hdr_l2_type = in->hdr_info[0].hdr_type;
-	strlcpy(tx_prop[0].hdr_name, hdr->hdr[IPA_IP_v4].name,
-		sizeof(tx_prop[0].hdr_name));
-
-	tx_prop[1].ip = IPA_IP_v6;
-	tx_prop[1].dst_pipe = IPA_CLIENT_WIGIG1_CONS;
-	tx_prop[1].hdr_l2_type = in->hdr_info[1].hdr_type;
-	strlcpy(tx_prop[1].hdr_name, hdr->hdr[IPA_IP_v6].name,
-		sizeof(tx_prop[1].hdr_name));
-
-	/* populate rx prop */
-	rx.num_props = 2;
-	rx.prop = rx_prop;
-
-	memset(rx_prop, 0, sizeof(rx_prop));
-	rx_prop[0].ip = IPA_IP_v4;
-	rx_prop[0].src_pipe = IPA_CLIENT_WIGIG_PROD;
-	rx_prop[0].hdr_l2_type = in->hdr_info[0].hdr_type;
-
-	rx_prop[1].ip = IPA_IP_v6;
-	rx_prop[1].src_pipe = IPA_CLIENT_WIGIG_PROD;
-	rx_prop[1].hdr_l2_type = in->hdr_info[1].hdr_type;
-
-	if (ipa_register_intf(in->netdev_name, &tx, &rx)) {
-		IPA_WIGIG_ERR("fail to add interface prop\n");
-		ret = -EFAULT;
-		goto fail_register;
-	}
-
-	if (ipa_wigig_send_wlan_msg(WLAN_AP_CONNECT,
-		in->netdev_name,
-		in->netdev_mac)) {
-		IPA_WIGIG_ERR("couldn't send msg to IPACM\n");
-		ret = -EFAULT;
-		goto fail_sendmsg;
-	}
-
-	list_add(&new_intf->link, &ipa_wigig_ctx->head_intf_list);
-
-	kfree(hdr);
-	mutex_unlock(&ipa_wigig_ctx->lock);
-
-	IPA_WIGIG_DBG("exit\n");
-	return 0;
-fail_sendmsg:
-	ipa_deregister_intf(in->netdev_name);
-fail_register:
-	del_hdr = kzalloc(sizeof(struct ipa_ioc_del_hdr) +
-		2 * sizeof(struct ipa_hdr_del), GFP_KERNEL);
-	if (del_hdr) {
-		del_hdr->commit = 1;
-		del_hdr->num_hdls = 2;
-		del_hdr->hdl[0].hdl = new_intf->partial_hdr_hdl[IPA_IP_v4];
-		del_hdr->hdl[1].hdl = new_intf->partial_hdr_hdl[IPA_IP_v6];
-		ipa_del_hdr(del_hdr);
-		kfree(del_hdr);
-	}
-	new_intf->partial_hdr_hdl[IPA_IP_v4] = 0;
-	new_intf->partial_hdr_hdl[IPA_IP_v6] = 0;
-fail_commit_hdr:
-	kfree(hdr);
-fail_alloc_hdr:
-	kfree(new_intf);
-fail:
-	mutex_unlock(&ipa_wigig_ctx->lock);
-	return ret;
-}
-EXPORT_SYMBOL(ipa_wigig_reg_intf);
-
-int ipa_wigig_dereg_intf(const char *netdev_name)
-{
-	int len, ret;
-	struct ipa_ioc_del_hdr *hdr = NULL;
-	struct ipa_wigig_intf_info *entry;
-	struct ipa_wigig_intf_info *next;
-
-	if (!netdev_name) {
-		IPA_WIGIG_ERR("no netdev name\n");
-		return -EINVAL;
-	}
-
-	IPA_WIGIG_DBG("netdev %s\n", netdev_name);
-
-	if (!ipa_wigig_ctx) {
-		IPA_WIGIG_ERR("wigig ctx is not initialized\n");
-		return -EPERM;
-	}
-
-	mutex_lock(&ipa_wigig_ctx->lock);
-
-	ret = -EFAULT;
-
-	list_for_each_entry_safe(entry, next, &ipa_wigig_ctx->head_intf_list,
-		link)
-		if (strcmp(entry->netdev_name, netdev_name) == 0) {
-			len = sizeof(struct ipa_ioc_del_hdr) +
-				2 * sizeof(struct ipa_hdr_del);
-			hdr = kzalloc(len, GFP_KERNEL);
-			if (hdr == NULL) {
-				mutex_unlock(&ipa_wigig_ctx->lock);
-				return -ENOMEM;
-			}
-
-			hdr->commit = 1;
-			hdr->num_hdls = 2;
-			hdr->hdl[0].hdl = entry->partial_hdr_hdl[0];
-			hdr->hdl[1].hdl = entry->partial_hdr_hdl[1];
-			IPA_WIGIG_DBG("IPv4 hdr hdl: %d IPv6 hdr hdl: %d\n",
-				hdr->hdl[0].hdl, hdr->hdl[1].hdl);
-
-			if (ipa_del_hdr(hdr)) {
-				IPA_WIGIG_ERR(
-					"fail to delete partial header\n");
-				ret = -EFAULT;
-				goto fail;
-			}
-
-			if (ipa_deregister_intf(entry->netdev_name)) {
-				IPA_WIGIG_ERR("fail to del interface props\n");
-				ret = -EFAULT;
-				goto fail;
-			}
-
-			if (ipa_wigig_send_wlan_msg(WLAN_AP_DISCONNECT,
-				entry->netdev_name,
-				entry->netdev_mac)) {
-				IPA_WIGIG_ERR("couldn't send msg to IPACM\n");
-				ret = -EFAULT;
-				goto fail;
-			}
-
-			list_del(&entry->link);
-			kfree(entry);
-
-			ret = 0;
-			break;
-		}
-
-	IPA_WIGIG_DBG("exit\n");
-fail:
-	kfree(hdr);
-	mutex_unlock(&ipa_wigig_ctx->lock);
-	return ret;
-}
-EXPORT_SYMBOL(ipa_wigig_dereg_intf);
-
-static void ipa_wigig_pm_cb(void *p, enum ipa_pm_cb_event event)
-{
-	IPA_WIGIG_DBG("received pm event %d\n", event);
-}
-
-static int ipa_wigig_store_pipe_info(struct ipa_wigig_pipe_setup_info *pipe,
-	unsigned int idx)
-{
-	IPA_WIGIG_DBG(
-		"idx %d: desc_ring HWHEAD_pa %pa, HWTAIL_pa %pa, status_ring HWHEAD_pa %pa, HWTAIL_pa %pa\n",
-		idx,
-		&pipe->desc_ring_HWHEAD_pa,
-		&pipe->desc_ring_HWTAIL_pa,
-		&pipe->status_ring_HWHEAD_pa,
-		&pipe->status_ring_HWTAIL_pa);
-
-	/* store regs */
-	ipa_wigig_ctx->pipes.flat[idx].desc_ring_HWHEAD_pa =
-		pipe->desc_ring_HWHEAD_pa;
-	ipa_wigig_ctx->pipes.flat[idx].desc_ring_HWTAIL_pa =
-		pipe->desc_ring_HWTAIL_pa;
-
-	ipa_wigig_ctx->pipes.flat[idx].status_ring_HWHEAD_pa =
-		pipe->status_ring_HWHEAD_pa;
-
-	ipa_wigig_ctx->pipes.flat[idx].status_ring_HWTAIL_pa =
-		pipe->status_ring_HWTAIL_pa;
-
-	IPA_WIGIG_DBG("exit\n");
-
-	return 0;
-}
-
-static u8 ipa_wigig_pipe_to_bit_val(int client)
-{
-	u8 shift_val;
-
-	switch (client) {
-	case IPA_CLIENT_WIGIG_PROD:
-		shift_val = 0x1 << IPA_CLIENT_WIGIG_PROD_IDX;
-		break;
-	case IPA_CLIENT_WIGIG1_CONS:
-		shift_val = 0x1 << IPA_CLIENT_WIGIG1_CONS_IDX;
-		break;
-	case IPA_CLIENT_WIGIG2_CONS:
-		shift_val = 0x1 << IPA_CLIENT_WIGIG2_CONS_IDX;
-		break;
-	case IPA_CLIENT_WIGIG3_CONS:
-		shift_val = 0x1 << IPA_CLIENT_WIGIG3_CONS_IDX;
-		break;
-	case IPA_CLIENT_WIGIG4_CONS:
-		shift_val = 0x1 << IPA_CLIENT_WIGIG4_CONS_IDX;
-		break;
-	default:
-		IPA_WIGIG_ERR("invalid pipe %d\n", client);
-		return 1;
-	}
-
-	return shift_val;
-}
-
-int ipa_wigig_conn_rx_pipe(struct ipa_wigig_conn_rx_in_params *in,
-	struct ipa_wigig_conn_out_params *out)
-{
-	int ret;
-	struct ipa_pm_register_params pm_params;
-
-	IPA_WIGIG_DBG("\n");
-
-	if (!in || !out) {
-		IPA_WIGIG_ERR("empty parameters. in=%pK out=%pK\n", in, out);
-		return -EINVAL;
-	}
-
-	if (!ipa_wigig_ctx) {
-		IPA_WIGIG_ERR("wigig ctx is not initialized\n");
-		return -EPERM;
-	}
-
-	ret = ipa_uc_state_check();
-	if (ret) {
-		IPA_WIGIG_ERR("uC not ready\n");
-		return ret;
-	}
-
-	if (ipa_wigig_init_smmu_params())
-		return -EINVAL;
-
-	if (ipa_wigig_ctx->smmu_en) {
-		IPA_WIGIG_ERR("IPA SMMU is enabled, wrong API used\n");
-		return -EFAULT;
-	}
-
-	memset(&pm_params, 0, sizeof(pm_params));
-	pm_params.name = "wigig";
-	pm_params.callback = ipa_wigig_pm_cb;
-	pm_params.user_data = NULL;
-	pm_params.group = IPA_PM_GROUP_DEFAULT;
-	if (ipa_pm_register(&pm_params, &ipa_wigig_ctx->ipa_pm_hdl)) {
-		IPA_WIGIG_ERR("fail to register ipa pm\n");
-		ret = -EFAULT;
-		goto fail_pm;
-	}
-	IPA_WIGIG_DBG("pm hdl %d\n", ipa_wigig_ctx->ipa_pm_hdl);
-
-	ret = ipa_wigig_uc_msi_init(true,
-		ipa_wigig_ctx->periph_baddr_pa,
-		ipa_wigig_ctx->pseudo_cause_pa,
-		ipa_wigig_ctx->int_gen_tx_pa,
-		ipa_wigig_ctx->int_gen_rx_pa,
-		ipa_wigig_ctx->dma_ep_misc_pa);
-	if (ret) {
-		IPA_WIGIG_ERR("failed configuring msi regs at uC\n");
-		ret = -EFAULT;
-		goto fail_msi;
-	}
-
-	if (ipa_conn_wigig_rx_pipe_i(in, out, &ipa_wigig_ctx->parent)) {
-		IPA_WIGIG_ERR("fail to connect rx pipe\n");
-		ret = -EFAULT;
-		goto fail_connect_pipe;
-	}
-
-	ipa_wigig_ctx->tx_notify = in->notify;
-	ipa_wigig_ctx->priv = in->priv;
-
-	if (ipa_wigig_ctx->parent)
-		ipa_wigig_init_debugfs(ipa_wigig_ctx->parent);
-
-	ipa_wigig_store_pipe_info(ipa_wigig_ctx->pipes.flat,
-		IPA_CLIENT_WIGIG_PROD_IDX);
-
-	ipa_wigig_ctx->conn_pipes |=
-		ipa_wigig_pipe_to_bit_val(IPA_CLIENT_WIGIG_PROD);
-
-	IPA_WIGIG_DBG("exit\n");
-
-	return 0;
-
-fail_connect_pipe:
-	ipa_wigig_uc_msi_init(false,
-		ipa_wigig_ctx->periph_baddr_pa,
-		ipa_wigig_ctx->pseudo_cause_pa,
-		ipa_wigig_ctx->int_gen_tx_pa,
-		ipa_wigig_ctx->int_gen_rx_pa,
-		ipa_wigig_ctx->dma_ep_misc_pa);
-fail_msi:
-	ipa_pm_deregister(ipa_wigig_ctx->ipa_pm_hdl);
-fail_pm:
-	return ret;
-}
-EXPORT_SYMBOL(ipa_wigig_conn_rx_pipe);
-
-static int ipa_wigig_client_to_idx(enum ipa_client_type client,
-	unsigned int *idx)
-{
-	switch (client) {
-	case IPA_CLIENT_WIGIG1_CONS:
-		*idx = 1;
-		break;
-	case IPA_CLIENT_WIGIG2_CONS:
-		*idx = 2;
-		break;
-	case IPA_CLIENT_WIGIG3_CONS:
-		*idx = 3;
-		break;
-	case IPA_CLIENT_WIGIG4_CONS:
-		*idx = 4;
-		break;
-	default:
-		IPA_WIGIG_ERR("invalid client %d\n", client);
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int ipa_wigig_clean_pipe_info(unsigned int idx)
-{
-	IPA_WIGIG_DBG("cleaning pipe %d info\n", idx);
-
-	if (idx >= IPA_WIGIG_MAX_PIPES) {
-		IPA_WIGIG_ERR("invalid index %d\n", idx);
-		return -EINVAL;
-	}
-
-	if (ipa_wigig_ctx->smmu_en) {
-		sg_free_table(
-			&ipa_wigig_ctx->pipes.smmu[idx].desc_ring_base);
-		sg_free_table(
-			&ipa_wigig_ctx->pipes.smmu[idx].status_ring_base);
-
-		memset(ipa_wigig_ctx->pipes.smmu + idx,
-			0,
-			sizeof(ipa_wigig_ctx->pipes.smmu[idx]));
-	} else {
-		memset(ipa_wigig_ctx->pipes.flat + idx, 0,
-			sizeof(ipa_wigig_ctx->pipes.flat[idx]));
-	}
-
-	IPA_WIGIG_DBG("exit\n");
-
-	return 0;
-}
-
-static int ipa_wigig_clone_sg_table(struct sg_table *source,
-	struct sg_table *dst)
-{
-	struct scatterlist *next, *s, *sglist;
-	int i, nents = source->nents;
-
-	if (sg_alloc_table(dst, nents, GFP_KERNEL))
-		return -EINVAL;
-	next = dst->sgl;
-	sglist = source->sgl;
-	for_each_sg(sglist, s, nents, i) {
-		*next = *s;
-		next = sg_next(next);
-	}
-
-	dst->nents = nents;
-	dst->orig_nents = source->orig_nents;
-
-	return 0;
-}
-
-static int ipa_wigig_store_pipe_smmu_info
-	(struct ipa_wigig_pipe_setup_info_smmu *pipe_smmu, unsigned int idx)
-{
-	int ret;
-
-	IPA_WIGIG_DBG(
-		"idx %d: desc_ring HWHEAD_pa %pa, HWTAIL_pa %pa, status_ring HWHEAD_pa %pa, HWTAIL_pa %pa, desc_ring_base 0x%llx, status_ring_base 0x%llx\n",
-		idx,
-		&pipe_smmu->desc_ring_HWHEAD_pa,
-		&pipe_smmu->desc_ring_HWTAIL_pa,
-		&pipe_smmu->status_ring_HWHEAD_pa,
-		&pipe_smmu->status_ring_HWTAIL_pa,
-		(unsigned long long)pipe_smmu->desc_ring_base_iova,
-		(unsigned long long)pipe_smmu->status_ring_base_iova);
-
-	/* store regs */
-	ipa_wigig_ctx->pipes.smmu[idx].desc_ring_HWHEAD_pa =
-		pipe_smmu->desc_ring_HWHEAD_pa;
-	ipa_wigig_ctx->pipes.smmu[idx].desc_ring_HWTAIL_pa =
-		pipe_smmu->desc_ring_HWTAIL_pa;
-
-	ipa_wigig_ctx->pipes.smmu[idx].status_ring_HWHEAD_pa =
-		pipe_smmu->status_ring_HWHEAD_pa;
-	ipa_wigig_ctx->pipes.smmu[idx].status_ring_HWTAIL_pa =
-		pipe_smmu->status_ring_HWTAIL_pa;
-
-	/* store rings IOVAs */
-	ipa_wigig_ctx->pipes.smmu[idx].desc_ring_base_iova =
-		pipe_smmu->desc_ring_base_iova;
-	ipa_wigig_ctx->pipes.smmu[idx].status_ring_base_iova =
-		pipe_smmu->status_ring_base_iova;
-
-	/* copy sgt */
-	ret = ipa_wigig_clone_sg_table(
-		&pipe_smmu->desc_ring_base,
-		&ipa_wigig_ctx->pipes.smmu[idx].desc_ring_base);
-	if (ret)
-		goto fail_desc;
-
-	ret = ipa_wigig_clone_sg_table(
-		&pipe_smmu->status_ring_base,
-		&ipa_wigig_ctx->pipes.smmu[idx].status_ring_base);
-	if (ret)
-		goto fail_stat;
-
-	IPA_WIGIG_DBG("exit\n");
-
-	return 0;
-fail_stat:
-	sg_free_table(&ipa_wigig_ctx->pipes.smmu[idx].desc_ring_base);
-	memset(&ipa_wigig_ctx->pipes.smmu[idx].desc_ring_base, 0,
-	       sizeof(ipa_wigig_ctx->pipes.smmu[idx].desc_ring_base));
-fail_desc:
-	return ret;
-}
-
-static int ipa_wigig_get_pipe_smmu_info(
-	struct ipa_wigig_pipe_setup_info_smmu **pipe_smmu, unsigned int idx)
-{
-	if (idx >= IPA_WIGIG_MAX_PIPES) {
-		IPA_WIGIG_ERR("exceeded pipe num %d > %d\n",
-			idx, IPA_WIGIG_MAX_PIPES);
-		return -EINVAL;
-	}
-
-	*pipe_smmu = &ipa_wigig_ctx->pipes.smmu[idx];
-
-	return 0;
-}
-
-static int ipa_wigig_get_pipe_info(
-	struct ipa_wigig_pipe_setup_info **pipe, unsigned int idx)
-{
-	if (idx >= IPA_WIGIG_MAX_PIPES) {
-		IPA_WIGIG_ERR("exceeded pipe num %d >= %d\n", idx,
-			IPA_WIGIG_MAX_PIPES);
-		return -EINVAL;
-	}
-
-	*pipe = &ipa_wigig_ctx->pipes.flat[idx];
-
-	return 0;
-}
-
-static int ipa_wigig_get_regs_addr(
-	void __iomem **desc_ring_h, void __iomem **desc_ring_t,
-	void __iomem **status_ring_h, void __iomem **status_ring_t,
-	unsigned int idx)
-{
-	struct ipa_wigig_pipe_setup_info *pipe;
-	struct ipa_wigig_pipe_setup_info_smmu *pipe_smmu;
-	int ret = 0;
-
-	IPA_WIGIG_DBG("\n");
-
-	if (idx >= IPA_WIGIG_MAX_PIPES) {
-		IPA_WIGIG_DBG("exceeded pipe num %d >= %d\n", idx,
-			IPA_WIGIG_MAX_PIPES);
-		return -EINVAL;
-	}
-
-	if (!ipa_wigig_ctx) {
-		IPA_WIGIG_DBG("wigig ctx is not initialized\n");
-		return -EPERM;
-	}
-
-	if (!(ipa_wigig_ctx->conn_pipes &
-		ipa_wigig_pipe_to_bit_val(IPA_CLIENT_WIGIG_PROD))) {
-		IPA_WIGIG_DBG(
-			"must connect rx pipe before connecting any client\n");
-		return -EINVAL;
-	}
-
-	if (ipa_wigig_ctx->smmu_en) {
-		ret = ipa_wigig_get_pipe_smmu_info(&pipe_smmu, idx);
-		if (ret)
-			return -EINVAL;
-
-		*desc_ring_h =
-			ioremap(pipe_smmu->desc_ring_HWHEAD_pa, sizeof(u32));
-		if (!*desc_ring_h) {
-			IPA_WIGIG_DBG(
-				"couldn't ioremap desc ring head address\n");
-			ret = -EINVAL;
-			goto fail_map_desc_h;
-		}
-		*desc_ring_t =
-			ioremap(pipe_smmu->desc_ring_HWTAIL_pa, sizeof(u32));
-		if (!*desc_ring_t) {
-			IPA_WIGIG_DBG(
-				"couldn't ioremap desc ring tail address\n");
-			ret = -EINVAL;
-			goto fail_map_desc_t;
-		}
-		*status_ring_h =
-			ioremap(pipe_smmu->status_ring_HWHEAD_pa, sizeof(u32));
-		if (!*status_ring_h) {
-			IPA_WIGIG_DBG(
-				"couldn't ioremap status ring head address\n");
-			ret = -EINVAL;
-			goto fail_map_status_h;
-		}
-		*status_ring_t =
-			ioremap(pipe_smmu->status_ring_HWTAIL_pa, sizeof(u32));
-		if (!*status_ring_t) {
-			IPA_WIGIG_DBG(
-				"couldn't ioremap status ring tail address\n");
-			ret = -EINVAL;
-			goto fail_map_status_t;
-		}
-	} else {
-		ret = ipa_wigig_get_pipe_info(&pipe, idx);
-		if (ret)
-			return -EINVAL;
-
-		*desc_ring_h = ioremap(pipe->desc_ring_HWHEAD_pa, sizeof(u32));
-		if (!*desc_ring_h) {
-			IPA_WIGIG_DBG(
-				"couldn't ioremap desc ring head address\n");
-			ret = -EINVAL;
-			goto fail_map_desc_h;
-		}
-		*desc_ring_t = ioremap(pipe->desc_ring_HWTAIL_pa, sizeof(u32));
-		if (!*desc_ring_t) {
-			IPA_WIGIG_DBG(
-				"couldn't ioremap desc ring tail address\n");
-			ret = -EINVAL;
-			goto fail_map_desc_t;
-		}
-		*status_ring_h =
-			ioremap(pipe->status_ring_HWHEAD_pa, sizeof(u32));
-		if (!*status_ring_h) {
-			IPA_WIGIG_DBG(
-				"couldn't ioremap status ring head address\n");
-			ret = -EINVAL;
-			goto fail_map_status_h;
-		}
-		*status_ring_t =
-			ioremap(pipe->status_ring_HWTAIL_pa, sizeof(u32));
-		if (!*status_ring_t) {
-			IPA_WIGIG_DBG(
-				"couldn't ioremap status ring tail address\n");
-			ret = -EINVAL;
-			goto fail_map_status_t;
-		}
-	}
-
-	IPA_WIGIG_DBG("exit\n");
-	return 0;
-
-fail_map_status_t:
-	iounmap(*status_ring_h);
-fail_map_status_h:
-	iounmap(*desc_ring_t);
-fail_map_desc_t:
-	iounmap(*desc_ring_h);
-fail_map_desc_h:
-	IPA_WIGIG_DBG("couldn't get regs information idx %d\n", idx);
-	return ret;
-}
-
-int ipa_wigig_save_regs(void)
-{
-	void __iomem *desc_ring_h = NULL, *desc_ring_t = NULL,
-		*status_ring_h = NULL, *status_ring_t = NULL,
-		*int_gen_rx_pa = NULL, *int_gen_tx_pa = NULL;
-	uint32_t readval;
-	u8 pipe_connected;
-	int i, ret = 0;
-
-	IPA_WIGIG_DBG("Start collecting pipes information\n");
-
-	if (!ipa_wigig_ctx) {
-		IPA_WIGIG_ERR("wigig ctx is not initialized\n");
-		return -EPERM;
-	}
-	if (!(ipa_wigig_ctx->conn_pipes &
-		ipa_wigig_pipe_to_bit_val(IPA_CLIENT_WIGIG_PROD))) {
-		IPA_WIGIG_ERR(
-			"must connect rx pipe before connecting any client\n");
-		return -EINVAL;
-	}
-
-	for (i = 0; i < IPA_WIGIG_MAX_PIPES; i++) {
-		pipe_connected = (ipa_wigig_ctx->conn_pipes & (0x1 << i));
-		if (pipe_connected) {
-			ret = ipa_wigig_get_regs_addr(
-				&desc_ring_h, &desc_ring_t,
-				&status_ring_h, &status_ring_t, i);
-
-			if (ret) {
-				IPA_WIGIG_ERR(
-					"couldn't get registers information on client %d\n",
-					i);
-				return -EINVAL;
-			}
-
-			IPA_WIGIG_DBG("collecting pipe info of index %d\n", i);
-			if (i == IPA_CLIENT_WIGIG_PROD_IDX) {
-				ipa_wigig_ctx->regs_save.pipes_val[i].dir = 0;
-			} else {
-				ipa_wigig_ctx->regs_save.pipes_val[i].dir = 1;
-				/* TX ids start from 2 */
-				ipa_wigig_ctx->regs_save.pipes_val[i]
-					.tx_ring_id = i + 1;
-			}
-
-			readval = readl_relaxed(desc_ring_h);
-			ipa_wigig_ctx->regs_save.pipes_val[i].desc_ring_HWHEAD =
-				readval;
-			readval = readl_relaxed(desc_ring_t);
-			ipa_wigig_ctx->regs_save.pipes_val[i].desc_ring_HWTAIL =
-				readval;
-			readval = readl_relaxed(status_ring_h);
-			ipa_wigig_ctx->regs_save.pipes_val[i]
-				.status_ring_HWHEAD = readval;
-			readval = readl_relaxed(status_ring_t);
-			ipa_wigig_ctx->regs_save.pipes_val[i]
-				.status_ring_HWTAIL = readval;
-			/* unmap all regs */
-			iounmap(desc_ring_h);
-			iounmap(desc_ring_t);
-			iounmap(status_ring_h);
-			iounmap(status_ring_t);
-		}
-	}
-	int_gen_rx_pa = ioremap(ipa_wigig_ctx->int_gen_rx_pa, sizeof(u32));
-	if (!int_gen_rx_pa) {
-		IPA_WIGIG_ERR("couldn't ioremap gen rx address\n");
-		ret = -EINVAL;
-		goto fail_map_gen_rx;
-	}
-	int_gen_tx_pa = ioremap(ipa_wigig_ctx->int_gen_tx_pa, sizeof(u32));
-	if (!int_gen_tx_pa) {
-		IPA_WIGIG_ERR("couldn't ioremap gen tx address\n");
-		ret = -EINVAL;
-		goto fail_map_gen_tx;
-	}
-
-	IPA_WIGIG_DBG("collecting int_gen_rx_pa info\n");
-	readval = readl_relaxed(int_gen_rx_pa);
-	ipa_wigig_ctx->regs_save.int_gen_rx_val = readval;
-
-	IPA_WIGIG_DBG("collecting int_gen_tx_pa info\n");
-	readval = readl_relaxed(int_gen_tx_pa);
-	ipa_wigig_ctx->regs_save.int_gen_tx_val = readval;
-
-	IPA_WIGIG_DBG("Finish collecting pipes info\n");
-	IPA_WIGIG_DBG("exit\n");
-
-	iounmap(int_gen_tx_pa);
-fail_map_gen_tx:
-	iounmap(int_gen_rx_pa);
-fail_map_gen_rx:
-	return ret;
-}
-
-static void ipa_wigig_clean_rx_buff_smmu_info(void)
-{
-	IPA_WIGIG_DBG("clearing rx buff smmu info\n");
-
-	sg_free_table(&ipa_wigig_ctx->rx_buff_smmu.data_buffer_base);
-	memset(&ipa_wigig_ctx->rx_buff_smmu,
-		0,
-		sizeof(ipa_wigig_ctx->rx_buff_smmu));
-
-	IPA_WIGIG_DBG("\n");
-}
-
-static int ipa_wigig_store_rx_buff_smmu_info(
-	struct ipa_wigig_rx_pipe_data_buffer_info_smmu *dbuff_smmu)
-{
-	IPA_WIGIG_DBG("\n");
-	if (ipa_wigig_clone_sg_table(&dbuff_smmu->data_buffer_base,
-		&ipa_wigig_ctx->rx_buff_smmu.data_buffer_base))
-		return -EINVAL;
-
-	ipa_wigig_ctx->rx_buff_smmu.data_buffer_base_iova =
-		dbuff_smmu->data_buffer_base_iova;
-	ipa_wigig_ctx->rx_buff_smmu.data_buffer_size =
-		dbuff_smmu->data_buffer_size;
-
-	IPA_WIGIG_DBG("exit\n");
-
-	return 0;
-}
-
-static int ipa_wigig_get_rx_buff_smmu_info(
-	struct ipa_wigig_rx_pipe_data_buffer_info_smmu **dbuff_smmu)
-{
-	IPA_WIGIG_DBG("\n");
-
-	*dbuff_smmu = &ipa_wigig_ctx->rx_buff_smmu;
-
-	IPA_WIGIG_DBG("exit\n");
-
-	return 0;
-}
-
-static int ipa_wigig_store_tx_buff_smmu_info(
-	struct ipa_wigig_tx_pipe_data_buffer_info_smmu *dbuff_smmu,
-	unsigned int idx)
-{
-	int result, i;
-	struct ipa_wigig_tx_pipe_data_buffer_info_smmu *tx_buff_smmu;
-
-	IPA_WIGIG_DBG("\n");
-
-	if (idx > (IPA_WIGIG_TX_PIPE_NUM - 1)) {
-		IPA_WIGIG_ERR("invalid tx index %d\n", idx);
-		return -EINVAL;
-	}
-
-	tx_buff_smmu = ipa_wigig_ctx->tx_buff_smmu + idx;
-
-	tx_buff_smmu->data_buffer_base =
-		kcalloc(dbuff_smmu->num_buffers,
-			sizeof(struct sg_table),
-			GFP_KERNEL);
-	if (!tx_buff_smmu->data_buffer_base)
-		return -ENOMEM;
-
-	tx_buff_smmu->data_buffer_base_iova =
-		kcalloc(dbuff_smmu->num_buffers, sizeof(u64), GFP_KERNEL);
-	if (!tx_buff_smmu->data_buffer_base_iova) {
-		result = -ENOMEM;
-		goto fail_iova;
-	}
-
-	for (i = 0; i < dbuff_smmu->num_buffers; i++) {
-		result = ipa_wigig_clone_sg_table(
-			dbuff_smmu->data_buffer_base + i,
-			tx_buff_smmu->data_buffer_base + i);
-		if (result)
-			goto fail_sg_clone;
-
-		tx_buff_smmu->data_buffer_base_iova[i] =
-			dbuff_smmu->data_buffer_base_iova[i];
-	}
-	tx_buff_smmu->num_buffers = dbuff_smmu->num_buffers;
-	tx_buff_smmu->data_buffer_size =
-		dbuff_smmu->data_buffer_size;
-
-	IPA_WIGIG_DBG("exit\n");
-
-	return 0;
-fail_sg_clone:
-	i--;
-	for (; i >= 0; i--)
-		sg_free_table(tx_buff_smmu->data_buffer_base + i);
-	kfree(tx_buff_smmu->data_buffer_base_iova);
-	tx_buff_smmu->data_buffer_base_iova = NULL;
-fail_iova:
-	kfree(tx_buff_smmu->data_buffer_base);
-	tx_buff_smmu->data_buffer_base = NULL;
-	return result;
-}
-
-static int ipa_wigig_clean_tx_buff_smmu_info(unsigned int idx)
-{
-	unsigned int i;
-	struct ipa_wigig_tx_pipe_data_buffer_info_smmu *dbuff_smmu;
-
-	IPA_WIGIG_DBG("\n");
-
-	if (idx > (IPA_WIGIG_TX_PIPE_NUM - 1)) {
-		IPA_WIGIG_ERR("invalid tx index %d\n", idx);
-		return -EINVAL;
-	}
-
-	dbuff_smmu = &ipa_wigig_ctx->tx_buff_smmu[idx];
-
-	if (!dbuff_smmu->data_buffer_base) {
-		IPA_WIGIG_ERR("no pa has been allocated\n");
-		return -EFAULT;
-	}
-
-	for (i = 0; i < dbuff_smmu->num_buffers; i++)
-		sg_free_table(dbuff_smmu->data_buffer_base + i);
-
-	kfree(dbuff_smmu->data_buffer_base);
-	dbuff_smmu->data_buffer_base = NULL;
-
-	kfree(dbuff_smmu->data_buffer_base_iova);
-	dbuff_smmu->data_buffer_base_iova = NULL;
-
-	dbuff_smmu->data_buffer_size = 0;
-	dbuff_smmu->num_buffers = 0;
-
-	IPA_WIGIG_DBG("exit\n");
-
-	return 0;
-}
-
-static int ipa_wigig_get_tx_buff_smmu_info(
-struct ipa_wigig_tx_pipe_data_buffer_info_smmu **dbuff_smmu,
-	unsigned int idx)
-{
-	if (idx > (IPA_WIGIG_TX_PIPE_NUM - 1)) {
-		IPA_WIGIG_ERR("invalid tx index %d\n", idx);
-		return -EINVAL;
-	}
-
-	*dbuff_smmu = &ipa_wigig_ctx->tx_buff_smmu[idx];
-
-	return 0;
-}
-
-static int ipa_wigig_store_rx_smmu_info
-	(struct ipa_wigig_conn_rx_in_params_smmu *in)
-{
-	int ret;
-
-	IPA_WIGIG_DBG("\n");
-
-	ret = ipa_wigig_store_pipe_smmu_info(&in->pipe_smmu,
-		IPA_CLIENT_WIGIG_PROD_IDX);
-	if (ret)
-		return ret;
-
-	if (!ipa_wigig_ctx->shared_cb) {
-		ret = ipa_wigig_store_rx_buff_smmu_info(&in->dbuff_smmu);
-		if (ret)
-			goto fail_buff;
-	}
-
-	IPA_WIGIG_DBG("exit\n");
-
-	return 0;
-
-fail_buff:
-	ipa_wigig_clean_pipe_info(IPA_CLIENT_WIGIG_PROD_IDX);
-	return ret;
-}
-
-static int ipa_wigig_store_client_smmu_info
-(struct ipa_wigig_conn_tx_in_params_smmu *in, enum ipa_client_type client)
-{
-	int ret;
-	unsigned int idx;
-
-	IPA_WIGIG_DBG("\n");
-
-	ret = ipa_wigig_client_to_idx(client, &idx);
-	if (ret)
-		return ret;
-
-	ret = ipa_wigig_store_pipe_smmu_info(&in->pipe_smmu, idx);
-	if (ret)
-		return ret;
-
-	if (!ipa_wigig_ctx->shared_cb) {
-		ret = ipa_wigig_store_tx_buff_smmu_info(
-			&in->dbuff_smmu, idx - 1);
-		if (ret)
-			goto fail_buff;
-	}
-
-	IPA_WIGIG_DBG("exit\n");
-
-	return 0;
-
-fail_buff:
-	ipa_wigig_clean_pipe_info(IPA_CLIENT_WIGIG_PROD_IDX);
-	return ret;
-}
-
-static int ipa_wigig_get_rx_smmu_info(
-	struct ipa_wigig_pipe_setup_info_smmu **pipe_smmu,
-	struct ipa_wigig_rx_pipe_data_buffer_info_smmu **dbuff_smmu)
-{
-	int ret;
-
-	ret = ipa_wigig_get_pipe_smmu_info(pipe_smmu,
-		IPA_CLIENT_WIGIG_PROD_IDX);
-	if (ret)
-		return ret;
-
-	ret = ipa_wigig_get_rx_buff_smmu_info(dbuff_smmu);
-	if (ret)
-		return ret;
-
-	return 0;
-}
-
-static int ipa_wigig_get_tx_smmu_info(
-	struct ipa_wigig_pipe_setup_info_smmu **pipe_smmu,
-	struct ipa_wigig_tx_pipe_data_buffer_info_smmu **dbuff_smmu,
-	enum ipa_client_type client)
-{
-	unsigned int idx;
-	int ret;
-
-	ret = ipa_wigig_client_to_idx(client, &idx);
-	if (ret)
-		return ret;
-
-	ret = ipa_wigig_get_pipe_smmu_info(pipe_smmu, idx);
-	if (ret)
-		return ret;
-
-	ret = ipa_wigig_get_tx_buff_smmu_info(dbuff_smmu, idx - 1);
-	if (ret)
-		return ret;
-
-	return 0;
-}
-
-static int ipa_wigig_clean_smmu_info(enum ipa_client_type client)
-{
-	int ret;
-
-	if (client == IPA_CLIENT_WIGIG_PROD) {
-		ret = ipa_wigig_clean_pipe_info(IPA_CLIENT_WIGIG_PROD_IDX);
-		if (ret)
-			return ret;
-		if (!ipa_wigig_ctx->shared_cb)
-			ipa_wigig_clean_rx_buff_smmu_info();
-	} else {
-		unsigned int idx;
-
-		ret = ipa_wigig_client_to_idx(client, &idx);
-		if (ret)
-			return ret;
-
-		ret = ipa_wigig_clean_pipe_info(idx);
-		if (ret)
-			return ret;
-
-		if (!ipa_wigig_ctx->shared_cb) {
-			ret = ipa_wigig_clean_tx_buff_smmu_info(idx - 1);
-			if (ret) {
-				IPA_WIGIG_ERR(
-					"cleaned tx pipe info but wasn't able to clean buff info, client %d\n"
-					, client);
-				WARN_ON(1);
-				return ret;
-			}
-		}
-	}
-
-	return 0;
-}
-int ipa_wigig_conn_rx_pipe_smmu(
-	struct ipa_wigig_conn_rx_in_params_smmu *in,
-	struct ipa_wigig_conn_out_params *out)
-{
-	int ret;
-	struct ipa_pm_register_params pm_params;
-
-	IPA_WIGIG_DBG("\n");
-
-	if (!in || !out) {
-		IPA_WIGIG_ERR("empty parameters. in=%pK out=%pK\n", in, out);
-		return -EINVAL;
-	}
-
-	if (!ipa_wigig_ctx) {
-		IPA_WIGIG_ERR("wigig ctx is not initialized\n");
-		return -EPERM;
-	}
-
-	ret = ipa_uc_state_check();
-	if (ret) {
-		IPA_WIGIG_ERR("uC not ready\n");
-		return ret;
-	}
-
-	if (ipa_wigig_init_smmu_params())
-		return -EINVAL;
-
-	if (!ipa_wigig_ctx->smmu_en) {
-		IPA_WIGIG_ERR("IPA SMMU is disabled, wrong API used\n");
-		return -EFAULT;
-	}
-
-	memset(&pm_params, 0, sizeof(pm_params));
-	pm_params.name = "wigig";
-	pm_params.callback = ipa_wigig_pm_cb;
-	pm_params.user_data = NULL;
-	pm_params.group = IPA_PM_GROUP_DEFAULT;
-	if (ipa_pm_register(&pm_params, &ipa_wigig_ctx->ipa_pm_hdl)) {
-		IPA_WIGIG_ERR("fail to register ipa pm\n");
-		ret = -EFAULT;
-		goto fail_pm;
-	}
-
-	ret = ipa_wigig_uc_msi_init(true,
-		ipa_wigig_ctx->periph_baddr_pa,
-		ipa_wigig_ctx->pseudo_cause_pa,
-		ipa_wigig_ctx->int_gen_tx_pa,
-		ipa_wigig_ctx->int_gen_rx_pa,
-		ipa_wigig_ctx->dma_ep_misc_pa);
-	if (ret) {
-		IPA_WIGIG_ERR("failed configuring msi regs at uC\n");
-		ret = -EFAULT;
-		goto fail_msi;
-	}
-
-	if (ipa_conn_wigig_rx_pipe_i(in, out, &ipa_wigig_ctx->parent)) {
-		IPA_WIGIG_ERR("fail to connect rx pipe\n");
-		ret = -EFAULT;
-		goto fail_connect_pipe;
-	}
-
-	if (ipa_wigig_ctx->parent)
-		ipa_wigig_init_debugfs(ipa_wigig_ctx->parent);
-
-	if (ipa_wigig_store_rx_smmu_info(in)) {
-		IPA_WIGIG_ERR("fail to store smmu data for rx pipe\n");
-		ret = -EFAULT;
-		goto fail_smmu_store;
-	}
-
-	ipa_wigig_ctx->tx_notify = in->notify;
-	ipa_wigig_ctx->priv = in->priv;
-
-	ipa_wigig_ctx->conn_pipes |=
-		ipa_wigig_pipe_to_bit_val(IPA_CLIENT_WIGIG_PROD);
-
-	IPA_WIGIG_DBG("exit\n");
-
-	return 0;
-
-fail_smmu_store:
-	ipa_disconn_wigig_pipe_i(IPA_CLIENT_WIGIG_PROD,
-		&in->pipe_smmu,
-		&in->dbuff_smmu);
-fail_connect_pipe:
-	ipa_wigig_uc_msi_init(false,
-		ipa_wigig_ctx->periph_baddr_pa,
-		ipa_wigig_ctx->pseudo_cause_pa,
-		ipa_wigig_ctx->int_gen_tx_pa,
-		ipa_wigig_ctx->int_gen_rx_pa,
-		ipa_wigig_ctx->dma_ep_misc_pa);
-fail_msi:
-	ipa_pm_deregister(ipa_wigig_ctx->ipa_pm_hdl);
-fail_pm:
-	return ret;
-}
-EXPORT_SYMBOL(ipa_wigig_conn_rx_pipe_smmu);
-
-int ipa_wigig_set_perf_profile(u32 max_supported_bw_mbps)
-{
-	IPA_WIGIG_DBG("setting throughput to %d\n", max_supported_bw_mbps);
-
-	if (!ipa_wigig_ctx) {
-		IPA_WIGIG_ERR("wigig ctx is not initialized\n");
-		return -EPERM;
-	}
-
-	IPA_WIGIG_DBG("ipa_pm handle %d\n", ipa_wigig_ctx->ipa_pm_hdl);
-	if (ipa_pm_set_throughput(ipa_wigig_ctx->ipa_pm_hdl,
-		max_supported_bw_mbps)) {
-		IPA_WIGIG_ERR("fail to setup pm perf profile\n");
-		return -EFAULT;
-	}
-	IPA_WIGIG_DBG("exit\n");
-
-	return 0;
-}
-EXPORT_SYMBOL(ipa_wigig_set_perf_profile);
-
-static int ipa_wigig_store_client_mac(enum ipa_client_type client,
-	const char *mac)
-{
-	unsigned int idx;
-
-	if (ipa_wigig_client_to_idx(client, &idx)) {
-		IPA_WIGIG_ERR("couldn't acquire idx\n");
-		return -EFAULT;
-	}
-	memcpy(ipa_wigig_ctx->clients_mac[idx - 1], mac, IPA_MAC_ADDR_SIZE);
-	return 0;
-}
-
-static int ipa_wigig_get_client_mac(enum ipa_client_type client, char *mac)
-{
-	unsigned int idx;
-
-	if (ipa_wigig_client_to_idx(client, &idx)) {
-		IPA_WIGIG_ERR("couldn't acquire idx\n");
-		return -EFAULT;
-	}
-	memcpy(mac, ipa_wigig_ctx->clients_mac[idx - 1], IPA_MAC_ADDR_SIZE);
-	return 0;
-}
-
-static int ipa_wigig_clean_client_mac(enum ipa_client_type client)
-{
-	char zero_mac[IPA_MAC_ADDR_SIZE] = { 0 };
-
-	return ipa_wigig_store_client_mac(client, zero_mac);
-}
-
-int ipa_wigig_conn_client(struct ipa_wigig_conn_tx_in_params *in,
-	struct ipa_wigig_conn_out_params *out)
-{
-	char dev_name[IPA_RESOURCE_NAME_MAX];
-	unsigned int idx;
-
-	IPA_WIGIG_DBG("\n");
-
-	if (!in || !out) {
-		IPA_WIGIG_ERR("empty parameters. in=%pK out=%pK\n", in, out);
-		return -EINVAL;
-	}
-
-	if (!ipa_wigig_ctx) {
-		IPA_WIGIG_ERR("wigig ctx is not initialized\n");
-		return -EPERM;
-	}
-
-	if (!(ipa_wigig_ctx->conn_pipes &
-		ipa_wigig_pipe_to_bit_val(IPA_CLIENT_WIGIG_PROD))) {
-		IPA_WIGIG_ERR(
-			"must connect rx pipe before connecting any client\n"
-		);
-		return -EINVAL;
-	}
-
-	if (ipa_wigig_ctx->smmu_en) {
-		IPA_WIGIG_ERR("IPA SMMU is enabled, wrong API used\n");
-		return -EFAULT;
-	}
-
-	if (ipa_uc_state_check()) {
-		IPA_WIGIG_ERR("uC not ready\n");
-		return -EFAULT;
-	}
-
-	if (ipa_wigig_get_devname(dev_name)) {
-		IPA_WIGIG_ERR("couldn't get dev name\n");
-		return -EFAULT;
-	}
-
-	if (ipa_conn_wigig_client_i(in, out, ipa_wigig_ctx->tx_notify,
-		ipa_wigig_ctx->priv)) {
-		IPA_WIGIG_ERR(
-			"fail to connect client. MAC [%X][%X][%X][%X][%X][%X]\n"
-		, in->client_mac[0], in->client_mac[1], in->client_mac[2]
-		, in->client_mac[3], in->client_mac[4], in->client_mac[5]);
-		return -EFAULT;
-	}
-
-	if (ipa_wigig_client_to_idx(out->client, &idx)) {
-		IPA_WIGIG_ERR("couldn't acquire idx\n");
-		goto fail_convert_client_to_idx;
-	}
-
-	ipa_wigig_store_pipe_info(&in->pipe, idx);
-
-	if (ipa_wigig_send_msg(WIGIG_CLIENT_CONNECT,
-		dev_name,
-		in->client_mac, out->client, false)) {
-		IPA_WIGIG_ERR("couldn't send msg to IPACM\n");
-		goto fail_sendmsg;
-	}
-
-	/* update connected clients */
-	ipa_wigig_ctx->conn_pipes |=
-		ipa_wigig_pipe_to_bit_val(out->client);
-
-	ipa_wigig_store_client_mac(out->client, in->client_mac);
-
-	IPA_WIGIG_DBG("exit\n");
-	return 0;
-
-fail_sendmsg:
-	ipa_wigig_clean_pipe_info(idx);
-fail_convert_client_to_idx:
-	ipa_disconn_wigig_pipe_i(out->client, NULL, NULL);
-	return -EINVAL;
-}
-EXPORT_SYMBOL(ipa_wigig_conn_client);
-
-int ipa_wigig_conn_client_smmu(
-	struct ipa_wigig_conn_tx_in_params_smmu *in,
-	struct ipa_wigig_conn_out_params *out)
-{
-	char netdev_name[IPA_RESOURCE_NAME_MAX];
-	int ret;
-
-	IPA_WIGIG_DBG("\n");
-
-	if (!in || !out) {
-		IPA_WIGIG_ERR("empty parameters. in=%pK out=%pK\n", in, out);
-		return -EINVAL;
-	}
-
-	if (!ipa_wigig_ctx) {
-		IPA_WIGIG_ERR("wigig ctx is not initialized\n");
-		return -EPERM;
-	}
-
-	if (!(ipa_wigig_ctx->conn_pipes &
-		ipa_wigig_pipe_to_bit_val(IPA_CLIENT_WIGIG_PROD))) {
-		IPA_WIGIG_ERR(
-			"must connect rx pipe before connecting any client\n"
-		);
-		return -EINVAL;
-	}
-
-	if (!ipa_wigig_ctx->smmu_en) {
-		IPA_WIGIG_ERR("IPA SMMU is disabled, wrong API used\n");
-		return -EFAULT;
-	}
-
-	ret = ipa_uc_state_check();
-	if (ret) {
-		IPA_WIGIG_ERR("uC not ready\n");
-		return ret;
-	}
-
-	if (ipa_wigig_get_devname(netdev_name)) {
-		IPA_WIGIG_ERR("couldn't get dev name\n");
-		return -EFAULT;
-	}
-
-	if (ipa_conn_wigig_client_i(in, out, ipa_wigig_ctx->tx_notify,
-		ipa_wigig_ctx->priv)) {
-		IPA_WIGIG_ERR(
-			"fail to connect client. MAC [%X][%X][%X][%X][%X][%X]\n"
-			, in->client_mac[0], in->client_mac[1]
-			, in->client_mac[2], in->client_mac[3]
-			, in->client_mac[4], in->client_mac[5]);
-		return -EFAULT;
-	}
-
-	if (ipa_wigig_send_msg(WIGIG_CLIENT_CONNECT,
-		netdev_name,
-		in->client_mac, out->client, false)) {
-		IPA_WIGIG_ERR("couldn't send msg to IPACM\n");
-		ret = -EFAULT;
-		goto fail_sendmsg;
-	}
-
-	ret = ipa_wigig_store_client_smmu_info(in, out->client);
-	if (ret)
-		goto fail_smmu;
-
-	/* update connected clients */
-	ipa_wigig_ctx->conn_pipes |=
-		ipa_wigig_pipe_to_bit_val(out->client);
-
-	ipa_wigig_store_client_mac(out->client, in->client_mac);
-
-	IPA_WIGIG_DBG("exit\n");
-	return 0;
-
-fail_smmu:
-	/*
-	 * wigig clients are disconnected with legacy message since there is
-	 * no need to send ep, client MAC is sufficient for disconnect
-	 */
-	ipa_wigig_send_wlan_msg(WLAN_CLIENT_DISCONNECT, netdev_name,
-		in->client_mac);
-fail_sendmsg:
-	ipa_disconn_wigig_pipe_i(out->client, &in->pipe_smmu, &in->dbuff_smmu);
-	return ret;
-}
-EXPORT_SYMBOL(ipa_wigig_conn_client_smmu);
-
-static inline int ipa_wigig_validate_client_type(enum ipa_client_type client)
-{
-	switch (client) {
-	case IPA_CLIENT_WIGIG_PROD:
-	case IPA_CLIENT_WIGIG1_CONS:
-	case IPA_CLIENT_WIGIG2_CONS:
-	case IPA_CLIENT_WIGIG3_CONS:
-	case IPA_CLIENT_WIGIG4_CONS:
-		break;
-	default:
-		IPA_WIGIG_ERR_RL("invalid client type %d\n", client);
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-int ipa_wigig_disconn_pipe(enum ipa_client_type client)
-{
-	int ret;
-	char dev_name[IPA_RESOURCE_NAME_MAX];
-	char client_mac[IPA_MAC_ADDR_SIZE];
-
-	IPA_WIGIG_DBG("\n");
-
-	ret = ipa_wigig_validate_client_type(client);
-	if (ret)
-		return ret;
-
-	if (client != IPA_CLIENT_WIGIG_PROD) {
-		if (ipa_wigig_get_devname(dev_name)) {
-			IPA_WIGIG_ERR("couldn't get dev name\n");
-			return -EFAULT;
-		}
-
-		if (ipa_wigig_get_client_mac(client, client_mac)) {
-			IPA_WIGIG_ERR("couldn't get client mac\n");
-			return -EFAULT;
-		}
-	}
-
-	IPA_WIGIG_DBG("disconnecting ipa_client_type %d\n", client);
-
-	if (ipa_wigig_is_smmu_enabled()) {
-		struct ipa_wigig_pipe_setup_info_smmu *pipe_smmu;
-		struct ipa_wigig_rx_pipe_data_buffer_info_smmu *rx_dbuff_smmu;
-		struct ipa_wigig_tx_pipe_data_buffer_info_smmu *tx_dbuff_smmu;
-
-		if (client == IPA_CLIENT_WIGIG_PROD) {
-			ret = ipa_wigig_get_rx_smmu_info(&pipe_smmu,
-				&rx_dbuff_smmu);
-			if (ret)
-				return ret;
-
-			ret = ipa_disconn_wigig_pipe_i(client,
-				pipe_smmu,
-				rx_dbuff_smmu);
-		} else {
-			ret = ipa_wigig_get_tx_smmu_info(&pipe_smmu,
-				&tx_dbuff_smmu, client);
-			if (ret)
-				return ret;
-
-			ret = ipa_disconn_wigig_pipe_i(client,
-				pipe_smmu,
-				tx_dbuff_smmu);
-		}
-
-	} else {
-		ret = ipa_disconn_wigig_pipe_i(client, NULL, NULL);
-	}
-
-	if (ret) {
-		IPA_WIGIG_ERR("couldn't disconnect client %d\n", client);
-		return ret;
-	}
-
-	/* RX will be disconnected last, deinit uC msi config */
-	if (client == IPA_CLIENT_WIGIG_PROD) {
-		IPA_WIGIG_DBG("Rx pipe disconnected, deIniting uc\n");
-		ret = ipa_wigig_uc_msi_init(false,
-			ipa_wigig_ctx->periph_baddr_pa,
-			ipa_wigig_ctx->pseudo_cause_pa,
-			ipa_wigig_ctx->int_gen_tx_pa,
-			ipa_wigig_ctx->int_gen_rx_pa,
-			ipa_wigig_ctx->dma_ep_misc_pa);
-		if (ret) {
-			IPA_WIGIG_ERR("failed unmapping msi regs\n");
-			WARN_ON(1);
-		}
-
-		ret = ipa_pm_deregister(ipa_wigig_ctx->ipa_pm_hdl);
-		if (ret) {
-			IPA_WIGIG_ERR("failed dereg pm\n");
-			WARN_ON(1);
-		}
-
-		ipa_wigig_ctx->conn_pipes &=
-			~ipa_wigig_pipe_to_bit_val(IPA_CLIENT_WIGIG_PROD);
-		WARN_ON(ipa_wigig_ctx->conn_pipes);
-	} else {
-		/*
-		 * wigig clients are disconnected with legacy message since
-		 * there is no need to send ep, client MAC is sufficient for
-		 * disconnect.
-		 */
-		ipa_wigig_send_wlan_msg(WLAN_CLIENT_DISCONNECT, dev_name,
-			client_mac);
-		ipa_wigig_clean_client_mac(client);
-	}
-	if (ipa_wigig_is_smmu_enabled())
-		ipa_wigig_clean_smmu_info(client);
-
-	IPA_WIGIG_DBG("exit\n");
-	return 0;
-}
-EXPORT_SYMBOL(ipa_wigig_disconn_pipe);
-
-int ipa_wigig_enable_pipe(enum ipa_client_type client)
-{
-	int ret;
-
-	IPA_WIGIG_DBG("\n");
-
-	ret = ipa_wigig_validate_client_type(client);
-	if (ret)
-		return ret;
-
-	IPA_WIGIG_DBG("enabling pipe %d\n", client);
-
-	ret = ipa_enable_wigig_pipe_i(client);
-	if (ret)
-		return ret;
-
-	/* do only when Rx pipe is enabled */
-	if (client == IPA_CLIENT_WIGIG_PROD) {
-		ret = ipa_pm_activate_sync(ipa_wigig_ctx->ipa_pm_hdl);
-		if (ret) {
-			IPA_WIGIG_ERR("fail to activate ipa pm\n");
-			ret = -EFAULT;
-			goto fail_pm_active;
-		}
-	}
-
-	IPA_WIGIG_DBG("exit\n");
-	return 0;
-
-fail_pm_active:
-	ipa_disable_wigig_pipe_i(client);
-	return ret;
-}
-EXPORT_SYMBOL(ipa_wigig_enable_pipe);
-
-int ipa_wigig_disable_pipe(enum ipa_client_type client)
-{
-	int ret;
-
-	IPA_WIGIG_DBG("\n");
-
-	ret = ipa_wigig_validate_client_type(client);
-	if (ret)
-		return ret;
-
-	ret = ipa_disable_wigig_pipe_i(client);
-	if (ret)
-		return ret;
-
-	/* do only when Rx pipe is disabled */
-	if (client == IPA_CLIENT_WIGIG_PROD) {
-		ret = ipa_pm_deactivate_sync(ipa_wigig_ctx->ipa_pm_hdl);
-		if (ret) {
-			IPA_WIGIG_ERR("fail to deactivate ipa pm\n");
-			return -EFAULT;
-		}
-	}
-
-	IPA_WIGIG_DBG("exit\n");
-	return 0;
-}
-EXPORT_SYMBOL(ipa_wigig_disable_pipe);
-
-int ipa_wigig_tx_dp(enum ipa_client_type dst, struct sk_buff *skb)
-{
-	int ret;
-
-	IPA_WIGIG_DBG_LOW("\n");
-
-	ret = ipa_wigig_validate_client_type(dst);
-	if (unlikely(ret))
-		return ret;
-
-	ret = ipa_tx_dp(dst, skb, NULL);
-	if (unlikely(ret))
-		return ret;
-
-	IPA_WIGIG_DBG_LOW("exit\n");
-	return 0;
-}
-EXPORT_SYMBOL(ipa_wigig_tx_dp);
-
-
-#ifdef CONFIG_DEBUG_FS
-#define IPA_MAX_MSG_LEN 4096
-
-static ssize_t ipa_wigig_read_conn_clients(struct file *file,
-		char __user *ubuf, size_t count, loff_t *ppos)
-{
-	int i;
-	int nbytes = 0;
-	u8 pipe_connected;
-	char *dbg_buff;
-	ssize_t ret;
-
-	dbg_buff = kzalloc(IPA_MAX_MSG_LEN, GFP_KERNEL);
-	if (!dbg_buff)
-		return -ENOMEM;
-
-	if (!ipa_wigig_ctx) {
-		nbytes += scnprintf(dbg_buff + nbytes,
-			IPA_MAX_MSG_LEN - nbytes,
-			"IPA WIGIG not initialized\n");
-		goto finish;
-	}
-
-	if (!ipa_wigig_ctx->conn_pipes) {
-		nbytes += scnprintf(dbg_buff + nbytes,
-			IPA_MAX_MSG_LEN - nbytes,
-			"no WIGIG pipes connected\n");
-		goto finish;
-	}
-
-	for (i = 0; i < IPA_WIGIG_MAX_PIPES; i++) {
-		pipe_connected = (ipa_wigig_ctx->conn_pipes & (0x1 << i));
-		switch (i) {
-		case 0:
-			nbytes += scnprintf(dbg_buff + nbytes,
-				IPA_MAX_MSG_LEN - nbytes,
-				"IPA_CLIENT_WIGIG_PROD");
-			break;
-		case 1:
-		case 2:
-		case 3:
-		case 4:
-			nbytes += scnprintf(dbg_buff + nbytes,
-				IPA_MAX_MSG_LEN - nbytes,
-				"IPA_CLIENT_WIGIG%d_CONS",
-				i);
-			break;
-		default:
-			IPA_WIGIG_ERR("invalid pipe %d\n", i);
-			nbytes += scnprintf(dbg_buff + nbytes,
-				IPA_MAX_MSG_LEN - nbytes,
-				"invalid pipe %d",
-				i);
-			break;
-		}
-		nbytes += scnprintf(dbg_buff + nbytes,
-			IPA_MAX_MSG_LEN - nbytes,
-			" %s connected\n", pipe_connected ? "is" : "not");
-	}
-
-finish:
-	ret = simple_read_from_buffer(
-		ubuf, count, ppos, dbg_buff, nbytes);
-	kfree(dbg_buff);
-	return ret;
-}
-
-static ssize_t ipa_wigig_read_smmu_status(struct file *file,
-	char __user *ubuf, size_t count, loff_t *ppos)
-{
-	int nbytes = 0;
-	char *dbg_buff;
-	ssize_t ret;
-
-	dbg_buff = kzalloc(IPA_MAX_MSG_LEN, GFP_KERNEL);
-	if (!dbg_buff)
-		return -ENOMEM;
-
-	if (!ipa_wigig_ctx) {
-		nbytes += scnprintf(dbg_buff + nbytes,
-			IPA_MAX_MSG_LEN - nbytes,
-			"IPA WIGIG not initialized\n");
-		goto finish;
-	}
-
-	if (ipa_wigig_ctx->smmu_en) {
-		nbytes += scnprintf(dbg_buff + nbytes,
-			IPA_MAX_MSG_LEN - nbytes,
-			"SMMU enabled\n");
-
-		if (ipa_wigig_ctx->shared_cb) {
-			nbytes += scnprintf(dbg_buff + nbytes,
-				IPA_MAX_MSG_LEN - nbytes,
-				"CB shared\n");
-		} else {
-			nbytes += scnprintf(dbg_buff + nbytes,
-				IPA_MAX_MSG_LEN - nbytes,
-				"CB not shared\n");
-		}
-	} else {
-		nbytes += scnprintf(dbg_buff + nbytes,
-			IPA_MAX_MSG_LEN - nbytes,
-			"SMMU in S1 bypass\n");
-	}
-finish:
-	ret = simple_read_from_buffer(
-		ubuf, count, ppos, dbg_buff, nbytes);
-	kfree(dbg_buff);
-	return ret;
-}
-static const struct file_operations ipa_wigig_conn_clients_ops = {
-	.read = ipa_wigig_read_conn_clients,
-};
-
-static const struct file_operations ipa_wigig_smmu_ops = {
-	.read = ipa_wigig_read_smmu_status,
-};
-
-static inline void ipa_wigig_deinit_debugfs(void)
-{
-	debugfs_remove(ipa_wigig_ctx->dent_conn_clients);
-	debugfs_remove(ipa_wigig_ctx->dent_smmu);
-}
-
-static int ipa_wigig_init_debugfs(struct dentry *parent)
-{
-	const mode_t read_only_mode = 0444;
-
-	ipa_wigig_ctx->dent_conn_clients =
-		debugfs_create_file("conn_clients", read_only_mode, parent,
-				NULL, &ipa_wigig_conn_clients_ops);
-	if (IS_ERR_OR_NULL(ipa_wigig_ctx->dent_conn_clients)) {
-		IPA_WIGIG_ERR("fail to create file %s\n", "conn_clients");
-		goto fail_conn_clients;
-	}
-
-	ipa_wigig_ctx->dent_smmu =
-		debugfs_create_file("smmu", read_only_mode, parent, NULL,
-				&ipa_wigig_smmu_ops);
-	if (IS_ERR_OR_NULL(ipa_wigig_ctx->dent_smmu)) {
-		IPA_WIGIG_ERR("fail to create file %s\n", "smmu");
-		goto fail_smmu;
-	}
-
-	return 0;
-fail_smmu:
-	debugfs_remove(ipa_wigig_ctx->dent_conn_clients);
-fail_conn_clients:
-	return -EFAULT;
-}
-#endif

ipa/ipa_clients/odu_bridge.c

@@ -1,1212 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2014-2019, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/debugfs.h>
-#include <linux/export.h>
-#include <linux/fs.h>
-#include <linux/if_ether.h>
-#include <linux/ioctl.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/msm_ipa.h>
-#include <linux/mutex.h>
-#include <linux/skbuff.h>
-#include <linux/types.h>
-#include <linux/ipv6.h>
-#include <net/addrconf.h>
-#include <linux/ipa.h>
-#include <linux/cdev.h>
-#include <linux/ipa_odu_bridge.h>
-#include "../ipa_common_i.h"
-#include "../ipa_v3/ipa_pm.h"
-
-#define ODU_BRIDGE_DRV_NAME "odu_ipa_bridge"
-
-#define ODU_BRIDGE_DBG(fmt, args...) \
-	do { \
-		pr_debug(ODU_BRIDGE_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-			ODU_BRIDGE_DRV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			ODU_BRIDGE_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-#define ODU_BRIDGE_DBG_LOW(fmt, args...) \
-	do { \
-		pr_debug(ODU_BRIDGE_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			ODU_BRIDGE_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-#define ODU_BRIDGE_ERR(fmt, args...) \
-	do { \
-		pr_err(ODU_BRIDGE_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-			ODU_BRIDGE_DRV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			ODU_BRIDGE_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-#define ODU_BRIDGE_FUNC_ENTRY() \
-	ODU_BRIDGE_DBG_LOW("ENTRY\n")
-#define ODU_BRIDGE_FUNC_EXIT() \
-	ODU_BRIDGE_DBG_LOW("EXIT\n")
-
-
-#define ODU_BRIDGE_IS_QMI_ADDR(daddr) \
-	(memcmp(&(daddr), &odu_bridge_ctx->llv6_addr, sizeof((daddr))) \
-		== 0)
-
-#define ODU_BRIDGE_IPV4_HDR_NAME "odu_br_ipv4"
-#define ODU_BRIDGE_IPV6_HDR_NAME "odu_br_ipv6"
-
-#define IPA_ODU_SYS_DESC_FIFO_SZ 0x800
-
-#ifdef CONFIG_COMPAT
-#define ODU_BRIDGE_IOC_SET_LLV6_ADDR32 _IOW(ODU_BRIDGE_IOC_MAGIC, \
-				ODU_BRIDGE_IOCTL_SET_LLV6_ADDR, \
-				compat_uptr_t)
-#endif
-
-#define IPA_ODU_VER_CHECK() \
-	do { \
-		ret = 0;\
-		if (ipa_get_hw_type() == IPA_HW_None) { \
-			pr_err("IPA HW is unknown\n"); \
-			ret = -EFAULT; \
-		} \
-		else if (ipa_get_hw_type() < IPA_HW_v3_0) \
-			ret = 1; \
-	} while (0)
-
-/**
- * struct stats - driver statistics, viewable using debugfs
- * @num_ul_packets: number of packets bridged in uplink direction
- * @num_dl_packets: number of packets bridged in downink direction
- * bridge
- * @num_lan_packets: number of packets bridged to APPS on bridge mode
- */
-struct stats {
-	u64 num_ul_packets;
-	u64 num_dl_packets;
-	u64 num_lan_packets;
-};
-
-/**
- * struct odu_bridge_ctx - ODU bridge driver context information
- * @class: kernel class pointer
- * @dev_num: kernel device number
- * @dev: kernel device struct pointer
- * @cdev: kernel character device struct
- * @netdev_name: network interface name
- * @device_ethaddr: network interface ethernet address
- * @priv: client's private data. to be used in client's callbacks
- * @tx_dp_notify: client callback for handling IPA ODU_PROD callback
- * @send_dl_skb: client callback for sending skb in downlink direction
- * @stats: statistics, how many packets were transmitted using the SW bridge
- * @is_conencted: is bridge connected ?
- * @is_suspended: is bridge suspended ?
- * @mode: ODU mode (router/bridge)
- * @lock: for the initialization, connect and disconnect synchronization
- * @llv6_addr: link local IPv6 address of ODU network interface
- * @odu_br_ipv4_hdr_hdl: handle for partial ipv4 ethernet header
- * @odu_br_ipv6_hdr_hdl: handle for partial ipv6 ethernet header
- * @odu_prod_hdl: handle for IPA_CLIENT_ODU_PROD pipe
- * @odu_emb_cons_hdl: handle for IPA_CLIENT_ODU_EMB_CONS pipe
- * @odu_teth_cons_hdl: handle for IPA_CLIENT_ODU_TETH_CONS pipe
- * @wakeup_request: client callback to wakeup
- */
-struct odu_bridge_ctx {
-	struct class *class;
-	dev_t dev_num;
-	struct device *dev;
-	struct cdev cdev;
-	char netdev_name[IPA_RESOURCE_NAME_MAX];
-	u8 device_ethaddr[ETH_ALEN];
-	void *priv;
-	ipa_notify_cb tx_dp_notify;
-	int (*send_dl_skb)(void *priv, struct sk_buff *skb);
-	struct stats stats;
-	bool is_connected;
-	bool is_suspended;
-	enum odu_bridge_mode mode;
-	struct mutex lock;
-	struct in6_addr llv6_addr;
-	uint32_t odu_br_ipv4_hdr_hdl;
-	uint32_t odu_br_ipv6_hdr_hdl;
-	u32 odu_prod_hdl;
-	u32 odu_emb_cons_hdl;
-	u32 odu_teth_cons_hdl;
-	u32 ipa_sys_desc_size;
-	void *logbuf;
-	void *logbuf_low;
-	void (*wakeup_request)(void *cl_priv);
-	u32 pm_hdl;
-};
-static struct odu_bridge_ctx *odu_bridge_ctx;
-
-#ifdef CONFIG_DEBUG_FS
-#define ODU_MAX_MSG_LEN 512
-static char dbg_buff[ODU_MAX_MSG_LEN];
-#endif
-
-static void odu_bridge_emb_cons_cb(void *priv, enum ipa_dp_evt_type evt,
-	unsigned long data)
-{
-	ODU_BRIDGE_FUNC_ENTRY();
-	if (evt != IPA_RECEIVE) {
-		ODU_BRIDGE_ERR("unexpected event\n");
-		WARN_ON(1);
-		return;
-	}
-	odu_bridge_ctx->send_dl_skb(priv, (struct sk_buff *)data);
-	odu_bridge_ctx->stats.num_dl_packets++;
-	ODU_BRIDGE_FUNC_EXIT();
-}
-
-static void odu_bridge_teth_cons_cb(void *priv, enum ipa_dp_evt_type evt,
-	unsigned long data)
-{
-	struct ipv6hdr *ipv6hdr;
-	struct sk_buff *skb = (struct sk_buff *)data;
-	struct sk_buff *skb_copied;
-
-	ODU_BRIDGE_FUNC_ENTRY();
-	if (evt != IPA_RECEIVE) {
-		ODU_BRIDGE_ERR("unexpected event\n");
-		WARN_ON(1);
-		return;
-	}
-
-	ipv6hdr = (struct ipv6hdr *)(skb->data + ETH_HLEN);
-	if (ipv6hdr->version == 6 &&
-	    ipv6_addr_is_multicast(&ipv6hdr->daddr)) {
-		ODU_BRIDGE_DBG_LOW("Multicast pkt, send to APPS and adapter\n");
-		skb_copied = skb_clone(skb, GFP_KERNEL);
-		if (skb_copied) {
-			odu_bridge_ctx->tx_dp_notify(odu_bridge_ctx->priv,
-						IPA_RECEIVE,
-						(unsigned long) skb_copied);
-			odu_bridge_ctx->stats.num_lan_packets++;
-		} else {
-			ODU_BRIDGE_ERR("No memory\n");
-		}
-	}
-
-	odu_bridge_ctx->send_dl_skb(priv, skb);
-	odu_bridge_ctx->stats.num_dl_packets++;
-	ODU_BRIDGE_FUNC_EXIT();
-}
-
-static int odu_bridge_connect_router(void)
-{
-	struct ipa_sys_connect_params odu_prod_params;
-	struct ipa_sys_connect_params odu_emb_cons_params;
-	int res;
-
-	ODU_BRIDGE_FUNC_ENTRY();
-
-	memset(&odu_prod_params, 0, sizeof(odu_prod_params));
-	memset(&odu_emb_cons_params, 0, sizeof(odu_emb_cons_params));
-
-	/* configure RX (ODU->IPA) EP */
-	odu_prod_params.client = IPA_CLIENT_ODU_PROD;
-	odu_prod_params.ipa_ep_cfg.hdr.hdr_len = ETH_HLEN;
-	odu_prod_params.ipa_ep_cfg.nat.nat_en = IPA_SRC_NAT;
-	odu_prod_params.desc_fifo_sz = odu_bridge_ctx->ipa_sys_desc_size;
-	odu_prod_params.priv = odu_bridge_ctx->priv;
-	odu_prod_params.notify = odu_bridge_ctx->tx_dp_notify;
-	res = ipa_setup_sys_pipe(&odu_prod_params,
-		&odu_bridge_ctx->odu_prod_hdl);
-	if (res) {
-		ODU_BRIDGE_ERR("fail to setup sys pipe ODU_PROD %d\n", res);
-		goto fail_odu_prod;
-	}
-
-	/* configure TX (IPA->ODU) EP */
-	odu_emb_cons_params.client = IPA_CLIENT_ODU_EMB_CONS;
-	odu_emb_cons_params.ipa_ep_cfg.hdr.hdr_len = ETH_HLEN;
-	odu_emb_cons_params.ipa_ep_cfg.nat.nat_en = IPA_BYPASS_NAT;
-	odu_emb_cons_params.desc_fifo_sz = odu_bridge_ctx->ipa_sys_desc_size;
-	odu_emb_cons_params.priv = odu_bridge_ctx->priv;
-	odu_emb_cons_params.notify = odu_bridge_emb_cons_cb;
-	res = ipa_setup_sys_pipe(&odu_emb_cons_params,
-		&odu_bridge_ctx->odu_emb_cons_hdl);
-	if (res) {
-		ODU_BRIDGE_ERR("fail to setup sys pipe ODU_EMB_CONS %d\n", res);
-		goto fail_odu_emb_cons;
-	}
-
-	ODU_BRIDGE_DBG("odu_prod_hdl = %d, odu_emb_cons_hdl = %d\n",
-		odu_bridge_ctx->odu_prod_hdl, odu_bridge_ctx->odu_emb_cons_hdl);
-
-	ODU_BRIDGE_FUNC_EXIT();
-
-	return 0;
-
-fail_odu_emb_cons:
-	ipa_teardown_sys_pipe(odu_bridge_ctx->odu_prod_hdl);
-	odu_bridge_ctx->odu_prod_hdl = 0;
-fail_odu_prod:
-	return res;
-}
-
-static int odu_bridge_connect_bridge(void)
-{
-	struct ipa_sys_connect_params odu_prod_params;
-	struct ipa_sys_connect_params odu_emb_cons_params;
-	struct ipa_sys_connect_params odu_teth_cons_params;
-	int res;
-
-	ODU_BRIDGE_FUNC_ENTRY();
-
-	memset(&odu_prod_params, 0, sizeof(odu_prod_params));
-	memset(&odu_emb_cons_params, 0, sizeof(odu_emb_cons_params));
-
-	/* configure RX (ODU->IPA) EP */
-	odu_prod_params.client = IPA_CLIENT_ODU_PROD;
-	odu_prod_params.desc_fifo_sz = IPA_ODU_SYS_DESC_FIFO_SZ;
-	odu_prod_params.priv = odu_bridge_ctx->priv;
-	odu_prod_params.notify = odu_bridge_ctx->tx_dp_notify;
-	odu_prod_params.skip_ep_cfg = true;
-	res = ipa_setup_sys_pipe(&odu_prod_params,
-		&odu_bridge_ctx->odu_prod_hdl);
-	if (res) {
-		ODU_BRIDGE_ERR("fail to setup sys pipe ODU_PROD %d\n", res);
-		goto fail_odu_prod;
-	}
-
-	/* configure TX tethered (IPA->ODU) EP */
-	odu_teth_cons_params.client = IPA_CLIENT_ODU_TETH_CONS;
-	odu_teth_cons_params.desc_fifo_sz = IPA_ODU_SYS_DESC_FIFO_SZ;
-	odu_teth_cons_params.priv = odu_bridge_ctx->priv;
-	odu_teth_cons_params.notify = odu_bridge_teth_cons_cb;
-	odu_teth_cons_params.skip_ep_cfg = true;
-	res = ipa_setup_sys_pipe(&odu_teth_cons_params,
-		&odu_bridge_ctx->odu_teth_cons_hdl);
-	if (res) {
-		ODU_BRIDGE_ERR("fail to setup sys pipe ODU_TETH_CONS %d\n",
-				res);
-		goto fail_odu_teth_cons;
-	}
-
-	/* configure TX embedded(IPA->ODU) EP */
-	odu_emb_cons_params.client = IPA_CLIENT_ODU_EMB_CONS;
-	odu_emb_cons_params.ipa_ep_cfg.hdr.hdr_len = ETH_HLEN;
-	odu_emb_cons_params.ipa_ep_cfg.nat.nat_en = IPA_BYPASS_NAT;
-	odu_emb_cons_params.desc_fifo_sz = IPA_ODU_SYS_DESC_FIFO_SZ;
-	odu_emb_cons_params.priv = odu_bridge_ctx->priv;
-	odu_emb_cons_params.notify = odu_bridge_emb_cons_cb;
-	res = ipa_setup_sys_pipe(&odu_emb_cons_params,
-		&odu_bridge_ctx->odu_emb_cons_hdl);
-	if (res) {
-		ODU_BRIDGE_ERR("fail to setup sys pipe ODU_EMB_CONS %d\n", res);
-		goto fail_odu_emb_cons;
-	}
-
-	ODU_BRIDGE_DBG_LOW("odu_prod_hdl = %d, odu_emb_cons_hdl = %d\n",
-		odu_bridge_ctx->odu_prod_hdl, odu_bridge_ctx->odu_emb_cons_hdl);
-	ODU_BRIDGE_DBG_LOW("odu_teth_cons_hdl = %d\n",
-		odu_bridge_ctx->odu_teth_cons_hdl);
-
-	ODU_BRIDGE_FUNC_EXIT();
-
-	return 0;
-
-fail_odu_emb_cons:
-	ipa_teardown_sys_pipe(odu_bridge_ctx->odu_teth_cons_hdl);
-	odu_bridge_ctx->odu_teth_cons_hdl = 0;
-fail_odu_teth_cons:
-	ipa_teardown_sys_pipe(odu_bridge_ctx->odu_prod_hdl);
-	odu_bridge_ctx->odu_prod_hdl = 0;
-fail_odu_prod:
-	return res;
-}
-
-static int odu_bridge_disconnect_router(void)
-{
-	int res;
-
-	ODU_BRIDGE_FUNC_ENTRY();
-
-	res = ipa_teardown_sys_pipe(odu_bridge_ctx->odu_prod_hdl);
-	if (res)
-		ODU_BRIDGE_ERR("teardown ODU PROD failed\n");
-	odu_bridge_ctx->odu_prod_hdl = 0;
-
-	res = ipa_teardown_sys_pipe(odu_bridge_ctx->odu_emb_cons_hdl);
-	if (res)
-		ODU_BRIDGE_ERR("teardown ODU EMB CONS failed\n");
-	odu_bridge_ctx->odu_emb_cons_hdl = 0;
-
-	ODU_BRIDGE_FUNC_EXIT();
-
-	return 0;
-}
-
-static int odu_bridge_disconnect_bridge(void)
-{
-	int res;
-
-	ODU_BRIDGE_FUNC_ENTRY();
-
-	res = ipa_teardown_sys_pipe(odu_bridge_ctx->odu_prod_hdl);
-	if (res)
-		ODU_BRIDGE_ERR("teardown ODU PROD failed\n");
-	odu_bridge_ctx->odu_prod_hdl = 0;
-
-	res = ipa_teardown_sys_pipe(odu_bridge_ctx->odu_teth_cons_hdl);
-	if (res)
-		ODU_BRIDGE_ERR("teardown ODU TETH CONS failed\n");
-	odu_bridge_ctx->odu_teth_cons_hdl = 0;
-
-	res = ipa_teardown_sys_pipe(odu_bridge_ctx->odu_emb_cons_hdl);
-	if (res)
-		ODU_BRIDGE_ERR("teardown ODU EMB CONS failed\n");
-	odu_bridge_ctx->odu_emb_cons_hdl = 0;
-
-	return 0;
-}
-
-/**
- * odu_bridge_disconnect() - Disconnect odu bridge
- *
- * Disconnect all pipes
- *
- * Return codes: 0- success, error otherwise
- */
-int odu_bridge_disconnect(void)
-{
-	int res;
-
-	ODU_BRIDGE_FUNC_ENTRY();
-
-	if (!odu_bridge_ctx) {
-		ODU_BRIDGE_ERR("Not initialized\n");
-		return -EFAULT;
-	}
-
-	if (!odu_bridge_ctx->is_connected) {
-		ODU_BRIDGE_ERR("Not connected\n");
-		return -EFAULT;
-	}
-
-	mutex_lock(&odu_bridge_ctx->lock);
-	if (odu_bridge_ctx->mode == ODU_BRIDGE_MODE_ROUTER) {
-		res = odu_bridge_disconnect_router();
-		if (res) {
-			ODU_BRIDGE_ERR("disconnect_router failed %d\n", res);
-			goto out;
-		}
-	} else {
-		res = odu_bridge_disconnect_bridge();
-		if (res) {
-			ODU_BRIDGE_ERR("disconnect_bridge failed %d\n", res);
-			goto out;
-		}
-	}
-
-	odu_bridge_ctx->is_connected = false;
-	res = 0;
-out:
-	mutex_unlock(&odu_bridge_ctx->lock);
-	ODU_BRIDGE_FUNC_EXIT();
-	return res;
-}
-EXPORT_SYMBOL(odu_bridge_disconnect);
-
-/**
- * odu_bridge_connect() - Connect odu bridge.
- *
- * Call to the mode-specific connect function for connection IPA pipes
- * Return codes: 0: success
- *		-EINVAL: invalid parameters
- *		-EPERM: Operation not permitted as the bridge is already
- *		connected
- */
-int odu_bridge_connect(void)
-{
-	int res;
-
-	ODU_BRIDGE_FUNC_ENTRY();
-
-	if (!odu_bridge_ctx) {
-		ODU_BRIDGE_ERR("Not initialized\n");
-		return -EFAULT;
-	}
-
-	if (odu_bridge_ctx->is_connected) {
-		ODU_BRIDGE_ERR("already connected\n");
-		return -EFAULT;
-	}
-
-	mutex_lock(&odu_bridge_ctx->lock);
-	if (odu_bridge_ctx->mode == ODU_BRIDGE_MODE_ROUTER) {
-		res = odu_bridge_connect_router();
-		if (res) {
-			ODU_BRIDGE_ERR("connect_router failed\n");
-			goto bail;
-		}
-	} else {
-		res = odu_bridge_connect_bridge();
-		if (res) {
-			ODU_BRIDGE_ERR("connect_bridge failed\n");
-			goto bail;
-		}
-	}
-
-	odu_bridge_ctx->is_connected = true;
-	res = 0;
-bail:
-	mutex_unlock(&odu_bridge_ctx->lock);
-	ODU_BRIDGE_FUNC_EXIT();
-	return res;
-}
-EXPORT_SYMBOL(odu_bridge_connect);
-
-/**
- * odu_bridge_set_mode() - Set bridge mode to Router/Bridge
- * @mode: mode to be set
- */
-static int odu_bridge_set_mode(enum odu_bridge_mode mode)
-{
-	int res;
-
-	ODU_BRIDGE_FUNC_ENTRY();
-
-	if (mode < 0 || mode >= ODU_BRIDGE_MODE_MAX) {
-		ODU_BRIDGE_ERR("Unsupported mode: %d\n", mode);
-		return -EFAULT;
-	}
-
-	ODU_BRIDGE_DBG_LOW("setting mode: %d\n", mode);
-	mutex_lock(&odu_bridge_ctx->lock);
-
-	if (odu_bridge_ctx->mode == mode) {
-		ODU_BRIDGE_DBG_LOW("same mode\n");
-		res = 0;
-		goto bail;
-	}
-
-	if (odu_bridge_ctx->is_connected) {
-		/* first disconnect the old configuration */
-		if (odu_bridge_ctx->mode == ODU_BRIDGE_MODE_ROUTER) {
-			res = odu_bridge_disconnect_router();
-			if (res) {
-				ODU_BRIDGE_ERR("disconnect_router failed\n");
-				goto bail;
-			}
-		} else {
-			res = odu_bridge_disconnect_bridge();
-			if (res) {
-				ODU_BRIDGE_ERR("disconnect_bridge failed\n");
-				goto bail;
-			}
-		}
-
-		/* connect the new configuration */
-		if (mode == ODU_BRIDGE_MODE_ROUTER) {
-			res = odu_bridge_connect_router();
-			if (res) {
-				ODU_BRIDGE_ERR("connect_router failed\n");
-				goto bail;
-			}
-		} else {
-			res = odu_bridge_connect_bridge();
-			if (res) {
-				ODU_BRIDGE_ERR("connect_bridge failed\n");
-				goto bail;
-			}
-		}
-	}
-	odu_bridge_ctx->mode = mode;
-	res = 0;
-bail:
-	mutex_unlock(&odu_bridge_ctx->lock);
-	ODU_BRIDGE_FUNC_EXIT();
-	return res;
-};
-
-/**
- * odu_bridge_set_llv6_addr() - Set link local ipv6 address
- * @llv6_addr: odu network interface link local address
- *
- * This function sets the link local ipv6 address provided by IOCTL
- */
-static int odu_bridge_set_llv6_addr(struct in6_addr *llv6_addr)
-{
-	struct in6_addr llv6_addr_host;
-
-	ODU_BRIDGE_FUNC_ENTRY();
-
-	llv6_addr_host.s6_addr32[0] = ntohl(llv6_addr->s6_addr32[0]);
-	llv6_addr_host.s6_addr32[1] = ntohl(llv6_addr->s6_addr32[1]);
-	llv6_addr_host.s6_addr32[2] = ntohl(llv6_addr->s6_addr32[2]);
-	llv6_addr_host.s6_addr32[3] = ntohl(llv6_addr->s6_addr32[3]);
-
-	memcpy(&odu_bridge_ctx->llv6_addr, &llv6_addr_host,
-				sizeof(odu_bridge_ctx->llv6_addr));
-	ODU_BRIDGE_DBG_LOW("LLV6 addr: %pI6c\n", &odu_bridge_ctx->llv6_addr);
-
-	ODU_BRIDGE_FUNC_EXIT();
-
-	return 0;
-};
-
-static long odu_bridge_ioctl(struct file *filp,
-			      unsigned int cmd,
-			      unsigned long arg)
-{
-	int res = 0;
-	struct in6_addr llv6_addr;
-
-	ODU_BRIDGE_DBG("cmd=%x nr=%d\n", cmd, _IOC_NR(cmd));
-
-	if ((_IOC_TYPE(cmd) != ODU_BRIDGE_IOC_MAGIC) ||
-	    (_IOC_NR(cmd) >= ODU_BRIDGE_IOCTL_MAX)) {
-		ODU_BRIDGE_ERR("Invalid ioctl\n");
-		return -ENOIOCTLCMD;
-	}
-
-	switch (cmd) {
-	case ODU_BRIDGE_IOC_SET_MODE:
-		ODU_BRIDGE_DBG("ODU_BRIDGE_IOC_SET_MODE ioctl called\n");
-		res = odu_bridge_set_mode(arg);
-		if (res) {
-			ODU_BRIDGE_ERR("Error, res = %d\n", res);
-			break;
-		}
-		break;
-
-	case ODU_BRIDGE_IOC_SET_LLV6_ADDR:
-		ODU_BRIDGE_DBG("ODU_BRIDGE_IOC_SET_LLV6_ADDR ioctl called\n");
-		res = copy_from_user(&llv6_addr,
-			(struct in6_addr *)arg,
-			sizeof(llv6_addr));
-		if (res) {
-			ODU_BRIDGE_ERR("Error, res = %d\n", res);
-			res = -EFAULT;
-			break;
-		}
-
-		res = odu_bridge_set_llv6_addr(&llv6_addr);
-		if (res) {
-			ODU_BRIDGE_ERR("Error, res = %d\n", res);
-			break;
-		}
-		break;
-
-	default:
-		ODU_BRIDGE_ERR("Unknown ioctl: %d\n", cmd);
-		WARN_ON(1);
-	}
-
-	return res;
-}
-
-#ifdef CONFIG_COMPAT
-static long compat_odu_bridge_ioctl(struct file *file,
-	unsigned int cmd, unsigned long arg)
-{
-	switch (cmd) {
-	case ODU_BRIDGE_IOC_SET_LLV6_ADDR32:
-		cmd = ODU_BRIDGE_IOC_SET_LLV6_ADDR;
-		break;
-	case ODU_BRIDGE_IOC_SET_MODE:
-		break;
-	default:
-		return -ENOIOCTLCMD;
-	}
-	return odu_bridge_ioctl(file, cmd, (unsigned long)compat_ptr(arg));
-}
-#endif
-
-#ifdef CONFIG_DEBUG_FS
-static struct dentry *dent;
-static struct dentry *dfile_stats;
-static struct dentry *dfile_mode;
-
-static ssize_t odu_debugfs_stats(struct file *file,
-				  char __user *ubuf,
-				  size_t count,
-				  loff_t *ppos)
-{
-	int nbytes = 0;
-
-	nbytes += scnprintf(&dbg_buff[nbytes],
-			    ODU_MAX_MSG_LEN - nbytes,
-			   "UL packets: %lld\n",
-			    odu_bridge_ctx->stats.num_ul_packets);
-	nbytes += scnprintf(&dbg_buff[nbytes],
-			    ODU_MAX_MSG_LEN - nbytes,
-			   "DL packets: %lld\n",
-			    odu_bridge_ctx->stats.num_dl_packets);
-	nbytes += scnprintf(&dbg_buff[nbytes],
-			    ODU_MAX_MSG_LEN - nbytes,
-			    "LAN packets: %lld\n",
-			    odu_bridge_ctx->stats.num_lan_packets);
-	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, nbytes);
-}
-
-static ssize_t odu_debugfs_hw_bridge_mode_write(struct file *file,
-					const char __user *ubuf,
-					size_t count,
-					loff_t *ppos)
-{
-	unsigned long missing;
-	enum odu_bridge_mode mode;
-
-	if (count >= sizeof(dbg_buff))
-		return -EFAULT;
-
-	missing = copy_from_user(dbg_buff, ubuf, count);
-	if (missing)
-		return -EFAULT;
-
-	if (count > 0)
-		dbg_buff[count-1] = '\0';
-
-	if (strcmp(dbg_buff, "router") == 0) {
-		mode = ODU_BRIDGE_MODE_ROUTER;
-	} else if (strcmp(dbg_buff, "bridge") == 0) {
-		mode = ODU_BRIDGE_MODE_BRIDGE;
-	} else {
-		ODU_BRIDGE_ERR("Bad mode, got %s,\n"
-			 "Use <router> or <bridge>.\n", dbg_buff);
-		return count;
-	}
-
-	odu_bridge_set_mode(mode);
-	return count;
-}
-
-static ssize_t odu_debugfs_hw_bridge_mode_read(struct file *file,
-					     char __user *ubuf,
-					     size_t count,
-					     loff_t *ppos)
-{
-	int nbytes = 0;
-
-	switch (odu_bridge_ctx->mode) {
-	case ODU_BRIDGE_MODE_ROUTER:
-		nbytes += scnprintf(&dbg_buff[nbytes],
-			ODU_MAX_MSG_LEN - nbytes,
-			"router\n");
-		break;
-	case ODU_BRIDGE_MODE_BRIDGE:
-		nbytes += scnprintf(&dbg_buff[nbytes],
-			ODU_MAX_MSG_LEN - nbytes,
-			"bridge\n");
-		break;
-	default:
-		nbytes += scnprintf(&dbg_buff[nbytes],
-			ODU_MAX_MSG_LEN - nbytes,
-			"mode error\n");
-		break;
-
-	}
-
-	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, nbytes);
-}
-
-const struct file_operations odu_stats_ops = {
-	.read = odu_debugfs_stats,
-};
-
-const struct file_operations odu_hw_bridge_mode_ops = {
-	.read = odu_debugfs_hw_bridge_mode_read,
-	.write = odu_debugfs_hw_bridge_mode_write,
-};
-
-static void odu_debugfs_init(void)
-{
-	const mode_t read_only_mode = 0444;
-	const mode_t read_write_mode = 0666;
-
-	dent = debugfs_create_dir("odu_ipa_bridge", 0);
-	if (IS_ERR(dent)) {
-		ODU_BRIDGE_ERR("fail to create folder odu_ipa_bridge\n");
-		return;
-	}
-
-	dfile_stats =
-		debugfs_create_file("stats", read_only_mode, dent,
-				    0, &odu_stats_ops);
-	if (!dfile_stats || IS_ERR(dfile_stats)) {
-		ODU_BRIDGE_ERR("fail to create file stats\n");
-		goto fail;
-	}
-
-	dfile_mode =
-		debugfs_create_file("mode", read_write_mode,
-				    dent, 0, &odu_hw_bridge_mode_ops);
-	if (!dfile_mode ||
-	    IS_ERR(dfile_mode)) {
-		ODU_BRIDGE_ERR("fail to create file dfile_mode\n");
-		goto fail;
-	}
-
-	return;
-fail:
-	debugfs_remove_recursive(dent);
-}
-
-static void odu_debugfs_destroy(void)
-{
-	debugfs_remove_recursive(dent);
-}
-
-#else
-static void odu_debugfs_init(void) {}
-static void odu_debugfs_destroy(void) {}
-#endif /* CONFIG_DEBUG_FS */
-
-
-static const struct file_operations odu_bridge_drv_fops = {
-	.owner = THIS_MODULE,
-	.unlocked_ioctl = odu_bridge_ioctl,
-#ifdef CONFIG_COMPAT
-	.compat_ioctl = compat_odu_bridge_ioctl,
-#endif
-};
-
-/**
- * odu_bridge_tx_dp() - Send skb to ODU bridge
- * @skb: skb to send
- * @metadata: metadata on packet
- *
- * This function handles uplink packet.
- * In Router Mode:
- *	packet is sent directly to IPA.
- * In Router Mode:
- *	packet is classified if it should arrive to network stack.
- *	QMI IP packet should arrive to APPS network stack
- *	IPv6 Multicast packet should arrive to APPS network stack and Q6
- *
- * Return codes: 0- success, error otherwise
- */
-int odu_bridge_tx_dp(struct sk_buff *skb, struct ipa_tx_meta *metadata)
-{
-	struct sk_buff *skb_copied = NULL;
-	struct ipv6hdr *ipv6hdr;
-	int res;
-
-	ODU_BRIDGE_FUNC_ENTRY();
-
-	switch (odu_bridge_ctx->mode) {
-	case ODU_BRIDGE_MODE_ROUTER:
-		/* Router mode - pass skb to IPA */
-		res = ipa_tx_dp(IPA_CLIENT_ODU_PROD, skb, metadata);
-		if (res) {
-			ODU_BRIDGE_DBG("tx dp failed %d\n", res);
-			goto out;
-		}
-		odu_bridge_ctx->stats.num_ul_packets++;
-		goto out;
-
-	case ODU_BRIDGE_MODE_BRIDGE:
-		ipv6hdr = (struct ipv6hdr *)(skb->data + ETH_HLEN);
-		if (ipv6hdr->version == 6 &&
-		    ODU_BRIDGE_IS_QMI_ADDR(ipv6hdr->daddr)) {
-			ODU_BRIDGE_DBG_LOW("QMI packet\n");
-			skb_copied = skb_clone(skb, GFP_KERNEL);
-			if (!skb_copied) {
-				ODU_BRIDGE_ERR("No memory\n");
-				return -ENOMEM;
-			}
-			odu_bridge_ctx->tx_dp_notify(odu_bridge_ctx->priv,
-						     IPA_RECEIVE,
-						     (unsigned long)skb_copied);
-			odu_bridge_ctx->tx_dp_notify(odu_bridge_ctx->priv,
-						     IPA_WRITE_DONE,
-						     (unsigned long)skb);
-			odu_bridge_ctx->stats.num_ul_packets++;
-			odu_bridge_ctx->stats.num_lan_packets++;
-			res = 0;
-			goto out;
-		}
-
-		if (ipv6hdr->version == 6 &&
-		    ipv6_addr_is_multicast(&ipv6hdr->daddr)) {
-			ODU_BRIDGE_DBG_LOW(
-				"Multicast pkt, send to APPS and IPA\n");
-			skb_copied = skb_clone(skb, GFP_KERNEL);
-			if (!skb_copied) {
-				ODU_BRIDGE_ERR("No memory\n");
-				return -ENOMEM;
-			}
-
-			res = ipa_tx_dp(IPA_CLIENT_ODU_PROD, skb, metadata);
-			if (res) {
-				ODU_BRIDGE_DBG("tx dp failed %d\n", res);
-				dev_kfree_skb(skb_copied);
-				goto out;
-			}
-
-			odu_bridge_ctx->tx_dp_notify(odu_bridge_ctx->priv,
-						     IPA_RECEIVE,
-						     (unsigned long)skb_copied);
-			odu_bridge_ctx->stats.num_ul_packets++;
-			odu_bridge_ctx->stats.num_lan_packets++;
-			goto out;
-		}
-
-		res = ipa_tx_dp(IPA_CLIENT_ODU_PROD, skb, metadata);
-		if (res) {
-			ODU_BRIDGE_DBG("tx dp failed %d\n", res);
-			goto out;
-		}
-		odu_bridge_ctx->stats.num_ul_packets++;
-		goto out;
-
-	default:
-		ODU_BRIDGE_ERR("Unsupported mode: %d\n", odu_bridge_ctx->mode);
-		WARN_ON(1);
-		res = -EFAULT;
-
-	}
-out:
-	ODU_BRIDGE_FUNC_EXIT();
-	return res;
-}
-EXPORT_SYMBOL(odu_bridge_tx_dp);
-
-static int odu_bridge_add_hdrs(void)
-{
-	struct ipa_ioc_add_hdr *hdrs;
-	struct ipa_hdr_add *ipv4_hdr;
-	struct ipa_hdr_add *ipv6_hdr;
-	struct ethhdr *eth_ipv4;
-	struct ethhdr *eth_ipv6;
-	int res;
-
-	ODU_BRIDGE_FUNC_ENTRY();
-	hdrs = kzalloc(sizeof(*hdrs) + sizeof(*ipv4_hdr) + sizeof(*ipv6_hdr),
-			GFP_KERNEL);
-	if (!hdrs) {
-		ODU_BRIDGE_ERR("no mem\n");
-		res = -ENOMEM;
-		goto out;
-	}
-	ipv4_hdr = &hdrs->hdr[0];
-	eth_ipv4 = (struct ethhdr *)(ipv4_hdr->hdr);
-	ipv6_hdr = &hdrs->hdr[1];
-	eth_ipv6 = (struct ethhdr *)(ipv6_hdr->hdr);
-	strlcpy(ipv4_hdr->name, ODU_BRIDGE_IPV4_HDR_NAME,
-		IPA_RESOURCE_NAME_MAX);
-	memcpy(eth_ipv4->h_source, odu_bridge_ctx->device_ethaddr, ETH_ALEN);
-	eth_ipv4->h_proto = htons(ETH_P_IP);
-	ipv4_hdr->hdr_len = ETH_HLEN;
-	ipv4_hdr->is_partial = 1;
-	ipv4_hdr->is_eth2_ofst_valid = 1;
-	ipv4_hdr->eth2_ofst = 0;
-	strlcpy(ipv6_hdr->name, ODU_BRIDGE_IPV6_HDR_NAME,
-		IPA_RESOURCE_NAME_MAX);
-	memcpy(eth_ipv6->h_source, odu_bridge_ctx->device_ethaddr, ETH_ALEN);
-	eth_ipv6->h_proto = htons(ETH_P_IPV6);
-	ipv6_hdr->hdr_len = ETH_HLEN;
-	ipv6_hdr->is_partial = 1;
-	ipv6_hdr->is_eth2_ofst_valid = 1;
-	ipv6_hdr->eth2_ofst = 0;
-	hdrs->commit = 1;
-	hdrs->num_hdrs = 2;
-	res = ipa_add_hdr(hdrs);
-	if (res) {
-		ODU_BRIDGE_ERR("Fail on Header-Insertion(%d)\n", res);
-		goto out_free_mem;
-	}
-	if (ipv4_hdr->status) {
-		ODU_BRIDGE_ERR("Fail on Header-Insertion ipv4(%d)\n",
-				ipv4_hdr->status);
-		res = ipv4_hdr->status;
-		goto out_free_mem;
-	}
-	if (ipv6_hdr->status) {
-		ODU_BRIDGE_ERR("Fail on Header-Insertion ipv6(%d)\n",
-				ipv6_hdr->status);
-		res = ipv6_hdr->status;
-		goto out_free_mem;
-	}
-	odu_bridge_ctx->odu_br_ipv4_hdr_hdl = ipv4_hdr->hdr_hdl;
-	odu_bridge_ctx->odu_br_ipv6_hdr_hdl = ipv6_hdr->hdr_hdl;
-
-	res = 0;
-out_free_mem:
-	kfree(hdrs);
-out:
-	ODU_BRIDGE_FUNC_EXIT();
-	return res;
-}
-
-static void odu_bridge_del_hdrs(void)
-{
-	struct ipa_ioc_del_hdr *del_hdr;
-	struct ipa_hdr_del *ipv4;
-	struct ipa_hdr_del *ipv6;
-	int result;
-
-	del_hdr = kzalloc(sizeof(*del_hdr) + sizeof(*ipv4) +
-			sizeof(*ipv6), GFP_KERNEL);
-	if (!del_hdr)
-		return;
-	del_hdr->commit = 1;
-	del_hdr->num_hdls = 2;
-	ipv4 = &del_hdr->hdl[0];
-	ipv4->hdl = odu_bridge_ctx->odu_br_ipv4_hdr_hdl;
-	ipv6 = &del_hdr->hdl[1];
-	ipv6->hdl = odu_bridge_ctx->odu_br_ipv6_hdr_hdl;
-	result = ipa_del_hdr(del_hdr);
-	if (result || ipv4->status || ipv6->status)
-		ODU_BRIDGE_ERR("ipa_del_hdr failed");
-	kfree(del_hdr);
-}
-
-/**
- * odu_bridge_register_properties() - set Tx/Rx properties for ipacm
- *
- * Register the network interface interface with Tx and Rx properties
- * Tx properties are for data flowing from IPA to adapter, they
- * have Header-Insertion properties both for Ipv4 and Ipv6 Ethernet framing.
- * Rx properties are for data flowing from adapter to IPA, they have
- * simple rule which always "hit".
- *
- */
-static int odu_bridge_register_properties(void)
-{
-	struct ipa_tx_intf tx_properties = {0};
-	struct ipa_ioc_tx_intf_prop properties[2] = { {0}, {0} };
-	struct ipa_ioc_tx_intf_prop *ipv4_property;
-	struct ipa_ioc_tx_intf_prop *ipv6_property;
-	struct ipa_ioc_rx_intf_prop rx_ioc_properties[2] = { {0}, {0} };
-	struct ipa_rx_intf rx_properties = {0};
-	struct ipa_ioc_rx_intf_prop *rx_ipv4_property;
-	struct ipa_ioc_rx_intf_prop *rx_ipv6_property;
-	int res = 0;
-
-	ODU_BRIDGE_FUNC_ENTRY();
-
-	tx_properties.prop = properties;
-	ipv4_property = &tx_properties.prop[0];
-	ipv4_property->ip = IPA_IP_v4;
-	ipv4_property->dst_pipe = IPA_CLIENT_ODU_EMB_CONS;
-	ipv4_property->hdr_l2_type = IPA_HDR_L2_ETHERNET_II;
-	strlcpy(ipv4_property->hdr_name, ODU_BRIDGE_IPV4_HDR_NAME,
-			IPA_RESOURCE_NAME_MAX);
-	ipv6_property = &tx_properties.prop[1];
-	ipv6_property->ip = IPA_IP_v6;
-	ipv6_property->dst_pipe = IPA_CLIENT_ODU_EMB_CONS;
-	ipv6_property->hdr_l2_type = IPA_HDR_L2_ETHERNET_II;
-	strlcpy(ipv6_property->hdr_name, ODU_BRIDGE_IPV6_HDR_NAME,
-			IPA_RESOURCE_NAME_MAX);
-	tx_properties.num_props = 2;
-
-	rx_properties.prop = rx_ioc_properties;
-	rx_ipv4_property = &rx_properties.prop[0];
-	rx_ipv4_property->ip = IPA_IP_v4;
-	rx_ipv4_property->attrib.attrib_mask = 0;
-	rx_ipv4_property->src_pipe = IPA_CLIENT_ODU_PROD;
-	rx_ipv4_property->hdr_l2_type = IPA_HDR_L2_ETHERNET_II;
-	rx_ipv6_property = &rx_properties.prop[1];
-	rx_ipv6_property->ip = IPA_IP_v6;
-	rx_ipv6_property->attrib.attrib_mask = 0;
-	rx_ipv6_property->src_pipe = IPA_CLIENT_ODU_PROD;
-	rx_ipv6_property->hdr_l2_type = IPA_HDR_L2_ETHERNET_II;
-	rx_properties.num_props = 2;
-
-	res = ipa_register_intf(odu_bridge_ctx->netdev_name, &tx_properties,
-		&rx_properties);
-	if (res) {
-		ODU_BRIDGE_ERR("fail on Tx/Rx properties registration %d\n",
-									res);
-	}
-
-	ODU_BRIDGE_FUNC_EXIT();
-
-	return res;
-}
-
-static void odu_bridge_deregister_properties(void)
-{
-	int res;
-
-	ODU_BRIDGE_FUNC_ENTRY();
-	res = ipa_deregister_intf(odu_bridge_ctx->netdev_name);
-	if (res)
-		ODU_BRIDGE_ERR("Fail on Tx prop deregister %d\n", res);
-	ODU_BRIDGE_FUNC_EXIT();
-}
-
-/**
- * odu_bridge_init() - Initialize the ODU bridge driver
- * @params: initialization parameters
- *
- * This function initialize all bridge internal data and register odu bridge to
- * kernel for IOCTL and debugfs.
- * Header addition and properties are registered to IPA driver.
- *
- * Return codes: 0: success,
- *		-EINVAL - Bad parameter
- *		Other negative value - Failure
- */
-int odu_bridge_init(struct odu_bridge_params *params)
-{
-	int res;
-
-	ODU_BRIDGE_FUNC_ENTRY();
-
-	if (!params) {
-		ODU_BRIDGE_ERR("null pointer params\n");
-		return -EINVAL;
-	}
-	if (!params->netdev_name) {
-		ODU_BRIDGE_ERR("null pointer params->netdev_name\n");
-		return -EINVAL;
-	}
-	if (!params->tx_dp_notify) {
-		ODU_BRIDGE_ERR("null pointer params->tx_dp_notify\n");
-		return -EINVAL;
-	}
-	if (!params->send_dl_skb) {
-		ODU_BRIDGE_ERR("null pointer params->send_dl_skb\n");
-		return -EINVAL;
-	}
-	if (odu_bridge_ctx) {
-		ODU_BRIDGE_ERR("Already initialized\n");
-		return -EFAULT;
-	}
-	if (!ipa_is_ready()) {
-		ODU_BRIDGE_ERR("IPA is not ready\n");
-		return -EFAULT;
-	}
-
-	ODU_BRIDGE_DBG("device_ethaddr=%pM\n", params->device_ethaddr);
-
-	odu_bridge_ctx = kzalloc(sizeof(*odu_bridge_ctx), GFP_KERNEL);
-	if (!odu_bridge_ctx) {
-		ODU_BRIDGE_ERR("kzalloc err.\n");
-		return -ENOMEM;
-	}
-
-	odu_bridge_ctx->class = class_create(THIS_MODULE, ODU_BRIDGE_DRV_NAME);
-	if (!odu_bridge_ctx->class) {
-		ODU_BRIDGE_ERR("Class_create err.\n");
-		res = -ENODEV;
-		goto fail_class_create;
-	}
-
-	res = alloc_chrdev_region(&odu_bridge_ctx->dev_num, 0, 1,
-				  ODU_BRIDGE_DRV_NAME);
-	if (res) {
-		ODU_BRIDGE_ERR("alloc_chrdev_region err.\n");
-		res = -ENODEV;
-		goto fail_alloc_chrdev_region;
-	}
-
-	odu_bridge_ctx->dev = device_create(odu_bridge_ctx->class, NULL,
-		odu_bridge_ctx->dev_num, odu_bridge_ctx, ODU_BRIDGE_DRV_NAME);
-	if (IS_ERR(odu_bridge_ctx->dev)) {
-		ODU_BRIDGE_ERR(":device_create err.\n");
-		res = -ENODEV;
-		goto fail_device_create;
-	}
-
-	cdev_init(&odu_bridge_ctx->cdev, &odu_bridge_drv_fops);
-	odu_bridge_ctx->cdev.owner = THIS_MODULE;
-	odu_bridge_ctx->cdev.ops = &odu_bridge_drv_fops;
-
-	res = cdev_add(&odu_bridge_ctx->cdev, odu_bridge_ctx->dev_num, 1);
-	if (res) {
-		ODU_BRIDGE_ERR(":cdev_add err=%d\n", -res);
-		res = -ENODEV;
-		goto fail_cdev_add;
-	}
-
-	odu_debugfs_init();
-
-	strlcpy(odu_bridge_ctx->netdev_name, params->netdev_name,
-		IPA_RESOURCE_NAME_MAX);
-	odu_bridge_ctx->priv = params->priv;
-	odu_bridge_ctx->tx_dp_notify = params->tx_dp_notify;
-	odu_bridge_ctx->send_dl_skb = params->send_dl_skb;
-	memcpy(odu_bridge_ctx->device_ethaddr, params->device_ethaddr,
-		ETH_ALEN);
-	odu_bridge_ctx->ipa_sys_desc_size = params->ipa_desc_size;
-	odu_bridge_ctx->mode = ODU_BRIDGE_MODE_ROUTER;
-
-	mutex_init(&odu_bridge_ctx->lock);
-
-	res = odu_bridge_add_hdrs();
-	if (res) {
-		ODU_BRIDGE_ERR("fail on odu_bridge_add_hdr %d\n", res);
-		goto fail_add_hdrs;
-	}
-
-	res = odu_bridge_register_properties();
-	if (res) {
-		ODU_BRIDGE_ERR("fail on register properties %d\n", res);
-		goto fail_register_properties;
-	}
-
-	ODU_BRIDGE_FUNC_EXIT();
-	return 0;
-
-fail_register_properties:
-	odu_bridge_del_hdrs();
-fail_add_hdrs:
-	odu_debugfs_destroy();
-fail_cdev_add:
-	device_destroy(odu_bridge_ctx->class, odu_bridge_ctx->dev_num);
-fail_device_create:
-	unregister_chrdev_region(odu_bridge_ctx->dev_num, 1);
-fail_alloc_chrdev_region:
-	class_destroy(odu_bridge_ctx->class);
-fail_class_create:
-	kfree(odu_bridge_ctx);
-	odu_bridge_ctx = NULL;
-	return res;
-}
-EXPORT_SYMBOL(odu_bridge_init);
-
-/**
- * odu_bridge_cleanup() - De-Initialize the ODU bridge driver
- *
- * Return codes: 0: success,
- *		-EINVAL - Bad parameter
- *		Other negative value - Failure
- */
-int odu_bridge_cleanup(void)
-{
-	ODU_BRIDGE_FUNC_ENTRY();
-
-	if (!odu_bridge_ctx) {
-		ODU_BRIDGE_ERR("Not initialized\n");
-		return -EFAULT;
-	}
-
-	if (odu_bridge_ctx->is_connected) {
-		ODU_BRIDGE_ERR("cannot deinit while bridge is conncetd\n");
-		return -EFAULT;
-	}
-
-	odu_bridge_deregister_properties();
-	odu_bridge_del_hdrs();
-	odu_debugfs_destroy();
-	cdev_del(&odu_bridge_ctx->cdev);
-	device_destroy(odu_bridge_ctx->class, odu_bridge_ctx->dev_num);
-	unregister_chrdev_region(odu_bridge_ctx->dev_num, 1);
-	class_destroy(odu_bridge_ctx->class);
-	ipc_log_context_destroy(odu_bridge_ctx->logbuf);
-	ipc_log_context_destroy(odu_bridge_ctx->logbuf_low);
-	kfree(odu_bridge_ctx);
-	odu_bridge_ctx = NULL;
-
-	ODU_BRIDGE_FUNC_EXIT();
-	return 0;
-}
-EXPORT_SYMBOL(odu_bridge_cleanup);
-
-MODULE_LICENSE("GPL v2");
-MODULE_DESCRIPTION("ODU bridge driver");

ipa/ipa_clients/rndis_ipa.c

@@ -1,2435 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2013-2019, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/atomic.h>
-#include <linux/errno.h>
-#include <linux/etherdevice.h>
-#include <linux/if_vlan.h>
-#include <linux/debugfs.h>
-#include <linux/in.h>
-#include <linux/stddef.h>
-#include <linux/ip.h>
-#include <linux/fs.h>
-#include <linux/module.h>
-#include <linux/msm_ipa.h>
-#include <linux/netdevice.h>
-#include <linux/skbuff.h>
-#include <linux/sched.h>
-#include <linux/ipa.h>
-#include <linux/random.h>
-#include <linux/rndis_ipa.h>
-#include <linux/workqueue.h>
-#include "../ipa_common_i.h"
-#include "../ipa_v3/ipa_pm.h"
-
-#define CREATE_TRACE_POINTS
-#include "rndis_ipa_trace.h"
-
-#define DRV_NAME "RNDIS_IPA"
-#define DEBUGFS_DIR_NAME "rndis_ipa"
-#define DEBUGFS_AGGR_DIR_NAME "rndis_ipa_aggregation"
-#define NETDEV_NAME "rndis"
-#define IPV4_HDR_NAME "rndis_eth_ipv4"
-#define IPV6_HDR_NAME "rndis_eth_ipv6"
-#define IPA_TO_USB_CLIENT IPA_CLIENT_USB_CONS
-#define INACTIVITY_MSEC_DELAY 100
-#define DEFAULT_OUTSTANDING_HIGH 64
-#define DEFAULT_OUTSTANDING_LOW 32
-#define DEBUGFS_TEMP_BUF_SIZE 4
-#define RNDIS_IPA_PKT_TYPE 0x00000001
-#define RNDIS_IPA_DFLT_RT_HDL 0
-#define FROM_IPA_TO_USB_BAMDMA 4
-#define FROM_USB_TO_IPA_BAMDMA 5
-#define BAM_DMA_MAX_PKT_NUMBER 10
-#define BAM_DMA_DATA_FIFO_SIZE \
-		(BAM_DMA_MAX_PKT_NUMBER * \
-			(ETH_FRAME_LEN + sizeof(struct rndis_pkt_hdr)))
-#define BAM_DMA_DESC_FIFO_SIZE \
-		(BAM_DMA_MAX_PKT_NUMBER * (sizeof(struct sps_iovec)))
-#define TX_TIMEOUT (5 * HZ)
-#define MIN_TX_ERROR_SLEEP_PERIOD 500
-#define DEFAULT_AGGR_TIME_LIMIT 1000 /* 1ms */
-#define DEFAULT_AGGR_PKT_LIMIT 0
-
-#define IPA_RNDIS_IPC_LOG_PAGES 50
-
-#define IPA_RNDIS_IPC_LOGGING(buf, fmt, args...) \
-	do { \
-		if (buf) \
-			ipc_log_string((buf), fmt, __func__, __LINE__, \
-				## args); \
-	} while (0)
-
-static void *ipa_rndis_logbuf;
-
-#define RNDIS_IPA_DEBUG(fmt, args...) \
-	do { \
-		pr_debug(DRV_NAME " %s:%d " fmt, __func__, __LINE__, ## args);\
-		if (ipa_rndis_logbuf) { \
-			IPA_RNDIS_IPC_LOGGING(ipa_rndis_logbuf, \
-				DRV_NAME " %s:%d " fmt, ## args); \
-		} \
-	} while (0)
-
-#define RNDIS_IPA_DEBUG_XMIT(fmt, args...) \
-	pr_debug(DRV_NAME " %s:%d " fmt, __func__, __LINE__, ## args)
-
-#define RNDIS_IPA_ERROR(fmt, args...) \
-	do { \
-		pr_err(DRV_NAME "@%s@%d@ctx:%s: "\
-			fmt, __func__, __LINE__, current->comm, ## args);\
-		if (ipa_rndis_logbuf) { \
-			IPA_RNDIS_IPC_LOGGING(ipa_rndis_logbuf, \
-				DRV_NAME " %s:%d " fmt, ## args); \
-		} \
-	} while (0)
-
-#define NULL_CHECK_RETVAL(ptr) \
-		do { \
-			if (!(ptr)) { \
-				RNDIS_IPA_ERROR("null pointer #ptr\n"); \
-				ret = -EINVAL; \
-			} \
-		} \
-		while (0)
-
-#define RNDIS_HDR_OFST(field) offsetof(struct rndis_pkt_hdr, field)
-#define RNDIS_IPA_LOG_ENTRY() RNDIS_IPA_DEBUG("begin\n")
-#define RNDIS_IPA_LOG_EXIT()  RNDIS_IPA_DEBUG("end\n")
-
-/**
- * enum rndis_ipa_state - specify the current driver internal state
- *  which is guarded by a state machine.
- *
- * The driver internal state changes due to its external API usage.
- * The driver saves its internal state to guard from caller illegal
- * call sequence.
- * states:
- * UNLOADED is the first state which is the default one and is also the state
- *  after the driver gets unloaded(cleanup).
- * INITIALIZED is the driver state once it finished registering
- *  the network device and all internal data struct were initialized
- * CONNECTED is the driver state once the USB pipes were connected to IPA
- * UP is the driver state after the interface mode was set to UP but the
- *  pipes are not connected yet - this state is meta-stable state.
- * CONNECTED_AND_UP is the driver state when the pipe were connected and
- *  the interface got UP request from the network stack. this is the driver
- *   idle operation state which allows it to transmit/receive data.
- * INVALID is a state which is not allowed.
- */
-enum rndis_ipa_state {
-	RNDIS_IPA_UNLOADED          = 0,
-	RNDIS_IPA_INITIALIZED       = 1,
-	RNDIS_IPA_CONNECTED         = 2,
-	RNDIS_IPA_UP                = 3,
-	RNDIS_IPA_CONNECTED_AND_UP  = 4,
-	RNDIS_IPA_INVALID           = 5,
-};
-
-/**
- * enum rndis_ipa_operation - enumerations used to describe the API operation
- *
- * Those enums are used as input for the driver state machine.
- */
-enum rndis_ipa_operation {
-	RNDIS_IPA_INITIALIZE,
-	RNDIS_IPA_CONNECT,
-	RNDIS_IPA_OPEN,
-	RNDIS_IPA_STOP,
-	RNDIS_IPA_DISCONNECT,
-	RNDIS_IPA_CLEANUP,
-};
-
-#define RNDIS_IPA_STATE_DEBUG(ctx) \
-	RNDIS_IPA_DEBUG("Driver state: %s\n",\
-	rndis_ipa_state_string((ctx)->state))
-
-
-/**
- * struct rndis_ipa_dev - main driver context parameters
- *
- * @net: network interface struct implemented by this driver
- * @directory: debugfs directory for various debugging switches
- * @tx_filter: flag that enable/disable Tx path to continue to IPA
- * @tx_dropped: number of filtered out Tx packets
- * @tx_dump_enable: dump all Tx packets
- * @rx_filter: flag that enable/disable Rx path to continue to IPA
- * @rx_dropped: number of filtered out Rx packets
- * @rx_dump_enable: dump all Rx packets
- * @icmp_filter: allow all ICMP packet to pass through the filters
- * @deaggregation_enable: enable/disable IPA HW deaggregation logic
- * @during_xmit_error: flags that indicate that the driver is in a middle
- *  of error handling in Tx path
- * @directory: holds all debug flags used by the driver to allow cleanup
- *  for driver unload
- * @eth_ipv4_hdr_hdl: saved handle for ipv4 header-insertion table
- * @eth_ipv6_hdr_hdl: saved handle for ipv6 header-insertion table
- * @usb_to_ipa_hdl: save handle for IPA pipe operations
- * @ipa_to_usb_hdl: save handle for IPA pipe operations
- * @outstanding_pkts: number of packets sent to IPA without TX complete ACKed
- * @outstanding_high: number of outstanding packets allowed
- * @outstanding_low: number of outstanding packets which shall cause
- *  to netdev queue start (after stopped due to outstanding_high reached)
- * @error_msec_sleep_time: number of msec for sleeping in case of Tx error
- * @state: current state of the driver
- * @host_ethaddr: holds the tethered PC ethernet address
- * @device_ethaddr: holds the device ethernet address
- * @device_ready_notify: callback supplied by USB core driver
- * This callback shall be called by the Netdev once the Netdev internal
- * state is changed to RNDIS_IPA_CONNECTED_AND_UP
- * @xmit_error_delayed_work: work item for cases where IPA driver Tx fails
- * @state_lock: used to protect the state variable.
- * @pm_hdl: handle for IPA PM framework
- * @is_vlan_mode: should driver work in vlan mode?
- */
-struct rndis_ipa_dev {
-	struct net_device *net;
-	bool tx_filter;
-	u32 tx_dropped;
-	bool tx_dump_enable;
-	bool rx_filter;
-	u32 rx_dropped;
-	bool rx_dump_enable;
-	bool icmp_filter;
-	bool deaggregation_enable;
-	bool during_xmit_error;
-	struct dentry *directory;
-	u32 eth_ipv4_hdr_hdl;
-	u32 eth_ipv6_hdr_hdl;
-	u32 usb_to_ipa_hdl;
-	u32 ipa_to_usb_hdl;
-	atomic_t outstanding_pkts;
-	u32 outstanding_high;
-	u32 outstanding_low;
-	u32 error_msec_sleep_time;
-	enum rndis_ipa_state state;
-	u8 host_ethaddr[ETH_ALEN];
-	u8 device_ethaddr[ETH_ALEN];
-	void (*device_ready_notify)(void);
-	struct delayed_work xmit_error_delayed_work;
-	spinlock_t state_lock; /* Spinlock for the state variable.*/
-	u32 pm_hdl;
-	bool is_vlan_mode;
-};
-
-/**
- * rndis_pkt_hdr - RNDIS_IPA representation of REMOTE_NDIS_PACKET_MSG
- * @msg_type: for REMOTE_NDIS_PACKET_MSG this value should be 1
- * @msg_len:  total message length in bytes, including RNDIS header an payload
- * @data_ofst: offset in bytes from start of the data_ofst to payload
- * @data_len: payload size in bytes
- * @zeroes: OOB place holder - not used for RNDIS_IPA.
- */
-struct rndis_pkt_hdr {
-	__le32	msg_type;
-	__le32	msg_len;
-	__le32	data_ofst;
-	__le32	data_len;
-	__le32  zeroes[7];
-} __packed__;
-
-static int rndis_ipa_open(struct net_device *net);
-static void rndis_ipa_packet_receive_notify
-	(void *private, enum ipa_dp_evt_type evt, unsigned long data);
-static void rndis_ipa_tx_complete_notify
-	(void *private, enum ipa_dp_evt_type evt, unsigned long data);
-static void rndis_ipa_tx_timeout(struct net_device *net);
-static int rndis_ipa_stop(struct net_device *net);
-static void rndis_ipa_enable_data_path(struct rndis_ipa_dev *rndis_ipa_ctx);
-static struct sk_buff *rndis_encapsulate_skb(struct sk_buff *skb,
-	struct rndis_ipa_dev *rndis_ipa_ctx);
-static void rndis_ipa_xmit_error(struct sk_buff *skb);
-static void rndis_ipa_xmit_error_aftercare_wq(struct work_struct *work);
-static void rndis_ipa_prepare_header_insertion
-	(int eth_type,
-	const char *hdr_name, struct ipa_hdr_add *add_hdr,
-	const void *dst_mac, const void *src_mac, bool is_vlan_mode);
-static int rndis_ipa_hdrs_cfg
-	(struct rndis_ipa_dev *rndis_ipa_ctx,
-	const void *dst_mac, const void *src_mac);
-static int rndis_ipa_hdrs_destroy(struct rndis_ipa_dev *rndis_ipa_ctx);
-static struct net_device_stats *rndis_ipa_get_stats(struct net_device *net);
-static int rndis_ipa_register_properties(char *netdev_name, bool is_vlan_mode);
-static int rndis_ipa_deregister_properties(char *netdev_name);
-static int rndis_ipa_register_pm_client(struct rndis_ipa_dev *rndis_ipa_ctx);
-static int rndis_ipa_deregister_pm_client(struct rndis_ipa_dev *rndis_ipa_ctx);
-static bool rx_filter(struct sk_buff *skb);
-static bool tx_filter(struct sk_buff *skb);
-static netdev_tx_t rndis_ipa_start_xmit
-	(struct sk_buff *skb, struct net_device *net);
-static int rndis_ipa_debugfs_atomic_open
-	(struct inode *inode, struct file *file);
-static int rndis_ipa_debugfs_aggr_open
-	(struct inode *inode, struct file *file);
-static ssize_t rndis_ipa_debugfs_aggr_write
-	(struct file *file,
-	const char __user *buf, size_t count, loff_t *ppos);
-static ssize_t rndis_ipa_debugfs_atomic_read
-	(struct file *file,
-	char __user *ubuf, size_t count, loff_t *ppos);
-static void rndis_ipa_dump_skb(struct sk_buff *skb);
-static void rndis_ipa_debugfs_init(struct rndis_ipa_dev *rndis_ipa_ctx);
-static void rndis_ipa_debugfs_destroy(struct rndis_ipa_dev *rndis_ipa_ctx);
-static int rndis_ipa_ep_registers_cfg
-	(u32 usb_to_ipa_hdl,
-	u32 ipa_to_usb_hdl, u32 max_xfer_size_bytes_to_dev,
-	u32 max_xfer_size_bytes_to_host, u32 mtu,
-	bool deaggr_enable,
-	bool is_vlan_mode);
-static int rndis_ipa_set_device_ethernet_addr
-	(u8 *dev_ethaddr,
-	u8 device_ethaddr[]);
-static enum rndis_ipa_state rndis_ipa_next_state
-	(enum rndis_ipa_state current_state,
-	enum rndis_ipa_operation operation);
-static const char *rndis_ipa_state_string(enum rndis_ipa_state state);
-static int rndis_ipa_init_module(void);
-static void rndis_ipa_cleanup_module(void);
-
-struct rndis_ipa_dev *rndis_ipa;
-
-static const struct net_device_ops rndis_ipa_netdev_ops = {
-	.ndo_open		= rndis_ipa_open,
-	.ndo_stop		= rndis_ipa_stop,
-	.ndo_start_xmit = rndis_ipa_start_xmit,
-	.ndo_tx_timeout = rndis_ipa_tx_timeout,
-	.ndo_get_stats = rndis_ipa_get_stats,
-	.ndo_set_mac_address = eth_mac_addr,
-};
-
-const struct file_operations rndis_ipa_debugfs_atomic_ops = {
-	.open = rndis_ipa_debugfs_atomic_open,
-	.read = rndis_ipa_debugfs_atomic_read,
-};
-
-const struct file_operations rndis_ipa_aggr_ops = {
-		.open = rndis_ipa_debugfs_aggr_open,
-		.write = rndis_ipa_debugfs_aggr_write,
-};
-
-static struct ipa_ep_cfg ipa_to_usb_ep_cfg = {
-	.mode = {
-		.mode = IPA_BASIC,
-		.dst  = IPA_CLIENT_APPS_LAN_CONS,
-	},
-	.hdr = {
-		.hdr_len = ETH_HLEN + sizeof(struct rndis_pkt_hdr),
-		.hdr_ofst_metadata_valid = false,
-		.hdr_ofst_metadata = 0,
-		.hdr_additional_const_len = ETH_HLEN,
-		.hdr_ofst_pkt_size_valid = true,
-		.hdr_ofst_pkt_size = 3 * sizeof(u32),
-		.hdr_a5_mux = false,
-		.hdr_remove_additional = false,
-		.hdr_metadata_reg_valid = false,
-	},
-	.hdr_ext = {
-		.hdr_pad_to_alignment = 0,
-		.hdr_total_len_or_pad_offset = 1 * sizeof(u32),
-		.hdr_payload_len_inc_padding = false,
-		.hdr_total_len_or_pad = IPA_HDR_TOTAL_LEN,
-		.hdr_total_len_or_pad_valid = true,
-		.hdr_little_endian = true,
-	},
-	.aggr = {
-		.aggr_en = IPA_ENABLE_AGGR,
-		.aggr = IPA_GENERIC,
-		.aggr_byte_limit = 4,
-		.aggr_time_limit = DEFAULT_AGGR_TIME_LIMIT,
-		.aggr_pkt_limit = DEFAULT_AGGR_PKT_LIMIT,
-	},
-	.deaggr = {
-		.deaggr_hdr_len = 0,
-		.packet_offset_valid = 0,
-		.packet_offset_location = 0,
-		.max_packet_len = 0,
-	},
-	.route = {
-		.rt_tbl_hdl = RNDIS_IPA_DFLT_RT_HDL,
-	},
-	.nat = {
-		.nat_en = IPA_SRC_NAT,
-	},
-};
-
-static struct ipa_ep_cfg usb_to_ipa_ep_cfg_deaggr_dis = {
-	.mode = {
-		.mode = IPA_BASIC,
-		.dst  = IPA_CLIENT_APPS_LAN_CONS,
-	},
-	.hdr = {
-		.hdr_len = ETH_HLEN + sizeof(struct rndis_pkt_hdr),
-		.hdr_ofst_metadata_valid = false,
-		.hdr_ofst_metadata = 0,
-		.hdr_additional_const_len = 0,
-		.hdr_ofst_pkt_size_valid = true,
-		.hdr_ofst_pkt_size = 3 * sizeof(u32) +
-			sizeof(struct rndis_pkt_hdr),
-		.hdr_a5_mux = false,
-		.hdr_remove_additional = false,
-		.hdr_metadata_reg_valid = true,
-	},
-	.hdr_ext = {
-		.hdr_pad_to_alignment = 0,
-		.hdr_total_len_or_pad_offset = 1 * sizeof(u32),
-		.hdr_payload_len_inc_padding = false,
-		.hdr_total_len_or_pad = IPA_HDR_TOTAL_LEN,
-		.hdr_total_len_or_pad_valid = true,
-		.hdr_little_endian = true,
-	},
-
-	.aggr = {
-		.aggr_en = IPA_BYPASS_AGGR,
-		.aggr = 0,
-		.aggr_byte_limit = 0,
-		.aggr_time_limit = 0,
-		.aggr_pkt_limit  = 0,
-	},
-	.deaggr = {
-		.deaggr_hdr_len = 0,
-		.packet_offset_valid = false,
-		.packet_offset_location = 0,
-		.max_packet_len = 0,
-	},
-
-	.route = {
-		.rt_tbl_hdl = RNDIS_IPA_DFLT_RT_HDL,
-	},
-	.nat = {
-		.nat_en = IPA_BYPASS_NAT,
-	},
-};
-
-static struct ipa_ep_cfg usb_to_ipa_ep_cfg_deaggr_en = {
-	.mode = {
-		.mode = IPA_BASIC,
-		.dst  = IPA_CLIENT_APPS_LAN_CONS,
-	},
-	.hdr = {
-		.hdr_len = ETH_HLEN,
-		.hdr_ofst_metadata_valid = false,
-		.hdr_ofst_metadata = 0,
-		.hdr_additional_const_len = 0,
-		.hdr_ofst_pkt_size_valid = true,
-		.hdr_ofst_pkt_size = 3 * sizeof(u32),
-		.hdr_a5_mux = false,
-		.hdr_remove_additional = false,
-		.hdr_metadata_reg_valid = true,
-	},
-	.hdr_ext = {
-		.hdr_pad_to_alignment = 0,
-		.hdr_total_len_or_pad_offset = 1 * sizeof(u32),
-		.hdr_payload_len_inc_padding = false,
-		.hdr_total_len_or_pad = IPA_HDR_TOTAL_LEN,
-		.hdr_total_len_or_pad_valid = true,
-		.hdr_little_endian = true,
-	},
-	.aggr = {
-		.aggr_en = IPA_ENABLE_DEAGGR,
-		.aggr = IPA_GENERIC,
-		.aggr_byte_limit = 0,
-		.aggr_time_limit = 0,
-		.aggr_pkt_limit  = 0,
-	},
-	.deaggr = {
-		.deaggr_hdr_len = sizeof(struct rndis_pkt_hdr),
-		.packet_offset_valid = true,
-		.packet_offset_location = 8,
-		.max_packet_len = 8192, /* Will be overridden*/
-	},
-	.route = {
-		.rt_tbl_hdl = RNDIS_IPA_DFLT_RT_HDL,
-	},
-	.nat = {
-		.nat_en = IPA_BYPASS_NAT,
-	},
-};
-
-/**
- * rndis_template_hdr - RNDIS template structure for RNDIS_IPA SW insertion
- * @msg_type: set for REMOTE_NDIS_PACKET_MSG (0x00000001)
- *  this value will be used for all data packets
- * @msg_len:  will add the skb length to get final size
- * @data_ofst: this field value will not be changed
- * @data_len: set as skb length to get final size
- * @zeroes: make sure all OOB data is not used
- */
-struct rndis_pkt_hdr rndis_template_hdr = {
-	.msg_type = RNDIS_IPA_PKT_TYPE,
-	.msg_len = sizeof(struct rndis_pkt_hdr),
-	.data_ofst = sizeof(struct rndis_pkt_hdr) - RNDIS_HDR_OFST(data_ofst),
-	.data_len = 0,
-	.zeroes = {0},
-};
-
-static void rndis_ipa_msg_free_cb(void *buff, u32 len, u32 type)
-{
-	kfree(buff);
-}
-
-/**
- * rndis_ipa_init() - create network device and initialize internal
- *  data structures
- * @params: in/out parameters required for initialization,
- *  see "struct ipa_usb_init_params" for more details
- *
- * Shall be called prior to pipe connection.
- * Detailed description:
- *  - allocate the network device
- *  - set default values for driver internal switches and stash them inside
- *     the netdev private field
- *  - set needed headroom for RNDIS header
- *  - create debugfs folder and files
- *  - create IPA resource manager client
- *  - set the ethernet address for the netdev to be added on SW Tx path
- *  - add header insertion rules for IPA driver (based on host/device Ethernet
- *     addresses given in input params and on RNDIS data template struct)
- *  - register tx/rx properties to IPA driver (will be later used
- *    by IPA configuration manager to configure rest of the IPA rules)
- *  - set the carrier state to "off" (until connect is called)
- *  - register the network device
- *  - set the out parameters
- *  - change driver internal state to INITIALIZED
- *
- * Returns negative errno, or zero on success
- */
-int rndis_ipa_init(struct ipa_usb_init_params *params)
-{
-	int result = 0;
-	struct net_device *net;
-	struct rndis_ipa_dev *rndis_ipa_ctx;
-	int ret;
-
-	RNDIS_IPA_LOG_ENTRY();
-	RNDIS_IPA_DEBUG("%s initializing\n", DRV_NAME);
-	ret = 0;
-	NULL_CHECK_RETVAL(params);
-	if (ret)
-		return ret;
-
-	RNDIS_IPA_DEBUG
-		("host_ethaddr=%pM, device_ethaddr=%pM\n",
-		params->host_ethaddr,
-		params->device_ethaddr);
-
-	net = alloc_etherdev(sizeof(struct rndis_ipa_dev));
-	if (!net) {
-		result = -ENOMEM;
-		RNDIS_IPA_ERROR("fail to allocate Ethernet device\n");
-		goto fail_alloc_etherdev;
-	}
-	RNDIS_IPA_DEBUG("network device was successfully allocated\n");
-
-	rndis_ipa_ctx = netdev_priv(net);
-	if (!rndis_ipa_ctx) {
-		result = -ENOMEM;
-		RNDIS_IPA_ERROR("fail to extract netdev priv\n");
-		goto fail_netdev_priv;
-	}
-	memset(rndis_ipa_ctx, 0, sizeof(*rndis_ipa_ctx));
-	RNDIS_IPA_DEBUG("rndis_ipa_ctx (private)=%pK\n", rndis_ipa_ctx);
-
-	spin_lock_init(&rndis_ipa_ctx->state_lock);
-
-	rndis_ipa_ctx->net = net;
-	rndis_ipa_ctx->tx_filter = false;
-	rndis_ipa_ctx->rx_filter = false;
-	rndis_ipa_ctx->icmp_filter = true;
-	rndis_ipa_ctx->tx_dropped = 0;
-	rndis_ipa_ctx->rx_dropped = 0;
-	rndis_ipa_ctx->tx_dump_enable = false;
-	rndis_ipa_ctx->rx_dump_enable = false;
-	rndis_ipa_ctx->deaggregation_enable = false;
-	rndis_ipa_ctx->outstanding_high = DEFAULT_OUTSTANDING_HIGH;
-	rndis_ipa_ctx->outstanding_low = DEFAULT_OUTSTANDING_LOW;
-	atomic_set(&rndis_ipa_ctx->outstanding_pkts, 0);
-	memcpy
-		(rndis_ipa_ctx->device_ethaddr, params->device_ethaddr,
-		sizeof(rndis_ipa_ctx->device_ethaddr));
-	memcpy
-		(rndis_ipa_ctx->host_ethaddr, params->host_ethaddr,
-		sizeof(rndis_ipa_ctx->host_ethaddr));
-	INIT_DELAYED_WORK
-		(&rndis_ipa_ctx->xmit_error_delayed_work,
-		rndis_ipa_xmit_error_aftercare_wq);
-	rndis_ipa_ctx->error_msec_sleep_time =
-		MIN_TX_ERROR_SLEEP_PERIOD;
-	RNDIS_IPA_DEBUG("internal data structures were set\n");
-
-	if (!params->device_ready_notify)
-		RNDIS_IPA_DEBUG("device_ready_notify() was not supplied\n");
-	rndis_ipa_ctx->device_ready_notify = params->device_ready_notify;
-
-	snprintf(net->name, sizeof(net->name), "%s%%d", NETDEV_NAME);
-	RNDIS_IPA_DEBUG
-		("Setting network interface driver name to: %s\n",
-		net->name);
-
-	net->netdev_ops = &rndis_ipa_netdev_ops;
-	net->watchdog_timeo = TX_TIMEOUT;
-
-	net->needed_headroom = sizeof(rndis_template_hdr);
-	RNDIS_IPA_DEBUG
-		("Needed headroom for RNDIS header set to %d\n",
-		net->needed_headroom);
-
-	rndis_ipa_debugfs_init(rndis_ipa_ctx);
-
-	result = rndis_ipa_set_device_ethernet_addr
-		(net->dev_addr, rndis_ipa_ctx->device_ethaddr);
-	if (result) {
-		RNDIS_IPA_ERROR("set device MAC failed\n");
-		goto fail_set_device_ethernet;
-	}
-	RNDIS_IPA_DEBUG("Device Ethernet address set %pM\n", net->dev_addr);
-
-	if (ipa_is_vlan_mode(IPA_VLAN_IF_RNDIS,
-		&rndis_ipa_ctx->is_vlan_mode)) {
-		RNDIS_IPA_ERROR("couldn't acquire vlan mode, is ipa ready?\n");
-		goto fail_get_vlan_mode;
-	}
-
-	RNDIS_IPA_DEBUG("is_vlan_mode %d\n", rndis_ipa_ctx->is_vlan_mode);
-
-	result = rndis_ipa_hdrs_cfg
-			(rndis_ipa_ctx,
-			params->host_ethaddr,
-			params->device_ethaddr);
-	if (result) {
-		RNDIS_IPA_ERROR("fail on ipa hdrs set\n");
-		goto fail_hdrs_cfg;
-	}
-	RNDIS_IPA_DEBUG("IPA header-insertion configed for Ethernet+RNDIS\n");
-
-	result = rndis_ipa_register_properties(net->name,
-		rndis_ipa_ctx->is_vlan_mode);
-	if (result) {
-		RNDIS_IPA_ERROR("fail on properties set\n");
-		goto fail_register_tx;
-	}
-	RNDIS_IPA_DEBUG("2 TX and 2 RX properties were registered\n");
-
-	netif_carrier_off(net);
-	RNDIS_IPA_DEBUG("set carrier off until pipes are connected\n");
-
-	result = register_netdev(net);
-	if (result) {
-		RNDIS_IPA_ERROR("register_netdev failed: %d\n", result);
-		goto fail_register_netdev;
-	}
-	RNDIS_IPA_DEBUG
-		("netdev:%s registration succeeded, index=%d\n",
-		net->name, net->ifindex);
-
-	rndis_ipa = rndis_ipa_ctx;
-	params->ipa_rx_notify = rndis_ipa_packet_receive_notify;
-	params->ipa_tx_notify = rndis_ipa_tx_complete_notify;
-	params->private = rndis_ipa_ctx;
-	params->skip_ep_cfg = false;
-	rndis_ipa_ctx->state = RNDIS_IPA_INITIALIZED;
-	RNDIS_IPA_STATE_DEBUG(rndis_ipa_ctx);
-	pr_info("RNDIS_IPA NetDev was initialized\n");
-
-	RNDIS_IPA_LOG_EXIT();
-
-	return 0;
-
-fail_register_netdev:
-	rndis_ipa_deregister_properties(net->name);
-fail_register_tx:
-	rndis_ipa_hdrs_destroy(rndis_ipa_ctx);
-fail_hdrs_cfg:
-fail_get_vlan_mode:
-fail_set_device_ethernet:
-	rndis_ipa_debugfs_destroy(rndis_ipa_ctx);
-fail_netdev_priv:
-	free_netdev(net);
-fail_alloc_etherdev:
-	return result;
-}
-EXPORT_SYMBOL(rndis_ipa_init);
-
-/**
- * rndis_ipa_pipe_connect_notify() - notify rndis_ipa Netdev that the USB pipes
- *  were connected
- * @usb_to_ipa_hdl: handle from IPA driver client for USB->IPA
- * @ipa_to_usb_hdl: handle from IPA driver client for IPA->USB
- * @private: same value that was set by init(), this parameter holds the
- *  network device pointer.
- * @max_transfer_byte_size: RNDIS protocol specific, the maximum size that
- *  the host expect
- * @max_packet_number: RNDIS protocol specific, the maximum packet number
- *  that the host expects
- *
- * Once USB driver finishes the pipe connection between IPA core
- * and USB core this method shall be called in order to
- * allow the driver to complete the data path configurations.
- * Detailed description:
- *  - configure the IPA end-points register
- *  - notify the Linux kernel for "carrier_on"
- *  - change the driver internal state
- *
- *  After this function is done the driver state changes to "Connected"  or
- *  Connected and Up.
- *  This API is expected to be called after initialization() or
- *  after a call to disconnect().
- *
- * Returns negative errno, or zero on success
- */
-int rndis_ipa_pipe_connect_notify(
-	u32 usb_to_ipa_hdl,
-	u32 ipa_to_usb_hdl,
-	u32 max_xfer_size_bytes_to_dev,
-	u32 max_packet_number_to_dev,
-	u32 max_xfer_size_bytes_to_host,
-	void *private)
-{
-	struct rndis_ipa_dev *rndis_ipa_ctx = private;
-	int next_state;
-	int result;
-	int ret;
-	unsigned long flags;
-	struct ipa_ecm_msg *rndis_msg;
-	struct ipa_msg_meta msg_meta;
-
-	RNDIS_IPA_LOG_ENTRY();
-
-	ret = 0;
-	NULL_CHECK_RETVAL(private);
-
-	if (ret)
-		return ret;
-
-	RNDIS_IPA_DEBUG
-		("usb_to_ipa_hdl=%d, ipa_to_usb_hdl=%d, private=0x%pK\n",
-		usb_to_ipa_hdl, ipa_to_usb_hdl, private);
-	RNDIS_IPA_DEBUG
-		("max_xfer_sz_to_dev=%d, max_pkt_num_to_dev=%d\n",
-		max_xfer_size_bytes_to_dev,
-		max_packet_number_to_dev);
-	RNDIS_IPA_DEBUG
-		("max_xfer_sz_to_host=%d\n",
-		max_xfer_size_bytes_to_host);
-
-	spin_lock_irqsave(&rndis_ipa_ctx->state_lock, flags);
-	next_state = rndis_ipa_next_state
-		(rndis_ipa_ctx->state,
-		RNDIS_IPA_CONNECT);
-	if (next_state == RNDIS_IPA_INVALID) {
-		spin_unlock_irqrestore(&rndis_ipa_ctx->state_lock, flags);
-		RNDIS_IPA_ERROR("use init()/disconnect() before connect()\n");
-		return -EPERM;
-	}
-	spin_unlock_irqrestore(&rndis_ipa_ctx->state_lock, flags);
-
-	if (usb_to_ipa_hdl >= IPA_CLIENT_MAX) {
-		RNDIS_IPA_ERROR
-			("usb_to_ipa_hdl(%d) - not valid ipa handle\n",
-			usb_to_ipa_hdl);
-		return -EINVAL;
-	}
-	if (ipa_to_usb_hdl >= IPA_CLIENT_MAX) {
-		RNDIS_IPA_ERROR
-			("ipa_to_usb_hdl(%d) - not valid ipa handle\n",
-			ipa_to_usb_hdl);
-		return -EINVAL;
-	}
-
-	result = rndis_ipa_register_pm_client(rndis_ipa_ctx);
-	if (result) {
-		RNDIS_IPA_ERROR("fail on PM register\n");
-		goto fail_register_pm;
-	}
-	RNDIS_IPA_DEBUG("PM client was registered\n");
-
-	rndis_ipa_ctx->ipa_to_usb_hdl = ipa_to_usb_hdl;
-	rndis_ipa_ctx->usb_to_ipa_hdl = usb_to_ipa_hdl;
-	if (max_packet_number_to_dev > 1)
-		rndis_ipa_ctx->deaggregation_enable = true;
-	else
-		rndis_ipa_ctx->deaggregation_enable = false;
-	result = rndis_ipa_ep_registers_cfg
-		(usb_to_ipa_hdl,
-		ipa_to_usb_hdl,
-		max_xfer_size_bytes_to_dev,
-		max_xfer_size_bytes_to_host,
-		rndis_ipa_ctx->net->mtu,
-		rndis_ipa_ctx->deaggregation_enable,
-		rndis_ipa_ctx->is_vlan_mode);
-	if (result) {
-		RNDIS_IPA_ERROR("fail on ep cfg\n");
-		goto fail;
-	}
-	RNDIS_IPA_DEBUG("end-points configured\n");
-
-	netif_stop_queue(rndis_ipa_ctx->net);
-	RNDIS_IPA_DEBUG("netif_stop_queue() was called\n");
-
-	netif_carrier_on(rndis_ipa_ctx->net);
-	if (!netif_carrier_ok(rndis_ipa_ctx->net)) {
-		RNDIS_IPA_ERROR("netif_carrier_ok error\n");
-		result = -EBUSY;
-		goto fail;
-	}
-	RNDIS_IPA_DEBUG("netif_carrier_on() was called\n");
-
-	rndis_msg = kzalloc(sizeof(*rndis_msg), GFP_KERNEL);
-	if (!rndis_msg) {
-		result = -ENOMEM;
-		goto fail;
-	}
-
-	memset(&msg_meta, 0, sizeof(struct ipa_msg_meta));
-	msg_meta.msg_type = ECM_CONNECT;
-	msg_meta.msg_len = sizeof(struct ipa_ecm_msg);
-	strlcpy(rndis_msg->name, rndis_ipa_ctx->net->name,
-		IPA_RESOURCE_NAME_MAX);
-	rndis_msg->ifindex = rndis_ipa_ctx->net->ifindex;
-
-	result = ipa_send_msg(&msg_meta, rndis_msg, rndis_ipa_msg_free_cb);
-	if (result) {
-		RNDIS_IPA_ERROR("fail to send ECM_CONNECT for rndis\n");
-		kfree(rndis_msg);
-		goto fail;
-	}
-
-	spin_lock_irqsave(&rndis_ipa_ctx->state_lock, flags);
-	next_state = rndis_ipa_next_state(rndis_ipa_ctx->state,
-					  RNDIS_IPA_CONNECT);
-	if (next_state == RNDIS_IPA_INVALID) {
-		spin_unlock_irqrestore(&rndis_ipa_ctx->state_lock, flags);
-		RNDIS_IPA_ERROR("use init()/disconnect() before connect()\n");
-		return -EPERM;
-	}
-	rndis_ipa_ctx->state = next_state;
-	spin_unlock_irqrestore(&rndis_ipa_ctx->state_lock, flags);
-
-	RNDIS_IPA_STATE_DEBUG(rndis_ipa_ctx);
-
-	if (next_state == RNDIS_IPA_CONNECTED_AND_UP)
-		rndis_ipa_enable_data_path(rndis_ipa_ctx);
-	else
-		RNDIS_IPA_DEBUG("queue shall be started after open()\n");
-
-	pr_info("RNDIS_IPA NetDev pipes were connected\n");
-
-	RNDIS_IPA_LOG_EXIT();
-
-	return 0;
-
-fail:
-	rndis_ipa_deregister_pm_client(rndis_ipa_ctx);
-fail_register_pm:
-	return result;
-}
-EXPORT_SYMBOL(rndis_ipa_pipe_connect_notify);
-
-/**
- * rndis_ipa_open() - notify Linux network stack to start sending packets
- * @net: the network interface supplied by the network stack
- *
- * Linux uses this API to notify the driver that the network interface
- * transitions to the up state.
- * The driver will instruct the Linux network stack to start
- * delivering data packets.
- * The driver internal state shall be changed to Up or Connected and Up
- *
- * Returns negative errno, or zero on success
- */
-static int rndis_ipa_open(struct net_device *net)
-{
-	struct rndis_ipa_dev *rndis_ipa_ctx;
-	int next_state;
-	unsigned long flags;
-
-	RNDIS_IPA_LOG_ENTRY();
-
-	rndis_ipa_ctx = netdev_priv(net);
-
-	spin_lock_irqsave(&rndis_ipa_ctx->state_lock, flags);
-
-	next_state = rndis_ipa_next_state(rndis_ipa_ctx->state, RNDIS_IPA_OPEN);
-	if (next_state == RNDIS_IPA_INVALID) {
-		spin_unlock_irqrestore(&rndis_ipa_ctx->state_lock, flags);
-		RNDIS_IPA_ERROR("can't bring driver up before initialize\n");
-		return -EPERM;
-	}
-
-	rndis_ipa_ctx->state = next_state;
-
-	spin_unlock_irqrestore(&rndis_ipa_ctx->state_lock, flags);
-
-	RNDIS_IPA_STATE_DEBUG(rndis_ipa_ctx);
-
-	if (next_state == RNDIS_IPA_CONNECTED_AND_UP)
-		rndis_ipa_enable_data_path(rndis_ipa_ctx);
-	else
-		RNDIS_IPA_DEBUG("queue shall be started after connect()\n");
-
-	pr_info("RNDIS_IPA NetDev was opened\n");
-
-	RNDIS_IPA_LOG_EXIT();
-
-	return 0;
-}
-
-/**
- * rndis_ipa_start_xmit() - send data from APPs to USB core via IPA core
- *  using SW path (Tx data path)
- * Tx path for this Netdev is Apps-processor->IPA->USB
- * @skb: packet received from Linux network stack destined for tethered PC
- * @net: the network device being used to send this packet (rndis0)
- *
- * Several conditions needed in order to send the packet to IPA:
- * - Transmit queue for the network driver is currently
- *   in "started" state
- * - The driver internal state is in Connected and Up state.
- * - Filters Tx switch are turned off
- * - The IPA resource manager state for the driver producer client
- *   is "Granted" which implies that all the resources in the dependency
- *   graph are valid for data flow.
- * - outstanding high boundary was not reached.
- *
- * In case the outstanding packets high boundary is reached, the driver will
- * stop the send queue until enough packets are processed by
- * the IPA core (based on calls to rndis_ipa_tx_complete_notify).
- *
- * In case all of the conditions are met, the network driver shall:
- *  - encapsulate the Ethernet packet with RNDIS header (REMOTE_NDIS_PACKET_MSG)
- *  - send the packet by using IPA Driver SW path (IP_PACKET_INIT)
- *  - Netdev status fields shall be updated based on the current Tx packet
- *
- * Returns NETDEV_TX_BUSY if retry should be made later,
- * or NETDEV_TX_OK on success.
- */
-static netdev_tx_t rndis_ipa_start_xmit(struct sk_buff *skb,
-					struct net_device *net)
-{
-	int ret;
-	netdev_tx_t status = NETDEV_TX_BUSY;
-	struct rndis_ipa_dev *rndis_ipa_ctx = netdev_priv(net);
-
-	netif_trans_update(net);
-
-	RNDIS_IPA_DEBUG_XMIT
-		("Tx, len=%d, skb->protocol=%d, outstanding=%d\n",
-		skb->len, skb->protocol,
-		atomic_read(&rndis_ipa_ctx->outstanding_pkts));
-
-	if (unlikely(netif_queue_stopped(net))) {
-		RNDIS_IPA_ERROR("interface queue is stopped\n");
-		goto out;
-	}
-
-	if (unlikely(rndis_ipa_ctx->tx_dump_enable))
-		rndis_ipa_dump_skb(skb);
-
-	if (unlikely(rndis_ipa_ctx->state != RNDIS_IPA_CONNECTED_AND_UP)) {
-		RNDIS_IPA_ERROR("Missing pipe connected and/or iface up\n");
-		return NETDEV_TX_BUSY;
-	}
-
-	if (unlikely(tx_filter(skb))) {
-		dev_kfree_skb_any(skb);
-		RNDIS_IPA_DEBUG("packet got filtered out on Tx path\n");
-		rndis_ipa_ctx->tx_dropped++;
-		status = NETDEV_TX_OK;
-		goto out;
-	}
-
-	ret = ipa_pm_activate(rndis_ipa_ctx->pm_hdl);
-	if (ret) {
-		RNDIS_IPA_DEBUG("Failed activate PM client\n");
-		netif_stop_queue(net);
-		goto fail_pm_activate;
-	}
-
-	if (atomic_read(&rndis_ipa_ctx->outstanding_pkts) >=
-				rndis_ipa_ctx->outstanding_high) {
-		RNDIS_IPA_DEBUG("Outstanding high boundary reached (%d)\n",
-				rndis_ipa_ctx->outstanding_high);
-		netif_stop_queue(net);
-		RNDIS_IPA_DEBUG("send  queue was stopped\n");
-		status = NETDEV_TX_BUSY;
-		goto out;
-	}
-
-	skb = rndis_encapsulate_skb(skb, rndis_ipa_ctx);
-	trace_rndis_tx_dp(skb->protocol);
-	ret = ipa_tx_dp(IPA_TO_USB_CLIENT, skb, NULL);
-	if (ret) {
-		RNDIS_IPA_ERROR("ipa transmit failed (%d)\n", ret);
-		goto fail_tx_packet;
-	}
-
-	atomic_inc(&rndis_ipa_ctx->outstanding_pkts);
-
-	status = NETDEV_TX_OK;
-	goto out;
-
-fail_tx_packet:
-	rndis_ipa_xmit_error(skb);
-out:
-	ipa_pm_deferred_deactivate(rndis_ipa_ctx->pm_hdl);
-fail_pm_activate:
-	RNDIS_IPA_DEBUG
-		("packet Tx done - %s\n",
-		(status == NETDEV_TX_OK) ? "OK" : "FAIL");
-
-	return status;
-}
-
-/**
- * rndis_ipa_tx_complete_notify() - notification for Netdev that the
- *  last packet was successfully sent
- * @private: driver context stashed by IPA driver upon pipe connect
- * @evt: event type (expected to be write-done event)
- * @data: data provided with event (this is actually the skb that
- *  holds the sent packet)
- *
- * This function will be called on interrupt bottom halve deferred context.
- * outstanding packets counter shall be decremented.
- * Network stack send queue will be re-started in case low outstanding
- * boundary is reached and queue was stopped before.
- * At the end the skb shall be freed.
- */
-static void rndis_ipa_tx_complete_notify(
-	void *private,
-	enum ipa_dp_evt_type evt,
-	unsigned long data)
-{
-	struct sk_buff *skb = (struct sk_buff *)data;
-	struct rndis_ipa_dev *rndis_ipa_ctx = private;
-	int ret;
-
-	ret = 0;
-	NULL_CHECK_RETVAL(private);
-	if (ret)
-		return;
-
-	trace_rndis_status_rcvd(skb->protocol);
-
-	RNDIS_IPA_DEBUG
-		("Tx-complete, len=%d, skb->prot=%d, outstanding=%d\n",
-		skb->len, skb->protocol,
-		atomic_read(&rndis_ipa_ctx->outstanding_pkts));
-
-	if (unlikely((evt != IPA_WRITE_DONE))) {
-		RNDIS_IPA_ERROR("unsupported event on TX call-back\n");
-		return;
-	}
-
-	if (unlikely(rndis_ipa_ctx->state != RNDIS_IPA_CONNECTED_AND_UP)) {
-		RNDIS_IPA_DEBUG
-		("dropping Tx-complete pkt, state=%s\n",
-		rndis_ipa_state_string(rndis_ipa_ctx->state));
-		goto out;
-	}
-
-	rndis_ipa_ctx->net->stats.tx_packets++;
-	rndis_ipa_ctx->net->stats.tx_bytes += skb->len;
-
-	if (atomic_read(&rndis_ipa_ctx->outstanding_pkts) > 0)
-		atomic_dec(&rndis_ipa_ctx->outstanding_pkts);
-
-	if
-		(netif_queue_stopped(rndis_ipa_ctx->net) &&
-		netif_carrier_ok(rndis_ipa_ctx->net) &&
-		atomic_read(&rndis_ipa_ctx->outstanding_pkts) <
-					(rndis_ipa_ctx->outstanding_low)) {
-		RNDIS_IPA_DEBUG("outstanding low boundary reached (%d)n",
-				rndis_ipa_ctx->outstanding_low);
-		netif_wake_queue(rndis_ipa_ctx->net);
-		RNDIS_IPA_DEBUG("send queue was awaken\n");
-	}
-
-out:
-	dev_kfree_skb_any(skb);
-}
-
-static void rndis_ipa_tx_timeout(struct net_device *net)
-{
-	struct rndis_ipa_dev *rndis_ipa_ctx = netdev_priv(net);
-	int outstanding = atomic_read(&rndis_ipa_ctx->outstanding_pkts);
-
-	RNDIS_IPA_ERROR
-		("possible IPA stall was detected, %d outstanding\n",
-		outstanding);
-
-	net->stats.tx_errors++;
-}
-
-/**
- * rndis_ipa_packet_receive_notify() - Rx notify for packet sent from
- *  tethered PC (USB->IPA).
- *  is USB->IPA->Apps-processor
- * @private: driver context
- * @evt: event type
- * @data: data provided with event
- *
- * Once IPA driver receives a packet from USB client this callback will be
- * called from bottom-half interrupt handling context (ipa Rx workqueue).
- *
- * Packets that shall be sent to Apps processor may be of two types:
- * 1) Packets that are destined for Apps (e.g: WEBSERVER running on Apps)
- * 2) Exception packets that need special handling (based on IPA core
- *    configuration, e.g: new TCP session or any other packets that IPA core
- *    can't handle)
- * If the next conditions are met, the packet shall be sent up to the
- * Linux network stack:
- *  - Driver internal state is Connected and Up
- *  - Notification received from IPA driver meets the expected type
- *    for Rx packet
- *  -Filters Rx switch are turned off
- *
- * Prior to the sending to the network stack:
- *  - Netdev struct shall be stashed to the skb as required by the network stack
- *  - Ethernet header shall be removed (skb->data shall point to the Ethernet
- *     payload, Ethernet still stashed under MAC header).
- *  - The skb->pkt_protocol shall be set based on the ethernet destination
- *     address, Can be Broadcast, Multicast or Other-Host, The later
- *     pkt-types packets shall be dropped in case the Netdev is not
- *     in  promisc mode.
- *   - Set the skb protocol field based on the EtherType field
- *
- * Netdev status fields shall be updated based on the current Rx packet
- */
-static void rndis_ipa_packet_receive_notify(
-		void *private,
-		enum ipa_dp_evt_type evt,
-		unsigned long data)
-{
-	struct sk_buff *skb = (struct sk_buff *)data;
-	struct rndis_ipa_dev *rndis_ipa_ctx = private;
-	int result;
-	unsigned int packet_len = skb->len;
-
-	RNDIS_IPA_DEBUG
-		("packet Rx, len=%d\n",
-		skb->len);
-
-	if (unlikely(rndis_ipa_ctx->rx_dump_enable))
-		rndis_ipa_dump_skb(skb);
-
-	if (unlikely(rndis_ipa_ctx->state != RNDIS_IPA_CONNECTED_AND_UP)) {
-		RNDIS_IPA_DEBUG("use connect()/up() before receive()\n");
-		RNDIS_IPA_DEBUG("packet dropped (length=%d)\n",
-				skb->len);
-		return;
-	}
-
-	if (evt != IPA_RECEIVE)	{
-		RNDIS_IPA_ERROR("a none IPA_RECEIVE event in driver RX\n");
-		return;
-	}
-
-	if (!rndis_ipa_ctx->deaggregation_enable)
-		skb_pull(skb, sizeof(struct rndis_pkt_hdr));
-
-	skb->dev = rndis_ipa_ctx->net;
-	skb->protocol = eth_type_trans(skb, rndis_ipa_ctx->net);
-
-	if (rx_filter(skb)) {
-		RNDIS_IPA_DEBUG("packet got filtered out on RX path\n");
-		rndis_ipa_ctx->rx_dropped++;
-		dev_kfree_skb_any(skb);
-		return;
-	}
-
-	trace_rndis_netif_ni(skb->protocol);
-	result = netif_rx_ni(skb);
-	if (result)
-		RNDIS_IPA_ERROR("fail on netif_rx_ni\n");
-	rndis_ipa_ctx->net->stats.rx_packets++;
-	rndis_ipa_ctx->net->stats.rx_bytes += packet_len;
-}
-
-/** rndis_ipa_stop() - notify the network interface to stop
- *   sending/receiving data
- *  @net: the network device being stopped.
- *
- * This API is used by Linux network stack to notify the network driver that
- * its state was changed to "down"
- * The driver will stop the "send" queue and change its internal
- * state to "Connected".
- * The Netdev shall be returned to be "Up" after rndis_ipa_open().
- */
-static int rndis_ipa_stop(struct net_device *net)
-{
-	struct rndis_ipa_dev *rndis_ipa_ctx = netdev_priv(net);
-	int next_state;
-	unsigned long flags;
-
-	RNDIS_IPA_LOG_ENTRY();
-
-	spin_lock_irqsave(&rndis_ipa_ctx->state_lock, flags);
-
-	next_state = rndis_ipa_next_state(rndis_ipa_ctx->state, RNDIS_IPA_STOP);
-	if (next_state == RNDIS_IPA_INVALID) {
-		spin_unlock_irqrestore(&rndis_ipa_ctx->state_lock, flags);
-		RNDIS_IPA_DEBUG("can't do network interface down without up\n");
-		return -EPERM;
-	}
-
-	rndis_ipa_ctx->state = next_state;
-
-	spin_unlock_irqrestore(&rndis_ipa_ctx->state_lock, flags);
-
-	netif_stop_queue(net);
-	pr_info("RNDIS_IPA NetDev queue is stopped\n");
-
-	RNDIS_IPA_STATE_DEBUG(rndis_ipa_ctx);
-
-	RNDIS_IPA_LOG_EXIT();
-
-	return 0;
-}
-
-/** rndis_ipa_disconnect() - notify rndis_ipa Netdev that the USB pipes
- *   were disconnected
- * @private: same value that was set by init(), this  parameter holds the
- *  network device pointer.
- *
- * USB shall notify the Netdev after disconnecting the pipe.
- * - The internal driver state shall returned to its previous
- *   state (Up or Initialized).
- * - Linux network stack shall be informed for carrier off to notify
- *   user space for pipe disconnect
- * - send queue shall be stopped
- * During the transition between the pipe disconnection to
- * the Netdev notification packets
- * are expected to be dropped by IPA driver or IPA core.
- */
-int rndis_ipa_pipe_disconnect_notify(void *private)
-{
-	struct rndis_ipa_dev *rndis_ipa_ctx = private;
-	int next_state;
-	int outstanding_dropped_pkts;
-	int retval;
-	int ret;
-	unsigned long flags;
-	struct ipa_ecm_msg *rndis_msg;
-	struct ipa_msg_meta msg_meta;
-
-	RNDIS_IPA_LOG_ENTRY();
-
-	ret = 0;
-	NULL_CHECK_RETVAL(rndis_ipa_ctx);
-	if (ret)
-		return ret;
-	RNDIS_IPA_DEBUG("private=0x%pK\n", private);
-
-	spin_lock_irqsave(&rndis_ipa_ctx->state_lock, flags);
-
-	next_state = rndis_ipa_next_state
-		(rndis_ipa_ctx->state,
-		RNDIS_IPA_DISCONNECT);
-	if (next_state == RNDIS_IPA_INVALID) {
-		spin_unlock_irqrestore(&rndis_ipa_ctx->state_lock, flags);
-		RNDIS_IPA_ERROR("can't disconnect before connect\n");
-		return -EPERM;
-	}
-	spin_unlock_irqrestore(&rndis_ipa_ctx->state_lock, flags);
-
-	if (rndis_ipa_ctx->during_xmit_error) {
-		RNDIS_IPA_DEBUG("canceling xmit-error delayed work\n");
-		cancel_delayed_work_sync(
-			&rndis_ipa_ctx->xmit_error_delayed_work);
-		rndis_ipa_ctx->during_xmit_error = false;
-	}
-
-	netif_carrier_off(rndis_ipa_ctx->net);
-	RNDIS_IPA_DEBUG("carrier_off notification was sent\n");
-
-	rndis_msg = kzalloc(sizeof(*rndis_msg), GFP_KERNEL);
-	if (!rndis_msg)
-		return -ENOMEM;
-
-	memset(&msg_meta, 0, sizeof(struct ipa_msg_meta));
-	msg_meta.msg_type = ECM_DISCONNECT;
-	msg_meta.msg_len = sizeof(struct ipa_ecm_msg);
-	strlcpy(rndis_msg->name, rndis_ipa_ctx->net->name,
-		IPA_RESOURCE_NAME_MAX);
-	rndis_msg->ifindex = rndis_ipa_ctx->net->ifindex;
-
-	retval = ipa_send_msg(&msg_meta, rndis_msg, rndis_ipa_msg_free_cb);
-	if (retval) {
-		RNDIS_IPA_ERROR("fail to send ECM_DISCONNECT for rndis\n");
-		kfree(rndis_msg);
-		return -EPERM;
-	}
-
-	netif_stop_queue(rndis_ipa_ctx->net);
-	RNDIS_IPA_DEBUG("queue stopped\n");
-
-	outstanding_dropped_pkts =
-		atomic_read(&rndis_ipa_ctx->outstanding_pkts);
-
-	rndis_ipa_ctx->net->stats.tx_dropped += outstanding_dropped_pkts;
-	atomic_set(&rndis_ipa_ctx->outstanding_pkts, 0);
-
-	retval = rndis_ipa_deregister_pm_client(rndis_ipa_ctx);
-	if (retval) {
-		RNDIS_IPA_ERROR("Fail to deregister PM\n");
-		return retval;
-	}
-	RNDIS_IPA_DEBUG("PM was successfully deregistered\n");
-
-	spin_lock_irqsave(&rndis_ipa_ctx->state_lock, flags);
-	next_state = rndis_ipa_next_state(rndis_ipa_ctx->state,
-					  RNDIS_IPA_DISCONNECT);
-	if (next_state == RNDIS_IPA_INVALID) {
-		spin_unlock_irqrestore(&rndis_ipa_ctx->state_lock, flags);
-		RNDIS_IPA_ERROR("can't disconnect before connect\n");
-		return -EPERM;
-	}
-	rndis_ipa_ctx->state = next_state;
-	spin_unlock_irqrestore(&rndis_ipa_ctx->state_lock, flags);
-
-	RNDIS_IPA_STATE_DEBUG(rndis_ipa_ctx);
-
-	pr_info("RNDIS_IPA NetDev pipes disconnected (%d outstanding clr)\n",
-		outstanding_dropped_pkts);
-
-	RNDIS_IPA_LOG_EXIT();
-
-	return 0;
-}
-EXPORT_SYMBOL(rndis_ipa_pipe_disconnect_notify);
-
-/**
- * rndis_ipa_cleanup() - unregister the network interface driver and free
- *  internal data structs.
- * @private: same value that was set by init(), this
- *   parameter holds the network device pointer.
- *
- * This function shall be called once the network interface is not
- * needed anymore, e.g: when the USB composition does not support it.
- * This function shall be called after the pipes were disconnected.
- * Detailed description:
- *  - remove header-insertion headers from IPA core
- *  - delete the driver dependency defined for IPA resource manager and
- *   destroy the producer resource.
- *  -  remove the debugfs entries
- *  - deregister the network interface from Linux network stack
- *  - free all internal data structs
- *
- * It is assumed that no packets shall be sent through HW bridging
- * during cleanup to avoid packets trying to add an header that is
- * removed during cleanup (IPA configuration manager should have
- * removed them at this point)
- */
-void rndis_ipa_cleanup(void *private)
-{
-	struct rndis_ipa_dev *rndis_ipa_ctx = private;
-	int next_state;
-	int ret;
-	unsigned long flags;
-
-	RNDIS_IPA_LOG_ENTRY();
-
-	RNDIS_IPA_DEBUG("private=0x%pK\n", private);
-
-	ret = 0;
-	NULL_CHECK_RETVAL(rndis_ipa_ctx);
-	if (ret)
-		return;
-
-	spin_lock_irqsave(&rndis_ipa_ctx->state_lock, flags);
-	next_state = rndis_ipa_next_state
-		(rndis_ipa_ctx->state,
-		RNDIS_IPA_CLEANUP);
-	if (next_state == RNDIS_IPA_INVALID) {
-		spin_unlock_irqrestore(&rndis_ipa_ctx->state_lock, flags);
-		RNDIS_IPA_ERROR("use disconnect()before clean()\n");
-		return;
-	}
-	spin_unlock_irqrestore(&rndis_ipa_ctx->state_lock, flags);
-
-	RNDIS_IPA_STATE_DEBUG(rndis_ipa_ctx);
-
-	ret = rndis_ipa_deregister_properties(rndis_ipa_ctx->net->name);
-	if (ret) {
-		RNDIS_IPA_ERROR("Fail to deregister Tx/Rx properties\n");
-		return;
-	}
-	RNDIS_IPA_DEBUG("deregister Tx/Rx properties was successful\n");
-
-	ret = rndis_ipa_hdrs_destroy(rndis_ipa_ctx);
-	if (ret)
-		RNDIS_IPA_ERROR(
-			"Failed removing RNDIS headers from IPA core. Continue anyway\n");
-	else
-		RNDIS_IPA_DEBUG("RNDIS headers were removed from IPA core\n");
-
-	rndis_ipa_debugfs_destroy(rndis_ipa_ctx);
-	RNDIS_IPA_DEBUG("debugfs remove was done\n");
-
-	unregister_netdev(rndis_ipa_ctx->net);
-	RNDIS_IPA_DEBUG("netdev unregistered\n");
-
-	spin_lock_irqsave(&rndis_ipa_ctx->state_lock, flags);
-	next_state = rndis_ipa_next_state(rndis_ipa_ctx->state,
-					  RNDIS_IPA_CLEANUP);
-	if (next_state == RNDIS_IPA_INVALID) {
-		spin_unlock_irqrestore(&rndis_ipa_ctx->state_lock, flags);
-		RNDIS_IPA_ERROR("use disconnect()before clean()\n");
-		return;
-	}
-	rndis_ipa_ctx->state = next_state;
-	spin_unlock_irqrestore(&rndis_ipa_ctx->state_lock, flags);
-	free_netdev(rndis_ipa_ctx->net);
-	pr_info("RNDIS_IPA NetDev was cleaned\n");
-
-	RNDIS_IPA_LOG_EXIT();
-}
-EXPORT_SYMBOL(rndis_ipa_cleanup);
-
-static void rndis_ipa_enable_data_path(struct rndis_ipa_dev *rndis_ipa_ctx)
-{
-	if (rndis_ipa_ctx->device_ready_notify) {
-		rndis_ipa_ctx->device_ready_notify();
-		RNDIS_IPA_DEBUG("USB device_ready_notify() was called\n");
-	} else {
-		RNDIS_IPA_DEBUG("device_ready_notify() not supplied\n");
-	}
-
-	netif_start_queue(rndis_ipa_ctx->net);
-	RNDIS_IPA_DEBUG("netif_start_queue() was called\n");
-}
-
-static void rndis_ipa_xmit_error(struct sk_buff *skb)
-{
-	bool retval;
-	struct rndis_ipa_dev *rndis_ipa_ctx = netdev_priv(skb->dev);
-	unsigned long delay_jiffies;
-	u8 rand_dealy_msec;
-
-	RNDIS_IPA_LOG_ENTRY();
-
-	RNDIS_IPA_DEBUG("starting Tx-queue backoff\n");
-
-	netif_stop_queue(rndis_ipa_ctx->net);
-	RNDIS_IPA_DEBUG("netif_stop_queue was called\n");
-
-	skb_pull(skb, sizeof(rndis_template_hdr));
-	rndis_ipa_ctx->net->stats.tx_errors++;
-
-	get_random_bytes(&rand_dealy_msec, sizeof(rand_dealy_msec));
-	delay_jiffies = msecs_to_jiffies(
-		rndis_ipa_ctx->error_msec_sleep_time + rand_dealy_msec);
-
-	retval = schedule_delayed_work(
-		&rndis_ipa_ctx->xmit_error_delayed_work, delay_jiffies);
-	if (!retval) {
-		RNDIS_IPA_ERROR("fail to schedule delayed work\n");
-		netif_start_queue(rndis_ipa_ctx->net);
-	} else {
-		RNDIS_IPA_DEBUG
-			("work scheduled to start Tx-queue in %d msec\n",
-			rndis_ipa_ctx->error_msec_sleep_time +
-			rand_dealy_msec);
-		rndis_ipa_ctx->during_xmit_error = true;
-	}
-
-	RNDIS_IPA_LOG_EXIT();
-}
-
-static void rndis_ipa_xmit_error_aftercare_wq(struct work_struct *work)
-{
-	struct rndis_ipa_dev *rndis_ipa_ctx;
-	struct delayed_work *delayed_work;
-
-	RNDIS_IPA_LOG_ENTRY();
-
-	RNDIS_IPA_DEBUG("Starting queue after xmit error\n");
-
-	delayed_work = to_delayed_work(work);
-	rndis_ipa_ctx = container_of
-		(delayed_work, struct rndis_ipa_dev,
-		xmit_error_delayed_work);
-
-	if (unlikely(rndis_ipa_ctx->state != RNDIS_IPA_CONNECTED_AND_UP)) {
-		RNDIS_IPA_ERROR
-			("error aftercare handling in bad state (%d)",
-			rndis_ipa_ctx->state);
-		return;
-	}
-
-	rndis_ipa_ctx->during_xmit_error = false;
-
-	netif_start_queue(rndis_ipa_ctx->net);
-	RNDIS_IPA_DEBUG("netif_start_queue() was called\n");
-
-	RNDIS_IPA_LOG_EXIT();
-}
-
-/**
- * rndis_ipa_prepare_header_insertion() - prepare the header insertion request
- *  for IPA driver
- * eth_type: the Ethernet type for this header-insertion header
- * hdr_name: string that shall represent this header in IPA data base
- * add_hdr: output for caller to be used with ipa_add_hdr() to configure
- *  the IPA core
- * dst_mac: tethered PC MAC (Ethernet) address to be added to packets
- *  for IPA->USB pipe
- * src_mac: device MAC (Ethernet) address to be added to packets
- *  for IPA->USB pipe
- * is_vlan_mode: should driver work in vlan mode?
- *
- * This function shall build the header-insertion block request for a
- * single Ethernet+RNDIS header)
- * this header shall be inserted for packets processed by IPA
- * and destined for USB client.
- * This header shall be used for HW bridging for packets destined for
- *  tethered PC.
- * For SW data-path, this header won't be used.
- */
-static void rndis_ipa_prepare_header_insertion(
-	int eth_type,
-	const char *hdr_name, struct ipa_hdr_add *add_hdr,
-	const void *dst_mac, const void *src_mac, bool is_vlan_mode)
-{
-	struct ethhdr *eth_hdr;
-	struct vlan_ethhdr *eth_vlan_hdr;
-
-	add_hdr->hdr_len = sizeof(rndis_template_hdr);
-	add_hdr->is_partial = false;
-	strlcpy(add_hdr->name, hdr_name, IPA_RESOURCE_NAME_MAX);
-
-	memcpy(add_hdr->hdr, &rndis_template_hdr, sizeof(rndis_template_hdr));
-	add_hdr->is_eth2_ofst_valid = true;
-	add_hdr->eth2_ofst = sizeof(rndis_template_hdr);
-
-	if (is_vlan_mode) {
-		eth_vlan_hdr = (struct vlan_ethhdr *)(add_hdr->hdr +
-			sizeof(rndis_template_hdr));
-		memcpy(eth_vlan_hdr->h_dest, dst_mac, ETH_ALEN);
-		memcpy(eth_vlan_hdr->h_source, src_mac, ETH_ALEN);
-		eth_vlan_hdr->h_vlan_encapsulated_proto = htons(eth_type);
-		eth_vlan_hdr->h_vlan_proto = htons(ETH_P_8021Q);
-		add_hdr->hdr_len += VLAN_ETH_HLEN;
-		add_hdr->type = IPA_HDR_L2_802_1Q;
-	} else {
-		eth_hdr = (struct ethhdr *)(add_hdr->hdr +
-			sizeof(rndis_template_hdr));
-		memcpy(eth_hdr->h_dest, dst_mac, ETH_ALEN);
-		memcpy(eth_hdr->h_source, src_mac, ETH_ALEN);
-		eth_hdr->h_proto = htons(eth_type);
-		add_hdr->hdr_len += ETH_HLEN;
-		add_hdr->type = IPA_HDR_L2_ETHERNET_II;
-	}
-}
-
-/**
- * rndis_ipa_hdrs_cfg() - configure header insertion block in IPA core
- *  to allow HW bridging
- * @rndis_ipa_ctx: main driver context
- * @dst_mac: destination MAC address (tethered PC)
- * @src_mac: source MAC address (MDM device)
- *
- * This function shall add 2 headers.
- * One header for Ipv4 and one header for Ipv6.
- * Both headers shall contain Ethernet header and RNDIS header, the only
- * difference shall be in the EtherTye field.
- * Headers will be committed to HW
- *
- * Returns negative errno, or zero on success
- */
-static int rndis_ipa_hdrs_cfg(
-	struct rndis_ipa_dev *rndis_ipa_ctx,
-	const void *dst_mac, const void *src_mac)
-{
-	struct ipa_ioc_add_hdr *hdrs;
-	struct ipa_hdr_add *ipv4_hdr;
-	struct ipa_hdr_add *ipv6_hdr;
-	int result = 0;
-
-	RNDIS_IPA_LOG_ENTRY();
-
-	hdrs = kzalloc
-		(sizeof(*hdrs) + sizeof(*ipv4_hdr) + sizeof(*ipv6_hdr),
-		GFP_KERNEL);
-	if (!hdrs) {
-		result = -ENOMEM;
-		goto fail_mem;
-	}
-
-	ipv4_hdr = &hdrs->hdr[0];
-	ipv6_hdr = &hdrs->hdr[1];
-	rndis_ipa_prepare_header_insertion
-		(ETH_P_IP, IPV4_HDR_NAME,
-		ipv4_hdr, dst_mac, src_mac, rndis_ipa_ctx->is_vlan_mode);
-	rndis_ipa_prepare_header_insertion
-		(ETH_P_IPV6, IPV6_HDR_NAME,
-		ipv6_hdr, dst_mac, src_mac, rndis_ipa_ctx->is_vlan_mode);
-
-	hdrs->commit = 1;
-	hdrs->num_hdrs = 2;
-	result = ipa_add_hdr(hdrs);
-	if (result) {
-		RNDIS_IPA_ERROR("Fail on Header-Insertion(%d)\n", result);
-		goto fail_add_hdr;
-	}
-	if (ipv4_hdr->status) {
-		RNDIS_IPA_ERROR("Fail on Header-Insertion ipv4(%d)\n",
-				ipv4_hdr->status);
-		result = ipv4_hdr->status;
-		goto fail_add_hdr;
-	}
-	if (ipv6_hdr->status) {
-		RNDIS_IPA_ERROR("Fail on Header-Insertion ipv6(%d)\n",
-				ipv6_hdr->status);
-		result = ipv6_hdr->status;
-		goto fail_add_hdr;
-	}
-	rndis_ipa_ctx->eth_ipv4_hdr_hdl = ipv4_hdr->hdr_hdl;
-	rndis_ipa_ctx->eth_ipv6_hdr_hdl = ipv6_hdr->hdr_hdl;
-
-	RNDIS_IPA_LOG_EXIT();
-
-fail_add_hdr:
-	kfree(hdrs);
-fail_mem:
-	return result;
-}
-
-/**
- * rndis_ipa_hdrs_destroy() - remove the IPA core configuration done for
- *  the driver data path bridging.
- * @rndis_ipa_ctx: the driver context
- *
- *  Revert the work done on rndis_ipa_hdrs_cfg(), which is,
- * remove 2 headers for Ethernet+RNDIS.
- */
-static int rndis_ipa_hdrs_destroy(struct rndis_ipa_dev *rndis_ipa_ctx)
-{
-	struct ipa_ioc_del_hdr *del_hdr;
-	struct ipa_hdr_del *ipv4;
-	struct ipa_hdr_del *ipv6;
-	int result;
-
-	del_hdr = kzalloc(sizeof(*del_hdr) + sizeof(*ipv4) +
-			sizeof(*ipv6), GFP_KERNEL);
-	if (!del_hdr)
-		return -ENOMEM;
-
-	del_hdr->commit = 1;
-	del_hdr->num_hdls = 2;
-
-	ipv4 = &del_hdr->hdl[0];
-	ipv4->hdl = rndis_ipa_ctx->eth_ipv4_hdr_hdl;
-	ipv6 = &del_hdr->hdl[1];
-	ipv6->hdl = rndis_ipa_ctx->eth_ipv6_hdr_hdl;
-
-	result = ipa_del_hdr(del_hdr);
-	if (result || ipv4->status || ipv6->status)
-		RNDIS_IPA_ERROR("ipa_del_hdr failed\n");
-	else
-		RNDIS_IPA_DEBUG("hdrs deletion done\n");
-
-	kfree(del_hdr);
-	return result;
-}
-
-static struct net_device_stats *rndis_ipa_get_stats(struct net_device *net)
-{
-	return &net->stats;
-}
-
-/**
- * rndis_ipa_register_properties() - set Tx/Rx properties needed
- *  by IPA configuration manager
- * @netdev_name: a string with the name of the network interface device
- * @is_vlan_mode: should driver work in vlan mode?
- *
- * Register Tx/Rx properties to allow user space configuration (IPA
- * Configuration Manager):
- *
- * - Two Tx properties (IPA->USB): specify the header names and pipe number
- *   that shall be used by user space for header-addition configuration
- *   for ipv4/ipv6 packets flowing from IPA to USB for HW bridging data.
- *   That header-addition header is added by the Netdev and used by user
- *   space to close the the HW bridge by adding filtering and routing rules
- *   that point to this header.
- *
- * - Two Rx properties (USB->IPA): these properties shall be used by user space
- *   to configure the IPA core to identify the packets destined
- *   for Apps-processor by configuring the unicast rules destined for
- *   the Netdev IP address.
- *   This rules shall be added based on the attribute mask supplied at
- *   this function, that is, always hit rule.
- */
-static int rndis_ipa_register_properties(char *netdev_name, bool is_vlan_mode)
-{
-	struct ipa_tx_intf tx_properties = {0};
-	struct ipa_ioc_tx_intf_prop properties[2] = { {0}, {0} };
-	struct ipa_ioc_tx_intf_prop *ipv4_property;
-	struct ipa_ioc_tx_intf_prop *ipv6_property;
-	struct ipa_ioc_rx_intf_prop rx_ioc_properties[2] = { {0}, {0} };
-	struct ipa_rx_intf rx_properties = {0};
-	struct ipa_ioc_rx_intf_prop *rx_ipv4_property;
-	struct ipa_ioc_rx_intf_prop *rx_ipv6_property;
-	enum ipa_hdr_l2_type hdr_l2_type = IPA_HDR_L2_ETHERNET_II;
-	int result = 0;
-
-	RNDIS_IPA_LOG_ENTRY();
-
-	if (is_vlan_mode)
-		hdr_l2_type = IPA_HDR_L2_802_1Q;
-
-	tx_properties.prop = properties;
-	ipv4_property = &tx_properties.prop[0];
-	ipv4_property->ip = IPA_IP_v4;
-	ipv4_property->dst_pipe = IPA_TO_USB_CLIENT;
-	strlcpy
-		(ipv4_property->hdr_name, IPV4_HDR_NAME,
-		IPA_RESOURCE_NAME_MAX);
-	ipv4_property->hdr_l2_type = hdr_l2_type;
-	ipv6_property = &tx_properties.prop[1];
-	ipv6_property->ip = IPA_IP_v6;
-	ipv6_property->dst_pipe = IPA_TO_USB_CLIENT;
-	strlcpy
-		(ipv6_property->hdr_name, IPV6_HDR_NAME,
-		IPA_RESOURCE_NAME_MAX);
-	ipv6_property->hdr_l2_type = hdr_l2_type;
-	tx_properties.num_props = 2;
-
-	rx_properties.prop = rx_ioc_properties;
-	rx_ipv4_property = &rx_properties.prop[0];
-	rx_ipv4_property->ip = IPA_IP_v4;
-	rx_ipv4_property->attrib.attrib_mask = 0;
-	rx_ipv4_property->src_pipe = IPA_CLIENT_USB_PROD;
-	rx_ipv4_property->hdr_l2_type = hdr_l2_type;
-	rx_ipv6_property = &rx_properties.prop[1];
-	rx_ipv6_property->ip = IPA_IP_v6;
-	rx_ipv6_property->attrib.attrib_mask = 0;
-	rx_ipv6_property->src_pipe = IPA_CLIENT_USB_PROD;
-	rx_ipv6_property->hdr_l2_type = hdr_l2_type;
-	rx_properties.num_props = 2;
-
-	result = ipa_register_intf("rndis0", &tx_properties, &rx_properties);
-	if (result)
-		RNDIS_IPA_ERROR("fail on Tx/Rx properties registration\n");
-	else
-		RNDIS_IPA_DEBUG("Tx/Rx properties registration done\n");
-
-	RNDIS_IPA_LOG_EXIT();
-
-	return result;
-}
-
-/**
- * rndis_ipa_deregister_properties() - remove the 2 Tx and 2 Rx properties
- * @netdev_name: a string with the name of the network interface device
- *
- * This function revert the work done on rndis_ipa_register_properties().
- */
-static int  rndis_ipa_deregister_properties(char *netdev_name)
-{
-	int result;
-
-	RNDIS_IPA_LOG_ENTRY();
-
-	result = ipa_deregister_intf(netdev_name);
-	if (result) {
-		RNDIS_IPA_DEBUG("Fail on Tx prop deregister\n");
-		return result;
-	}
-	RNDIS_IPA_LOG_EXIT();
-
-	return 0;
-}
-
-
-
-static void rndis_ipa_pm_cb(void *p, enum ipa_pm_cb_event event)
-{
-	struct rndis_ipa_dev *rndis_ipa_ctx = p;
-
-	RNDIS_IPA_LOG_ENTRY();
-
-	if (event != IPA_PM_CLIENT_ACTIVATED) {
-		RNDIS_IPA_ERROR("unexpected event %d\n", event);
-		WARN_ON(1);
-		return;
-	}
-	RNDIS_IPA_DEBUG("Resource Granted\n");
-
-	if (netif_queue_stopped(rndis_ipa_ctx->net)) {
-		RNDIS_IPA_DEBUG("starting queue\n");
-		netif_start_queue(rndis_ipa_ctx->net);
-	} else {
-		RNDIS_IPA_DEBUG("queue already awake\n");
-	}
-
-	RNDIS_IPA_LOG_EXIT();
-}
-
-
-static int rndis_ipa_register_pm_client(struct rndis_ipa_dev *rndis_ipa_ctx)
-{
-	int result;
-	struct ipa_pm_register_params pm_reg;
-
-	memset(&pm_reg, 0, sizeof(pm_reg));
-
-	pm_reg.name = rndis_ipa_ctx->net->name;
-	pm_reg.user_data = rndis_ipa_ctx;
-	pm_reg.callback = rndis_ipa_pm_cb;
-	pm_reg.group = IPA_PM_GROUP_APPS;
-	result = ipa_pm_register(&pm_reg, &rndis_ipa_ctx->pm_hdl);
-	if (result) {
-		RNDIS_IPA_ERROR("failed to create IPA PM client %d\n", result);
-		return result;
-	}
-	return 0;
-}
-
-static int rndis_ipa_deregister_pm_client(struct rndis_ipa_dev *rndis_ipa_ctx)
-{
-	ipa_pm_deactivate_sync(rndis_ipa_ctx->pm_hdl);
-	ipa_pm_deregister(rndis_ipa_ctx->pm_hdl);
-	rndis_ipa_ctx->pm_hdl = ~0;
-	return 0;
-}
-
-
-/**
- * rndis_encapsulate_skb() - encapsulate the given Ethernet skb with
- *  an RNDIS header
- * @skb: packet to be encapsulated with the RNDIS header
- * @rndis_ipa_ctx: main driver context
- *
- * Shall use a template header for RNDIS and update it with the given
- * skb values.
- * Ethernet is expected to be already encapsulate the packet.
- */
-static struct sk_buff *rndis_encapsulate_skb(struct sk_buff *skb,
-	struct rndis_ipa_dev *rndis_ipa_ctx)
-{
-	struct rndis_pkt_hdr *rndis_hdr;
-	int payload_byte_len = skb->len;
-
-	/* if there is no room in this skb, allocate a new one */
-	if (unlikely(skb_headroom(skb) < sizeof(rndis_template_hdr))) {
-		struct sk_buff *new_skb = skb_copy_expand(skb,
-			sizeof(rndis_template_hdr), 0, GFP_ATOMIC);
-		if (!new_skb) {
-			RNDIS_IPA_ERROR("no memory for skb expand\n");
-			return skb;
-		}
-		RNDIS_IPA_DEBUG("skb expanded. old %pK new %pK\n",
-			skb, new_skb);
-		dev_kfree_skb_any(skb);
-		skb = new_skb;
-	}
-
-	if (rndis_ipa_ctx->is_vlan_mode)
-		if (unlikely(skb->protocol != htons(ETH_P_8021Q)))
-			RNDIS_IPA_DEBUG(
-				"ether_type != ETH_P_8021Q && vlan, prot = 0x%X\n"
-				, skb->protocol);
-
-	/* make room at the head of the SKB to put the RNDIS header */
-	rndis_hdr = (struct rndis_pkt_hdr *)skb_push(skb,
-					sizeof(rndis_template_hdr));
-
-	memcpy(rndis_hdr, &rndis_template_hdr, sizeof(*rndis_hdr));
-	rndis_hdr->msg_len +=  payload_byte_len;
-	rndis_hdr->data_len +=  payload_byte_len;
-
-	return skb;
-}
-
-/**
- * rx_filter() - logic that decide if the current skb is to be filtered out
- * @skb: skb that may be sent up to the network stack
- *
- * This function shall do Rx packet filtering on the Netdev level.
- */
-static bool rx_filter(struct sk_buff *skb)
-{
-	struct rndis_ipa_dev *rndis_ipa_ctx = netdev_priv(skb->dev);
-
-	return rndis_ipa_ctx->rx_filter;
-}
-
-/**
- * tx_filter() - logic that decide if the current skb is to be filtered out
- * @skb: skb that may be sent to the USB core
- *
- * This function shall do Tx packet filtering on the Netdev level.
- * ICMP filter bypass is possible to allow only ICMP packet to be
- * sent (pings and etc)
- */
-
-static bool tx_filter(struct sk_buff *skb)
-{
-	struct rndis_ipa_dev *rndis_ipa_ctx = netdev_priv(skb->dev);
-	bool is_icmp;
-
-	if (likely(!rndis_ipa_ctx->tx_filter))
-		return false;
-
-	is_icmp = (skb->protocol == htons(ETH_P_IP)	&&
-		ip_hdr(skb)->protocol == IPPROTO_ICMP);
-
-	if ((!rndis_ipa_ctx->icmp_filter) && is_icmp)
-		return false;
-
-	return true;
-}
-
-
-/**
- * rndis_ipa_ep_registers_cfg() - configure the USB endpoints
- * @usb_to_ipa_hdl: handle received from ipa_connect which represents
- *  the USB to IPA end-point
- * @ipa_to_usb_hdl: handle received from ipa_connect which represents
- *  the IPA to USB end-point
- * @max_xfer_size_bytes_to_dev: the maximum size, in bytes, that the device
- *  expects to receive from the host. supplied on REMOTE_NDIS_INITIALIZE_CMPLT.
- * @max_xfer_size_bytes_to_host: the maximum size, in bytes, that the host
- *  expects to receive from the device. supplied on REMOTE_NDIS_INITIALIZE_MSG.
- * @mtu: the netdev MTU size, in bytes
- * @deaggr_enable: should deaggregation be enabled?
- * @is_vlan_mode: should driver work in vlan mode?
- *
- * USB to IPA pipe:
- *  - de-aggregation
- *  - Remove Ethernet header
- *  - Remove RNDIS header
- *  - SRC NAT
- *  - Default routing(0)
- * IPA to USB Pipe:
- *  - aggregation
- *  - Add Ethernet header
- *  - Add RNDIS header
- */
-static int rndis_ipa_ep_registers_cfg(
-	u32 usb_to_ipa_hdl,
-	u32 ipa_to_usb_hdl,
-	u32 max_xfer_size_bytes_to_dev,
-	u32 max_xfer_size_bytes_to_host,
-	u32 mtu,
-	bool deaggr_enable,
-	bool is_vlan_mode)
-{
-	int result;
-	struct ipa_ep_cfg *usb_to_ipa_ep_cfg;
-	int add = 0;
-
-	if (deaggr_enable) {
-		usb_to_ipa_ep_cfg = &usb_to_ipa_ep_cfg_deaggr_en;
-		RNDIS_IPA_DEBUG("deaggregation enabled\n");
-	} else {
-		usb_to_ipa_ep_cfg = &usb_to_ipa_ep_cfg_deaggr_dis;
-		RNDIS_IPA_DEBUG("deaggregation disabled\n");
-		add = sizeof(struct rndis_pkt_hdr);
-	}
-
-	if (is_vlan_mode) {
-		usb_to_ipa_ep_cfg->hdr.hdr_len =
-			VLAN_ETH_HLEN + add;
-		ipa_to_usb_ep_cfg.hdr.hdr_len =
-			VLAN_ETH_HLEN + sizeof(struct rndis_pkt_hdr);
-		ipa_to_usb_ep_cfg.hdr.hdr_additional_const_len = VLAN_ETH_HLEN;
-	} else {
-		usb_to_ipa_ep_cfg->hdr.hdr_len =
-			ETH_HLEN + add;
-		ipa_to_usb_ep_cfg.hdr.hdr_len =
-			ETH_HLEN + sizeof(struct rndis_pkt_hdr);
-		ipa_to_usb_ep_cfg.hdr.hdr_additional_const_len = ETH_HLEN;
-	}
-
-	usb_to_ipa_ep_cfg->deaggr.max_packet_len = max_xfer_size_bytes_to_dev;
-	result = ipa_cfg_ep(usb_to_ipa_hdl, usb_to_ipa_ep_cfg);
-	if (result) {
-		pr_err("failed to configure USB to IPA point\n");
-		return result;
-	}
-	RNDIS_IPA_DEBUG("IPA<-USB end-point configured\n");
-
-	ipa_to_usb_ep_cfg.aggr.aggr_byte_limit =
-		(max_xfer_size_bytes_to_host - mtu) / 1024;
-
-	if (ipa_to_usb_ep_cfg.aggr.aggr_byte_limit == 0) {
-		ipa_to_usb_ep_cfg.aggr.aggr_time_limit = 0;
-		ipa_to_usb_ep_cfg.aggr.aggr_pkt_limit = 1;
-	} else {
-		ipa_to_usb_ep_cfg.aggr.aggr_time_limit =
-			DEFAULT_AGGR_TIME_LIMIT;
-		ipa_to_usb_ep_cfg.aggr.aggr_pkt_limit =
-			DEFAULT_AGGR_PKT_LIMIT;
-	}
-
-	RNDIS_IPA_DEBUG(
-		"RNDIS aggregation param: en=%d byte_limit=%d time_limit=%d pkt_limit=%d\n"
-		, ipa_to_usb_ep_cfg.aggr.aggr_en,
-		ipa_to_usb_ep_cfg.aggr.aggr_byte_limit,
-		ipa_to_usb_ep_cfg.aggr.aggr_time_limit,
-		ipa_to_usb_ep_cfg.aggr.aggr_pkt_limit);
-
-	/* enable hdr_metadata_reg_valid */
-	usb_to_ipa_ep_cfg->hdr.hdr_metadata_reg_valid = true;
-
-	result = ipa_cfg_ep(ipa_to_usb_hdl, &ipa_to_usb_ep_cfg);
-	if (result) {
-		pr_err("failed to configure IPA to USB end-point\n");
-		return result;
-	}
-	RNDIS_IPA_DEBUG("IPA->USB end-point configured\n");
-
-	return 0;
-}
-
-/**
- * rndis_ipa_set_device_ethernet_addr() - set device Ethernet address
- * @dev_ethaddr: device Ethernet address
- *
- * Returns 0 for success, negative otherwise
- */
-static int rndis_ipa_set_device_ethernet_addr(
-	u8 *dev_ethaddr,
-	u8 device_ethaddr[])
-{
-	if (!is_valid_ether_addr(device_ethaddr))
-		return -EINVAL;
-	memcpy(dev_ethaddr, device_ethaddr, ETH_ALEN);
-
-	return 0;
-}
-
-/** rndis_ipa_next_state - return the next state of the driver
- * @current_state: the current state of the driver
- * @operation: an enum which represent the operation being made on the driver
- *  by its API.
- *
- * This function implements the driver internal state machine.
- * Its decisions are based on the driver current state and the operation
- * being made.
- * In case the operation is invalid this state machine will return
- * the value RNDIS_IPA_INVALID to inform the caller for a forbidden sequence.
- */
-static enum rndis_ipa_state rndis_ipa_next_state(
-		enum rndis_ipa_state current_state,
-		enum rndis_ipa_operation operation)
-{
-	int next_state = RNDIS_IPA_INVALID;
-
-	switch (current_state) {
-	case RNDIS_IPA_UNLOADED:
-		if (operation == RNDIS_IPA_INITIALIZE)
-			next_state = RNDIS_IPA_INITIALIZED;
-		break;
-	case RNDIS_IPA_INITIALIZED:
-		if (operation == RNDIS_IPA_CONNECT)
-			next_state = RNDIS_IPA_CONNECTED;
-		else if (operation == RNDIS_IPA_OPEN)
-			next_state = RNDIS_IPA_UP;
-		else if (operation == RNDIS_IPA_CLEANUP)
-			next_state = RNDIS_IPA_UNLOADED;
-		break;
-	case RNDIS_IPA_CONNECTED:
-		if (operation == RNDIS_IPA_DISCONNECT)
-			next_state = RNDIS_IPA_INITIALIZED;
-		else if (operation == RNDIS_IPA_OPEN)
-			next_state = RNDIS_IPA_CONNECTED_AND_UP;
-		break;
-	case RNDIS_IPA_UP:
-		if (operation == RNDIS_IPA_STOP)
-			next_state = RNDIS_IPA_INITIALIZED;
-		else if (operation == RNDIS_IPA_CONNECT)
-			next_state = RNDIS_IPA_CONNECTED_AND_UP;
-		else if (operation == RNDIS_IPA_CLEANUP)
-			next_state = RNDIS_IPA_UNLOADED;
-		break;
-	case RNDIS_IPA_CONNECTED_AND_UP:
-		if (operation == RNDIS_IPA_STOP)
-			next_state = RNDIS_IPA_CONNECTED;
-		else if (operation == RNDIS_IPA_DISCONNECT)
-			next_state = RNDIS_IPA_UP;
-		break;
-	default:
-		RNDIS_IPA_ERROR("State is not supported\n");
-		break;
-	}
-
-	RNDIS_IPA_DEBUG
-		("state transition ( %s -> %s )- %s\n",
-		rndis_ipa_state_string(current_state),
-		rndis_ipa_state_string(next_state),
-		next_state == RNDIS_IPA_INVALID ?
-		"Forbidden" : "Allowed");
-
-	return next_state;
-}
-
-/**
- * rndis_ipa_state_string - return the state string representation
- * @state: enum which describe the state
- */
-static const char *rndis_ipa_state_string(enum rndis_ipa_state state)
-{
-	switch (state) {
-	case RNDIS_IPA_UNLOADED:
-		return "RNDIS_IPA_UNLOADED";
-	case RNDIS_IPA_INITIALIZED:
-		return "RNDIS_IPA_INITIALIZED";
-	case RNDIS_IPA_CONNECTED:
-		return "RNDIS_IPA_CONNECTED";
-	case RNDIS_IPA_UP:
-		return "RNDIS_IPA_UP";
-	case RNDIS_IPA_CONNECTED_AND_UP:
-		return "RNDIS_IPA_CONNECTED_AND_UP";
-	default:
-		return "Not supported";
-	}
-}
-
-static void rndis_ipa_dump_skb(struct sk_buff *skb)
-{
-	int i;
-	u32 *cur = (u32 *)skb->data;
-	u8 *byte;
-
-	RNDIS_IPA_DEBUG
-		("packet dump start for skb->len=%d\n",
-		skb->len);
-
-	for (i = 0; i < (skb->len / 4); i++) {
-		byte = (u8 *)(cur + i);
-		pr_info
-			("%2d %08x   %02x %02x %02x %02x\n",
-			i, *(cur + i),
-			byte[0], byte[1], byte[2], byte[3]);
-	}
-	RNDIS_IPA_DEBUG
-		("packet dump ended for skb->len=%d\n", skb->len);
-}
-
-#ifdef CONFIG_DEBUG_FS
-/**
- * Creates the root folder for the driver
- */
-static void rndis_ipa_debugfs_init(struct rndis_ipa_dev *rndis_ipa_ctx)
-{
-	const mode_t flags_read_write = 0666;
-	const mode_t flags_read_only = 0444;
-	const mode_t  flags_write_only = 0222;
-	struct dentry *file;
-	struct dentry *aggr_directory;
-
-	RNDIS_IPA_LOG_ENTRY();
-
-	if (!rndis_ipa_ctx)
-		return;
-
-	rndis_ipa_ctx->directory = debugfs_create_dir(DEBUGFS_DIR_NAME, NULL);
-	if (!rndis_ipa_ctx->directory) {
-		RNDIS_IPA_ERROR("could not create debugfs directory entry\n");
-		goto fail_directory;
-	}
-
-	file = debugfs_create_bool
-		("tx_filter", flags_read_write,
-		rndis_ipa_ctx->directory, &rndis_ipa_ctx->tx_filter);
-	if (!file) {
-		RNDIS_IPA_ERROR("could not create debugfs tx_filter file\n");
-		goto fail_file;
-	}
-
-	file = debugfs_create_bool
-		("rx_filter", flags_read_write,
-		rndis_ipa_ctx->directory, &rndis_ipa_ctx->rx_filter);
-	if (!file) {
-		RNDIS_IPA_ERROR("could not create debugfs rx_filter file\n");
-		goto fail_file;
-	}
-
-	file = debugfs_create_bool
-		("icmp_filter", flags_read_write,
-		rndis_ipa_ctx->directory, &rndis_ipa_ctx->icmp_filter);
-	if (!file) {
-		RNDIS_IPA_ERROR("could not create debugfs icmp_filter file\n");
-		goto fail_file;
-	}
-
-	file = debugfs_create_u32
-		("outstanding_high", flags_read_write,
-		rndis_ipa_ctx->directory,
-		&rndis_ipa_ctx->outstanding_high);
-	if (!file) {
-		RNDIS_IPA_ERROR("could not create outstanding_high file\n");
-		goto fail_file;
-	}
-
-	file = debugfs_create_u32
-		("outstanding_low", flags_read_write,
-		rndis_ipa_ctx->directory,
-		&rndis_ipa_ctx->outstanding_low);
-	if (!file) {
-		RNDIS_IPA_ERROR("could not create outstanding_low file\n");
-		goto fail_file;
-	}
-
-	file = debugfs_create_file
-		("outstanding", flags_read_only,
-		rndis_ipa_ctx->directory,
-		rndis_ipa_ctx, &rndis_ipa_debugfs_atomic_ops);
-	if (!file) {
-		RNDIS_IPA_ERROR("could not create outstanding file\n");
-		goto fail_file;
-	}
-
-	file = debugfs_create_u8
-		("state", flags_read_only,
-		rndis_ipa_ctx->directory, (u8 *)&rndis_ipa_ctx->state);
-	if (!file) {
-		RNDIS_IPA_ERROR("could not create state file\n");
-		goto fail_file;
-	}
-
-	file = debugfs_create_u32
-		("tx_dropped", flags_read_only,
-		rndis_ipa_ctx->directory, &rndis_ipa_ctx->tx_dropped);
-	if (!file) {
-		RNDIS_IPA_ERROR("could not create tx_dropped file\n");
-		goto fail_file;
-	}
-
-	file = debugfs_create_u32
-		("rx_dropped", flags_read_only,
-		rndis_ipa_ctx->directory, &rndis_ipa_ctx->rx_dropped);
-	if (!file) {
-		RNDIS_IPA_ERROR("could not create rx_dropped file\n");
-		goto fail_file;
-	}
-
-	aggr_directory = debugfs_create_dir
-		(DEBUGFS_AGGR_DIR_NAME,
-		rndis_ipa_ctx->directory);
-	if (!aggr_directory) {
-		RNDIS_IPA_ERROR("could not create debugfs aggr entry\n");
-		goto fail_directory;
-	}
-
-	file = debugfs_create_file
-		("aggr_value_set", flags_write_only,
-		aggr_directory,
-		rndis_ipa_ctx, &rndis_ipa_aggr_ops);
-	if (!file) {
-		RNDIS_IPA_ERROR("could not create aggr_value_set file\n");
-		goto fail_file;
-	}
-
-	file = debugfs_create_u8
-		("aggr_enable", flags_read_write,
-		aggr_directory, (u8 *)&ipa_to_usb_ep_cfg.aggr.aggr_en);
-	if (!file) {
-		RNDIS_IPA_ERROR("could not create aggr_enable file\n");
-		goto fail_file;
-	}
-
-	file = debugfs_create_u8
-		("aggr_type", flags_read_write,
-		aggr_directory, (u8 *)&ipa_to_usb_ep_cfg.aggr.aggr);
-	if (!file) {
-		RNDIS_IPA_ERROR("could not create aggr_type file\n");
-		goto fail_file;
-	}
-
-	file = debugfs_create_u32
-		("aggr_byte_limit", flags_read_write,
-		aggr_directory,
-		&ipa_to_usb_ep_cfg.aggr.aggr_byte_limit);
-	if (!file) {
-		RNDIS_IPA_ERROR("could not create aggr_byte_limit file\n");
-		goto fail_file;
-	}
-
-	file = debugfs_create_u32
-		("aggr_time_limit", flags_read_write,
-		aggr_directory,
-		&ipa_to_usb_ep_cfg.aggr.aggr_time_limit);
-	if (!file) {
-		RNDIS_IPA_ERROR("could not create aggr_time_limit file\n");
-		goto fail_file;
-	}
-
-	file = debugfs_create_u32
-		("aggr_pkt_limit", flags_read_write,
-		aggr_directory,
-		&ipa_to_usb_ep_cfg.aggr.aggr_pkt_limit);
-	if (!file) {
-		RNDIS_IPA_ERROR("could not create aggr_pkt_limit file\n");
-		goto fail_file;
-	}
-
-	file = debugfs_create_bool
-		("tx_dump_enable", flags_read_write,
-		rndis_ipa_ctx->directory,
-		&rndis_ipa_ctx->tx_dump_enable);
-	if (!file) {
-		RNDIS_IPA_ERROR("fail to create tx_dump_enable file\n");
-		goto fail_file;
-	}
-
-	file = debugfs_create_bool
-		("rx_dump_enable", flags_read_write,
-		rndis_ipa_ctx->directory,
-		&rndis_ipa_ctx->rx_dump_enable);
-	if (!file) {
-		RNDIS_IPA_ERROR("fail to create rx_dump_enable file\n");
-		goto fail_file;
-	}
-
-	file = debugfs_create_bool
-		("deaggregation_enable", flags_read_write,
-		rndis_ipa_ctx->directory,
-		&rndis_ipa_ctx->deaggregation_enable);
-	if (!file) {
-		RNDIS_IPA_ERROR("fail to create deaggregation_enable file\n");
-		goto fail_file;
-	}
-
-	file = debugfs_create_u32
-		("error_msec_sleep_time", flags_read_write,
-		rndis_ipa_ctx->directory,
-		&rndis_ipa_ctx->error_msec_sleep_time);
-	if (!file) {
-		RNDIS_IPA_ERROR("fail to create error_msec_sleep_time file\n");
-		goto fail_file;
-	}
-
-	file = debugfs_create_bool
-		("during_xmit_error", flags_read_only,
-		rndis_ipa_ctx->directory,
-		&rndis_ipa_ctx->during_xmit_error);
-	if (!file) {
-		RNDIS_IPA_ERROR("fail to create during_xmit_error file\n");
-		goto fail_file;
-	}
-
-	file = debugfs_create_bool("is_vlan_mode", flags_read_only,
-		rndis_ipa_ctx->directory,
-		&rndis_ipa_ctx->is_vlan_mode);
-	if (!file) {
-		RNDIS_IPA_ERROR("fail to create is_vlan_mode file\n");
-		goto fail_file;
-	}
-
-	RNDIS_IPA_DEBUG("debugfs entries were created\n");
-	RNDIS_IPA_LOG_EXIT();
-
-	return;
-fail_file:
-	debugfs_remove_recursive(rndis_ipa_ctx->directory);
-fail_directory:
-	return;
-}
-
-static void rndis_ipa_debugfs_destroy(struct rndis_ipa_dev *rndis_ipa_ctx)
-{
-	debugfs_remove_recursive(rndis_ipa_ctx->directory);
-}
-
-#else /* !CONFIG_DEBUG_FS */
-
-static void rndis_ipa_debugfs_init(struct rndis_ipa_dev *rndis_ipa_ctx) {}
-
-static void rndis_ipa_debugfs_destroy(struct rndis_ipa_dev *rndis_ipa_ctx) {}
-
-#endif /* CONFIG_DEBUG_FS*/
-
-static int rndis_ipa_debugfs_aggr_open
-		(struct inode *inode,
-		struct file *file)
-{
-	struct rndis_ipa_dev *rndis_ipa_ctx = inode->i_private;
-
-	file->private_data = rndis_ipa_ctx;
-
-	return 0;
-}
-
-static ssize_t rndis_ipa_debugfs_aggr_write
-	(struct file *file,
-	const char __user *buf, size_t count, loff_t *ppos)
-{
-	struct rndis_ipa_dev *rndis_ipa_ctx = NULL;
-	int result;
-
-	if (file == NULL)
-		return -EFAULT;
-	rndis_ipa_ctx = file->private_data;
-
-	result = ipa_cfg_ep(rndis_ipa_ctx->usb_to_ipa_hdl, &ipa_to_usb_ep_cfg);
-	if (result) {
-		pr_err("failed to re-configure USB to IPA point\n");
-		return result;
-	}
-	pr_info("IPA<-USB end-point re-configured\n");
-
-	return count;
-}
-
-static int rndis_ipa_debugfs_atomic_open(struct inode *inode, struct file *file)
-{
-	struct rndis_ipa_dev *rndis_ipa_ctx = inode->i_private;
-
-	RNDIS_IPA_LOG_ENTRY();
-
-	file->private_data = &rndis_ipa_ctx->outstanding_pkts;
-
-	RNDIS_IPA_LOG_EXIT();
-
-	return 0;
-}
-
-static ssize_t rndis_ipa_debugfs_atomic_read
-	(struct file *file, char __user *ubuf, size_t count, loff_t *ppos)
-{
-	int nbytes;
-	u8 atomic_str[DEBUGFS_TEMP_BUF_SIZE] = {0};
-	atomic_t *atomic_var = file->private_data;
-
-	RNDIS_IPA_LOG_ENTRY();
-
-	nbytes = scnprintf
-		(atomic_str, sizeof(atomic_str), "%d\n",
-		atomic_read(atomic_var));
-
-	RNDIS_IPA_LOG_EXIT();
-
-	return simple_read_from_buffer(ubuf, count, ppos, atomic_str, nbytes);
-}
-
-static int rndis_ipa_init_module(void)
-{
-	ipa_rndis_logbuf = ipc_log_context_create(IPA_RNDIS_IPC_LOG_PAGES,
-		"ipa_rndis", 0);
-	if (ipa_rndis_logbuf == NULL)
-		RNDIS_IPA_DEBUG("failed to create IPC log, continue...\n");
-
-	pr_info("RNDIS_IPA module is loaded.\n");
-	return 0;
-}
-
-static void rndis_ipa_cleanup_module(void)
-{
-	if (ipa_rndis_logbuf)
-		ipc_log_context_destroy(ipa_rndis_logbuf);
-	ipa_rndis_logbuf = NULL;
-
-	pr_info("RNDIS_IPA module is unloaded.\n");
-}
-
-MODULE_LICENSE("GPL v2");
-MODULE_DESCRIPTION("RNDIS_IPA network interface");
-
-late_initcall(rndis_ipa_init_module);
-module_exit(rndis_ipa_cleanup_module);

ipa/ipa_clients/rndis_ipa_trace.h

@@ -1,74 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2015-2019, The Linux Foundation. All rights reserved.
- */
-
-#undef TRACE_SYSTEM
-#define TRACE_SYSTEM rndis_ipa
-#define TRACE_INCLUDE_FILE rndis_ipa_trace
-
-#if !defined(_RNDIS_IPA_TRACE_H) || defined(TRACE_HEADER_MULTI_READ)
-#define _RNDIS_IPA_TRACE_H
-
-#include <linux/tracepoint.h>
-
-TRACE_EVENT(
-	rndis_netif_ni,
-
-	TP_PROTO(unsigned long proto),
-
-	TP_ARGS(proto),
-
-	TP_STRUCT__entry(
-		__field(unsigned long,	proto)
-	),
-
-	TP_fast_assign(
-		__entry->proto = proto;
-	),
-
-	TP_printk("proto =%lu\n", __entry->proto)
-);
-
-TRACE_EVENT(
-	rndis_tx_dp,
-
-	TP_PROTO(unsigned long proto),
-
-	TP_ARGS(proto),
-
-	TP_STRUCT__entry(
-		__field(unsigned long,	proto)
-	),
-
-	TP_fast_assign(
-		__entry->proto = proto;
-	),
-
-	TP_printk("proto =%lu\n", __entry->proto)
-);
-
-TRACE_EVENT(
-	rndis_status_rcvd,
-
-	TP_PROTO(unsigned long proto),
-
-	TP_ARGS(proto),
-
-	TP_STRUCT__entry(
-		__field(unsigned long,	proto)
-	),
-
-	TP_fast_assign(
-		__entry->proto = proto;
-	),
-
-	TP_printk("proto =%lu\n", __entry->proto)
-);
-
-#endif /* _RNDIS_IPA_TRACE_H */
-
-/* This part must be outside protection */
-#undef TRACE_INCLUDE_PATH
-#define TRACE_INCLUDE_PATH ../../techpack/dataipa/ipa/ipa_clients
-#include <trace/define_trace.h>

ipa/ipa_common_i.h

@@ -1,477 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2012-2019, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/ipa_mhi.h>
-#include <linux/ipa_qmi_service_v01.h>
-
-#ifndef _IPA_COMMON_I_H_
-#define _IPA_COMMON_I_H_
-#include <linux/errno.h>
-#include <linux/ipc_logging.h>
-#include <linux/ipa.h>
-#include <linux/ipa_uc_offload.h>
-#include <linux/ipa_wdi3.h>
-#include <linux/ipa_wigig.h>
-#include <linux/ratelimit.h>
-
-#define WARNON_RATELIMIT_BURST 1
-#define IPA_RATELIMIT_BURST 1
-
-#define __FILENAME__ \
-	(strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__)
-
-#define IPA_ACTIVE_CLIENTS_PREP_EP(log_info, client) \
-		log_info.file = __FILENAME__; \
-		log_info.line = __LINE__; \
-		log_info.type = EP; \
-		log_info.id_string = (client < 0 || client >= IPA_CLIENT_MAX) \
-			? "Invalid Client" : ipa_clients_strings[client]
-
-#define IPA_ACTIVE_CLIENTS_PREP_SIMPLE(log_info) \
-		log_info.file = __FILENAME__; \
-		log_info.line = __LINE__; \
-		log_info.type = SIMPLE; \
-		log_info.id_string = __func__
-
-#define IPA_ACTIVE_CLIENTS_PREP_RESOURCE(log_info, resource_name) \
-		log_info.file = __FILENAME__; \
-		log_info.line = __LINE__; \
-		log_info.type = RESOURCE; \
-		log_info.id_string = resource_name
-
-#define IPA_ACTIVE_CLIENTS_PREP_SPECIAL(log_info, id_str) \
-		log_info.file = __FILENAME__; \
-		log_info.line = __LINE__; \
-		log_info.type = SPECIAL; \
-		log_info.id_string = id_str
-
-#define IPA_ACTIVE_CLIENTS_INC_EP(client) \
-	do { \
-		struct ipa_active_client_logging_info log_info; \
-		IPA_ACTIVE_CLIENTS_PREP_EP(log_info, client); \
-		ipa_inc_client_enable_clks(&log_info); \
-	} while (0)
-
-#define IPA_ACTIVE_CLIENTS_DEC_EP(client) \
-	do { \
-		struct ipa_active_client_logging_info log_info; \
-		IPA_ACTIVE_CLIENTS_PREP_EP(log_info, client); \
-		ipa_dec_client_disable_clks(&log_info); \
-	} while (0)
-
-#define IPA_ACTIVE_CLIENTS_INC_SIMPLE() \
-	do { \
-		struct ipa_active_client_logging_info log_info; \
-		IPA_ACTIVE_CLIENTS_PREP_SIMPLE(log_info); \
-		ipa_inc_client_enable_clks(&log_info); \
-	} while (0)
-
-#define IPA_ACTIVE_CLIENTS_DEC_SIMPLE() \
-	do { \
-		struct ipa_active_client_logging_info log_info; \
-		IPA_ACTIVE_CLIENTS_PREP_SIMPLE(log_info); \
-		ipa_dec_client_disable_clks(&log_info); \
-	} while (0)
-
-#define IPA_ACTIVE_CLIENTS_INC_RESOURCE(resource_name) \
-	do { \
-		struct ipa_active_client_logging_info log_info; \
-		IPA_ACTIVE_CLIENTS_PREP_RESOURCE(log_info, resource_name); \
-		ipa_inc_client_enable_clks(&log_info); \
-	} while (0)
-
-#define IPA_ACTIVE_CLIENTS_DEC_RESOURCE(resource_name) \
-	do { \
-		struct ipa_active_client_logging_info log_info; \
-		IPA_ACTIVE_CLIENTS_PREP_RESOURCE(log_info, resource_name); \
-		ipa_dec_client_disable_clks(&log_info); \
-	} while (0)
-
-#define IPA_ACTIVE_CLIENTS_INC_SPECIAL(id_str) \
-	do { \
-		struct ipa_active_client_logging_info log_info; \
-		IPA_ACTIVE_CLIENTS_PREP_SPECIAL(log_info, id_str); \
-		ipa_inc_client_enable_clks(&log_info); \
-	} while (0)
-
-#define IPA_ACTIVE_CLIENTS_DEC_SPECIAL(id_str) \
-	do { \
-		struct ipa_active_client_logging_info log_info; \
-		IPA_ACTIVE_CLIENTS_PREP_SPECIAL(log_info, id_str); \
-		ipa_dec_client_disable_clks(&log_info); \
-	} while (0)
-
-/*
- * Printing one warning message in 5 seconds if multiple warning messages
- * are coming back to back.
- */
-
-#define WARN_ON_RATELIMIT_IPA(condition)				\
-({								\
-	static DEFINE_RATELIMIT_STATE(_rs,			\
-				DEFAULT_RATELIMIT_INTERVAL,	\
-				WARNON_RATELIMIT_BURST);	\
-	int rtn = !!(condition);				\
-								\
-	if (unlikely(rtn && __ratelimit(&_rs)))			\
-		WARN_ON(rtn);					\
-})
-
-/*
- * Printing one error message in 5 seconds if multiple error messages
- * are coming back to back.
- */
-
-#define pr_err_ratelimited_ipa(fmt, args...)				\
-({									\
-	static DEFINE_RATELIMIT_STATE(_rs,				\
-				      DEFAULT_RATELIMIT_INTERVAL,	\
-				      IPA_RATELIMIT_BURST);		\
-									\
-	if (__ratelimit(&_rs))						\
-		pr_err(fmt, ## args);					\
-})
-
-#define ipa_assert_on(condition)\
-do {\
-	if (unlikely(condition))\
-		ipa_assert();\
-} while (0)
-
-#define IPA_CLIENT_IS_PROD(x) \
-	(x < IPA_CLIENT_MAX && (x & 0x1) == 0)
-#define IPA_CLIENT_IS_CONS(x) \
-	(x < IPA_CLIENT_MAX && (x & 0x1) == 1)
-
-#define IPA_GSI_CHANNEL_STOP_SLEEP_MIN_USEC (1000)
-#define IPA_GSI_CHANNEL_STOP_SLEEP_MAX_USEC (2000)
-
-enum ipa_active_client_log_type {
-	EP,
-	SIMPLE,
-	RESOURCE,
-	SPECIAL,
-	INVALID
-};
-
-struct ipa_active_client_logging_info {
-	const char *id_string;
-	char *file;
-	int line;
-	enum ipa_active_client_log_type type;
-};
-
-/**
- * struct ipa_mem_buffer - IPA memory buffer
- * @base: base
- * @phys_base: physical base address
- * @size: size of memory buffer
- */
-struct ipa_mem_buffer {
-	void *base;
-	dma_addr_t phys_base;
-	u32 size;
-};
-
-/**
- * enum ipa3_mhi_burst_mode - MHI channel burst mode state
- *
- * Values are according to MHI specification
- * @IPA_MHI_BURST_MODE_DEFAULT: burst mode enabled for HW channels,
- * disabled for SW channels
- * @IPA_MHI_BURST_MODE_RESERVED:
- * @IPA_MHI_BURST_MODE_DISABLE: Burst mode is disabled for this channel
- * @IPA_MHI_BURST_MODE_ENABLE: Burst mode is enabled for this channel
- *
- */
-enum ipa3_mhi_burst_mode {
-	IPA_MHI_BURST_MODE_DEFAULT,
-	IPA_MHI_BURST_MODE_RESERVED,
-	IPA_MHI_BURST_MODE_DISABLE,
-	IPA_MHI_BURST_MODE_ENABLE,
-};
-
-/**
- * enum ipa_hw_mhi_channel_states - MHI channel state machine
- *
- * Values are according to MHI specification
- * @IPA_HW_MHI_CHANNEL_STATE_DISABLE: Channel is disabled and not processed by
- *	the host or device.
- * @IPA_HW_MHI_CHANNEL_STATE_ENABLE: A channel is enabled after being
- *	initialized and configured by host, including its channel context and
- *	associated transfer ring. While this state, the channel is not active
- *	and the device does not process transfer.
- * @IPA_HW_MHI_CHANNEL_STATE_RUN: The device processes transfers and doorbell
- *	for channels.
- * @IPA_HW_MHI_CHANNEL_STATE_SUSPEND: Used to halt operations on the channel.
- *	The device does not process transfers for the channel in this state.
- *	This state is typically used to synchronize the transition to low power
- *	modes.
- * @IPA_HW_MHI_CHANNEL_STATE_STOP: Used to halt operations on the channel.
- *	The device does not process transfers for the channel in this state.
- * @IPA_HW_MHI_CHANNEL_STATE_ERROR: The device detected an error in an element
- *	from the transfer ring associated with the channel.
- * @IPA_HW_MHI_CHANNEL_STATE_INVALID: Invalid state. Shall not be in use in
- *	operational scenario.
- */
-enum ipa_hw_mhi_channel_states {
-	IPA_HW_MHI_CHANNEL_STATE_DISABLE	= 0,
-	IPA_HW_MHI_CHANNEL_STATE_ENABLE		= 1,
-	IPA_HW_MHI_CHANNEL_STATE_RUN		= 2,
-	IPA_HW_MHI_CHANNEL_STATE_SUSPEND	= 3,
-	IPA_HW_MHI_CHANNEL_STATE_STOP		= 4,
-	IPA_HW_MHI_CHANNEL_STATE_ERROR		= 5,
-	IPA_HW_MHI_CHANNEL_STATE_INVALID	= 0xFF
-};
-
-/**
- * Structure holding the parameters for IPA_CPU_2_HW_CMD_MHI_DL_UL_SYNC_INFO
- * command. Parameters are sent as 32b immediate parameters.
- * @isDlUlSyncEnabled: Flag to indicate if DL UL Syncronization is enabled
- * @UlAccmVal: UL Timer Accumulation value (Period after which device will poll
- *	for UL data)
- * @ulMsiEventThreshold: Threshold at which HW fires MSI to host for UL events
- * @dlMsiEventThreshold: Threshold at which HW fires MSI to host for DL events
- */
-union IpaHwMhiDlUlSyncCmdData_t {
-	struct IpaHwMhiDlUlSyncCmdParams_t {
-		u32 isDlUlSyncEnabled:8;
-		u32 UlAccmVal:8;
-		u32 ulMsiEventThreshold:8;
-		u32 dlMsiEventThreshold:8;
-	} params;
-	u32 raw32b;
-};
-
-struct ipa_mhi_ch_ctx {
-	u8 chstate;/*0-7*/
-	u8 brstmode:2;/*8-9*/
-	u8 pollcfg:6;/*10-15*/
-	u16 rsvd;/*16-31*/
-	u32 chtype;
-	u32 erindex;
-	u64 rbase;
-	u64 rlen;
-	u64 rp;
-	u64 wp;
-} __packed;
-
-struct ipa_mhi_ev_ctx {
-	u32 intmodc:16;
-	u32 intmodt:16;
-	u32 ertype;
-	u32 msivec;
-	u64 rbase;
-	u64 rlen;
-	u64 rp;
-	u64 wp;
-} __packed;
-
-struct ipa_mhi_init_uc_engine {
-	struct ipa_mhi_msi_info *msi;
-	u32 mmio_addr;
-	u32 host_ctrl_addr;
-	u32 host_data_addr;
-	u32 first_ch_idx;
-	u32 first_er_idx;
-	union IpaHwMhiDlUlSyncCmdData_t *ipa_cached_dl_ul_sync_info;
-};
-
-struct ipa_mhi_init_gsi_engine {
-	u32 first_ch_idx;
-};
-
-struct ipa_mhi_init_engine {
-	struct ipa_mhi_init_uc_engine uC;
-	struct ipa_mhi_init_gsi_engine gsi;
-};
-
-struct start_gsi_channel {
-	enum ipa_hw_mhi_channel_states state;
-	struct ipa_mhi_msi_info *msi;
-	struct ipa_mhi_ev_ctx *ev_ctx_host;
-	u64 event_context_addr;
-	struct ipa_mhi_ch_ctx *ch_ctx_host;
-	u64 channel_context_addr;
-	void (*ch_err_cb)(struct gsi_chan_err_notify *notify);
-	void (*ev_err_cb)(struct gsi_evt_err_notify *notify);
-	void *channel;
-	bool assert_bit40;
-	struct gsi_mhi_channel_scratch *mhi;
-	unsigned long *cached_gsi_evt_ring_hdl;
-	uint8_t evchid;
-};
-
-struct start_uc_channel {
-	enum ipa_hw_mhi_channel_states state;
-	u8 index;
-	u8 id;
-};
-
-struct start_mhi_channel {
-	struct start_uc_channel uC;
-	struct start_gsi_channel gsi;
-};
-
-struct ipa_mhi_connect_params_internal {
-	struct ipa_sys_connect_params *sys;
-	u8 channel_id;
-	struct start_mhi_channel start;
-};
-
-/**
- * struct ipa_hdr_offset_entry - IPA header offset entry
- * @link: entry's link in global header offset entries list
- * @offset: the offset
- * @bin: bin
- * @ipacm_installed: indicate if installed by ipacm
- */
-struct ipa_hdr_offset_entry {
-	struct list_head link;
-	u32 offset;
-	u32 bin;
-	bool ipacm_installed;
-};
-
-extern const char *ipa_clients_strings[];
-
-#define IPA_IPC_LOGGING(buf, fmt, args...) \
-	do { \
-		if (buf) \
-			ipc_log_string((buf), fmt, __func__, __LINE__, \
-				## args); \
-	} while (0)
-
-void ipa_inc_client_enable_clks(struct ipa_active_client_logging_info *id);
-void ipa_dec_client_disable_clks(struct ipa_active_client_logging_info *id);
-int ipa_inc_client_enable_clks_no_block(
-	struct ipa_active_client_logging_info *id);
-int ipa_suspend_resource_no_block(enum ipa_rm_resource_name resource);
-int ipa_resume_resource(enum ipa_rm_resource_name name);
-int ipa_suspend_resource_sync(enum ipa_rm_resource_name resource);
-int ipa_set_required_perf_profile(enum ipa_voltage_level floor_voltage,
-	u32 bandwidth_mbps);
-void *ipa_get_ipc_logbuf(void);
-void *ipa_get_ipc_logbuf_low(void);
-void ipa_assert(void);
-
-/* MHI */
-int ipa_mhi_init_engine(struct ipa_mhi_init_engine *params);
-int ipa_connect_mhi_pipe(struct ipa_mhi_connect_params_internal *in,
-		u32 *clnt_hdl);
-int ipa_disconnect_mhi_pipe(u32 clnt_hdl);
-bool ipa_mhi_stop_gsi_channel(enum ipa_client_type client);
-int ipa_qmi_enable_force_clear_datapath_send(
-	struct ipa_enable_force_clear_datapath_req_msg_v01 *req);
-int ipa_qmi_disable_force_clear_datapath_send(
-	struct ipa_disable_force_clear_datapath_req_msg_v01 *req);
-int ipa_generate_tag_process(void);
-int ipa_disable_sps_pipe(enum ipa_client_type client);
-int ipa_mhi_reset_channel_internal(enum ipa_client_type client);
-int ipa_mhi_start_channel_internal(enum ipa_client_type client);
-bool ipa_mhi_sps_channel_empty(enum ipa_client_type client);
-int ipa_mhi_resume_channels_internal(enum ipa_client_type client,
-		bool LPTransitionRejected, bool brstmode_enabled,
-		union __packed gsi_channel_scratch ch_scratch, u8 index);
-int ipa_mhi_handle_ipa_config_req(struct ipa_config_req_msg_v01 *config_req);
-int ipa_mhi_query_ch_info(enum ipa_client_type client,
-		struct gsi_chan_info *ch_info);
-int ipa_mhi_destroy_channel(enum ipa_client_type client);
-int ipa_mhi_is_using_dma(bool *flag);
-const char *ipa_mhi_get_state_str(int state);
-
-/* MHI uC */
-int ipa_uc_mhi_send_dl_ul_sync_info(union IpaHwMhiDlUlSyncCmdData_t *cmd);
-int ipa_uc_mhi_init
-	(void (*ready_cb)(void), void (*wakeup_request_cb)(void));
-void ipa_uc_mhi_cleanup(void);
-int ipa_uc_mhi_reset_channel(int channelHandle);
-int ipa_uc_mhi_suspend_channel(int channelHandle);
-int ipa_uc_mhi_stop_event_update_channel(int channelHandle);
-int ipa_uc_mhi_print_stats(char *dbg_buff, int size);
-
-/* uC */
-int ipa_uc_state_check(void);
-
-/* general */
-void ipa_get_holb(int ep_idx, struct ipa_ep_cfg_holb *holb);
-void ipa_set_tag_process_before_gating(bool val);
-bool ipa_has_open_aggr_frame(enum ipa_client_type client);
-int ipa_setup_uc_ntn_pipes(struct ipa_ntn_conn_in_params *in,
-	ipa_notify_cb notify, void *priv, u8 hdr_len,
-	struct ipa_ntn_conn_out_params *outp);
-
-int ipa_tear_down_uc_offload_pipes(int ipa_ep_idx_ul, int ipa_ep_idx_dl,
-	struct ipa_ntn_conn_in_params *params);
-u8 *ipa_write_64(u64 w, u8 *dest);
-u8 *ipa_write_32(u32 w, u8 *dest);
-u8 *ipa_write_16(u16 hw, u8 *dest);
-u8 *ipa_write_8(u8 b, u8 *dest);
-u8 *ipa_pad_to_64(u8 *dest);
-u8 *ipa_pad_to_32(u8 *dest);
-int ipa_ntn_uc_reg_rdyCB(void (*ipauc_ready_cb)(void *user_data),
-			      void *user_data);
-void ipa_ntn_uc_dereg_rdyCB(void);
-
-int ipa_conn_wdi_pipes(struct ipa_wdi_conn_in_params *in,
-	struct ipa_wdi_conn_out_params *out,
-	ipa_wdi_meter_notifier_cb wdi_notify);
-
-int ipa_disconn_wdi_pipes(int ipa_ep_idx_tx, int ipa_ep_idx_rx);
-
-int ipa_enable_wdi_pipes(int ipa_ep_idx_tx, int ipa_ep_idx_rx);
-
-int ipa_disable_wdi_pipes(int ipa_ep_idx_tx, int ipa_ep_idx_rx);
-
-const char *ipa_get_version_string(enum ipa_hw_type ver);
-int ipa_start_gsi_channel(u32 clnt_hdl);
-
-int ipa_smmu_store_sgt(struct sg_table **out_ch_ptr,
-		struct sg_table *in_sgt_ptr);
-int ipa_smmu_free_sgt(struct sg_table **out_sgt_ptr);
-
-int ipa_ut_module_init(void);
-void ipa_ut_module_exit(void);
-
-int ipa_wigig_internal_init(
-	struct ipa_wdi_uc_ready_params *inout,
-	ipa_wigig_misc_int_cb int_notify,
-	phys_addr_t *uc_db_pa);
-
-int ipa_conn_wigig_rx_pipe_i(void *in, struct ipa_wigig_conn_out_params *out,
-	struct dentry **parent);
-
-int ipa_conn_wigig_client_i(void *in, struct ipa_wigig_conn_out_params *out,
-	ipa_notify_cb tx_notify,
-	void *priv);
-
-int ipa_wigig_uc_msi_init(
-	bool init,
-	phys_addr_t periph_baddr_pa,
-	phys_addr_t pseudo_cause_pa,
-	phys_addr_t int_gen_tx_pa,
-	phys_addr_t int_gen_rx_pa,
-	phys_addr_t dma_ep_misc_pa);
-
-int ipa_disconn_wigig_pipe_i(enum ipa_client_type client,
-	struct ipa_wigig_pipe_setup_info_smmu *pipe_smmu,
-	void *dbuff);
-
-int ipa_enable_wigig_pipe_i(enum ipa_client_type client);
-
-int ipa_disable_wigig_pipe_i(enum ipa_client_type client);
-
-int ipa_wigig_send_msg(int msg_type,
-	const char *netdev_name, u8 *mac,
-	enum ipa_client_type client, bool to_wigig);
-
-int ipa_wigig_save_regs(void);
-
-void ipa_register_client_callback(int (*client_cb)(bool is_lock),
-			bool (*teth_port_state)(void), u32 ipa_ep_idx);
-
-void ipa_deregister_client_callback(u32 ipa_ep_idx);
-
-#endif /* _IPA_COMMON_I_H_ */

ipa/ipa_rm.c

@@ -1,1184 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2013-2018, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/slab.h>
-#include <linux/workqueue.h>
-#include <linux/ipa.h>
-#include "ipa_rm_dependency_graph.h"
-#include "ipa_rm_i.h"
-#include "ipa_common_i.h"
-
-static const char *resource_name_to_str[IPA_RM_RESOURCE_MAX] = {
-	__stringify(IPA_RM_RESOURCE_Q6_PROD),
-	__stringify(IPA_RM_RESOURCE_Q6_CONS),
-	__stringify(IPA_RM_RESOURCE_USB_PROD),
-	__stringify(IPA_RM_RESOURCE_USB_CONS),
-	__stringify(IPA_RM_RESOURCE_USB_DPL_DUMMY_PROD),
-	__stringify(IPA_RM_RESOURCE_USB_DPL_CONS),
-	__stringify(IPA_RM_RESOURCE_HSIC_PROD),
-	__stringify(IPA_RM_RESOURCE_HSIC_CONS),
-	__stringify(IPA_RM_RESOURCE_STD_ECM_PROD),
-	__stringify(IPA_RM_RESOURCE_APPS_CONS),
-	__stringify(IPA_RM_RESOURCE_RNDIS_PROD),
-	__stringify(RESERVED_CONS_11),
-	__stringify(IPA_RM_RESOURCE_WWAN_0_PROD),
-	__stringify(RESERVED_CONS_13),
-	__stringify(IPA_RM_RESOURCE_WLAN_PROD),
-	__stringify(IPA_RM_RESOURCE_WLAN_CONS),
-	__stringify(IPA_RM_RESOURCE_ODU_ADAPT_PROD),
-	__stringify(IPA_RM_RESOURCE_ODU_ADAPT_CONS),
-	__stringify(IPA_RM_RESOURCE_MHI_PROD),
-	__stringify(IPA_RM_RESOURCE_MHI_CONS),
-	__stringify(IPA_RM_RESOURCE_ETHERNET_PROD),
-	__stringify(IPA_RM_RESOURCE_ETHERNET_CONS),
-};
-
-struct ipa_rm_profile_vote_type {
-	enum ipa_voltage_level volt[IPA_RM_RESOURCE_MAX];
-	enum ipa_voltage_level curr_volt;
-	u32 bw_resources[IPA_RM_RESOURCE_MAX];
-	u32 curr_bw;
-};
-
-struct ipa_rm_context_type {
-	struct ipa_rm_dep_graph *dep_graph;
-	struct workqueue_struct *ipa_rm_wq;
-	spinlock_t ipa_rm_lock;
-	struct ipa_rm_profile_vote_type prof_vote;
-};
-static struct ipa_rm_context_type *ipa_rm_ctx;
-
-struct ipa_rm_notify_ipa_work_type {
-	struct work_struct		work;
-	enum ipa_voltage_level		volt;
-	u32				bandwidth_mbps;
-};
-
-/**
- * ipa_rm_create_resource() - create resource
- * @create_params: [in] parameters needed
- *                  for resource initialization
- *
- * Returns: 0 on success, negative on failure
- *
- * This function is called by IPA RM client to initialize client's resources.
- * This API should be called before any other IPA RM API on a given resource
- * name.
- *
- */
-int ipa_rm_create_resource(struct ipa_rm_create_params *create_params)
-{
-	struct ipa_rm_resource *resource;
-	unsigned long flags;
-	int result;
-
-	if (unlikely(!ipa_rm_ctx)) {
-		IPA_RM_ERR("IPA RM was not initialized\n");
-		return -EINVAL;
-	}
-
-	if (!create_params) {
-		IPA_RM_ERR("invalid args\n");
-		return -EINVAL;
-	}
-	IPA_RM_DBG("%s\n", ipa_rm_resource_str(create_params->name));
-
-	if (create_params->floor_voltage < 0 ||
-		create_params->floor_voltage >= IPA_VOLTAGE_MAX) {
-		IPA_RM_ERR("invalid voltage %d\n",
-			create_params->floor_voltage);
-		return -EINVAL;
-	}
-
-	spin_lock_irqsave(&ipa_rm_ctx->ipa_rm_lock, flags);
-	if (ipa_rm_dep_graph_get_resource(ipa_rm_ctx->dep_graph,
-					  create_params->name,
-					  &resource) == 0) {
-		IPA_RM_ERR("resource already exists\n");
-		result = -EEXIST;
-		goto bail;
-	}
-	result = ipa_rm_resource_create(create_params,
-			&resource);
-	if (result) {
-		IPA_RM_ERR("ipa_rm_resource_create() failed\n");
-		goto bail;
-	}
-	result = ipa_rm_dep_graph_add(ipa_rm_ctx->dep_graph, resource);
-	if (result) {
-		IPA_RM_ERR("ipa_rm_dep_graph_add() failed\n");
-		ipa_rm_resource_delete(resource);
-		goto bail;
-	}
-bail:
-	spin_unlock_irqrestore(&ipa_rm_ctx->ipa_rm_lock, flags);
-	IPA_RM_DBG("EXIT with %d\n", result);
-
-	return result;
-}
-EXPORT_SYMBOL(ipa_rm_create_resource);
-
-/**
- * ipa_rm_delete_resource() - delete resource
- * @resource_name: name of resource to be deleted
- *
- * Returns: 0 on success, negative on failure
- *
- * This function is called by IPA RM client to delete client's resources.
- *
- */
-int ipa_rm_delete_resource(enum ipa_rm_resource_name resource_name)
-{
-	struct ipa_rm_resource *resource;
-	unsigned long flags;
-	int result;
-
-	if (unlikely(!ipa_rm_ctx)) {
-		IPA_RM_ERR("IPA RM was not initialized\n");
-		return -EINVAL;
-	}
-
-	IPA_RM_DBG("%s\n", ipa_rm_resource_str(resource_name));
-	spin_lock_irqsave(&ipa_rm_ctx->ipa_rm_lock, flags);
-	if (ipa_rm_dep_graph_get_resource(ipa_rm_ctx->dep_graph,
-					resource_name,
-						&resource) != 0) {
-		IPA_RM_ERR("resource does not exist\n");
-		result = -EINVAL;
-		goto bail;
-	}
-	result = ipa_rm_resource_delete(resource);
-	if (result) {
-		IPA_RM_ERR("ipa_rm_resource_delete() failed\n");
-		goto bail;
-	}
-	result = ipa_rm_dep_graph_remove(ipa_rm_ctx->dep_graph,
-								resource_name);
-	if (result) {
-		IPA_RM_ERR("ipa_rm_dep_graph_remove() failed\n");
-		goto bail;
-	}
-bail:
-	spin_unlock_irqrestore(&ipa_rm_ctx->ipa_rm_lock, flags);
-	IPA_RM_DBG("EXIT with %d\n", result);
-
-	return result;
-}
-EXPORT_SYMBOL(ipa_rm_delete_resource);
-
-static int _ipa_rm_add_dependency(enum ipa_rm_resource_name resource_name,
-			enum ipa_rm_resource_name depends_on_name,
-			bool userspace_dep)
-{
-	unsigned long flags;
-	int result;
-
-	if (unlikely(!ipa_rm_ctx)) {
-		IPA_RM_ERR("IPA RM was not initialized\n");
-		return -EINVAL;
-	}
-
-	IPA_RM_DBG("%s -> %s\n", ipa_rm_resource_str(resource_name),
-				 ipa_rm_resource_str(depends_on_name));
-	spin_lock_irqsave(&ipa_rm_ctx->ipa_rm_lock, flags);
-	result = ipa_rm_dep_graph_add_dependency(
-						ipa_rm_ctx->dep_graph,
-						resource_name,
-						depends_on_name,
-						userspace_dep);
-	spin_unlock_irqrestore(&ipa_rm_ctx->ipa_rm_lock, flags);
-	IPA_RM_DBG("EXIT with %d\n", result);
-
-	return result;
-}
-
-/**
- * ipa_rm_add_dependency() - create dependency between 2 resources
- * @resource_name: name of dependent resource
- * @depends_on_name: name of its dependency
- *
- * Returns: 0 on success, negative on failure
- *
- * Side effects: IPA_RM_RESORCE_GRANTED could be generated
- * in case client registered with IPA RM
- */
-int ipa_rm_add_dependency(enum ipa_rm_resource_name resource_name,
-			enum ipa_rm_resource_name depends_on_name)
-{
-	return _ipa_rm_add_dependency(resource_name, depends_on_name, false);
-}
-EXPORT_SYMBOL(ipa_rm_add_dependency);
-
-/**
- * ipa_rm_add_dependency_from_ioctl() - create dependency between 2 resources
- * @resource_name: name of dependent resource
- * @depends_on_name: name of its dependency
- *
- * This function is expected to be called from IOCTL and the dependency will be
- * marked as is was added by the userspace.
- *
- * Returns: 0 on success, negative on failure
- *
- * Side effects: IPA_RM_RESORCE_GRANTED could be generated
- * in case client registered with IPA RM
- */
-int ipa_rm_add_dependency_from_ioctl(enum ipa_rm_resource_name resource_name,
-			enum ipa_rm_resource_name depends_on_name)
-{
-	return _ipa_rm_add_dependency(resource_name, depends_on_name, true);
-}
-
-static int _ipa_rm_add_dependency_sync(enum ipa_rm_resource_name resource_name,
-		enum ipa_rm_resource_name depends_on_name,
-		bool userspsace_dep)
-{
-	int result;
-	struct ipa_rm_resource *consumer;
-	unsigned long time;
-	unsigned long flags;
-
-	if (unlikely(!ipa_rm_ctx)) {
-		IPA_RM_ERR("IPA RM was not initialized\n");
-		return -EINVAL;
-	}
-
-	IPA_RM_DBG("%s -> %s\n", ipa_rm_resource_str(resource_name),
-				 ipa_rm_resource_str(depends_on_name));
-	spin_lock_irqsave(&ipa_rm_ctx->ipa_rm_lock, flags);
-	result = ipa_rm_dep_graph_add_dependency(
-						ipa_rm_ctx->dep_graph,
-						resource_name,
-						depends_on_name,
-						userspsace_dep);
-	spin_unlock_irqrestore(&ipa_rm_ctx->ipa_rm_lock, flags);
-	if (result == -EINPROGRESS) {
-		ipa_rm_dep_graph_get_resource(ipa_rm_ctx->dep_graph,
-				depends_on_name,
-				&consumer);
-		IPA_RM_DBG("%s waits for GRANT of %s.\n",
-				ipa_rm_resource_str(resource_name),
-				ipa_rm_resource_str(depends_on_name));
-		time = wait_for_completion_timeout(
-				&((struct ipa_rm_resource_cons *)consumer)->
-				request_consumer_in_progress,
-				HZ * 5);
-		result = 0;
-		if (!time) {
-			IPA_RM_ERR("TIMEOUT waiting for %s GRANT event.",
-					ipa_rm_resource_str(depends_on_name));
-			result = -ETIME;
-		} else {
-			IPA_RM_DBG("%s waited for %s GRANT %lu time.\n",
-				ipa_rm_resource_str(resource_name),
-				ipa_rm_resource_str(depends_on_name),
-				time);
-		}
-	}
-	IPA_RM_DBG("EXIT with %d\n", result);
-
-	return result;
-}
-/**
- * ipa_rm_add_dependency_sync() - Create a dependency between 2 resources
- * in a synchronized fashion. In case a producer resource is in GRANTED state
- * and the newly added consumer resource is in RELEASED state, the consumer
- * entity will be requested and the function will block until the consumer
- * is granted.
- * @resource_name: name of dependent resource
- * @depends_on_name: name of its dependency
- *
- * This function is expected to be called from IOCTL and the dependency will be
- * marked as is was added by the userspace.
- *
- * Returns: 0 on success, negative on failure
- *
- * Side effects: May block. See documentation above.
- */
-int ipa_rm_add_dependency_sync(enum ipa_rm_resource_name resource_name,
-		enum ipa_rm_resource_name depends_on_name)
-{
-	return _ipa_rm_add_dependency_sync(resource_name, depends_on_name,
-		false);
-}
-EXPORT_SYMBOL(ipa_rm_add_dependency_sync);
-
-/**
- * ipa_rm_add_dependency_sync_from_ioctl() - Create a dependency between 2
- * resources in a synchronized fashion. In case a producer resource is in
- * GRANTED state and the newly added consumer resource is in RELEASED state,
- * the consumer entity will be requested and the function will block until
- * the consumer is granted.
- * @resource_name: name of dependent resource
- * @depends_on_name: name of its dependency
- *
- * Returns: 0 on success, negative on failure
- *
- * Side effects: May block. See documentation above.
- */
-int ipa_rm_add_dependency_sync_from_ioctl(
-	enum ipa_rm_resource_name resource_name,
-	enum ipa_rm_resource_name depends_on_name)
-{
-	return _ipa_rm_add_dependency_sync(resource_name, depends_on_name,
-		true);
-}
-
-static int _ipa_rm_delete_dependency(enum ipa_rm_resource_name resource_name,
-			enum ipa_rm_resource_name depends_on_name,
-			bool userspace_dep)
-{
-	unsigned long flags;
-	int result;
-
-	if (unlikely(!ipa_rm_ctx)) {
-		IPA_RM_ERR("IPA RM was not initialized\n");
-		return -EINVAL;
-	}
-
-	IPA_RM_DBG("%s -> %s\n", ipa_rm_resource_str(resource_name),
-				 ipa_rm_resource_str(depends_on_name));
-	spin_lock_irqsave(&ipa_rm_ctx->ipa_rm_lock, flags);
-	result = ipa_rm_dep_graph_delete_dependency(
-			  ipa_rm_ctx->dep_graph,
-			  resource_name,
-			  depends_on_name,
-			  userspace_dep);
-	spin_unlock_irqrestore(&ipa_rm_ctx->ipa_rm_lock, flags);
-	IPA_RM_DBG("EXIT with %d\n", result);
-
-	return result;
-}
-
-/**
- * ipa_rm_delete_dependency() - delete dependency between 2 resources
- * @resource_name: name of dependent resource
- * @depends_on_name: name of its dependency
- *
- * Returns: 0 on success, negative on failure
- *
- * Side effects: IPA_RM_RESORCE_GRANTED could be generated
- * in case client registered with IPA RM
- */
-int ipa_rm_delete_dependency(enum ipa_rm_resource_name resource_name,
-			enum ipa_rm_resource_name depends_on_name)
-{
-	return _ipa_rm_delete_dependency(resource_name, depends_on_name, false);
-}
-EXPORT_SYMBOL(ipa_rm_delete_dependency);
-
-/**
- * ipa_rm_delete_dependency_fron_ioctl() - delete dependency between 2 resources
- * @resource_name: name of dependent resource
- * @depends_on_name: name of its dependency
- *
- * This function is expected to be called from IOCTL and the dependency will be
- * marked as is was added by the userspace.
- *
- * Returns: 0 on success, negative on failure
- *
- * Side effects: IPA_RM_RESORCE_GRANTED could be generated
- * in case client registered with IPA RM
- */
-int ipa_rm_delete_dependency_from_ioctl(enum ipa_rm_resource_name resource_name,
-			enum ipa_rm_resource_name depends_on_name)
-{
-	return _ipa_rm_delete_dependency(resource_name, depends_on_name, true);
-}
-
-/**
- * ipa_rm_request_resource() - request resource
- * @resource_name: [in] name of the requested resource
- *
- * Returns: 0 on success, negative on failure
- *
- * All registered callbacks are called with IPA_RM_RESOURCE_GRANTED
- * on successful completion of this operation.
- */
-int ipa_rm_request_resource(enum ipa_rm_resource_name resource_name)
-{
-	struct ipa_rm_resource *resource;
-	unsigned long flags;
-	int result;
-
-	if (unlikely(!ipa_rm_ctx)) {
-		IPA_RM_ERR("IPA RM was not initialized\n");
-		return -EINVAL;
-	}
-
-	if (!IPA_RM_RESORCE_IS_PROD(resource_name)) {
-		IPA_RM_ERR("can be called on PROD only\n");
-		return -EINVAL;
-	}
-	spin_lock_irqsave(&ipa_rm_ctx->ipa_rm_lock, flags);
-	if (ipa_rm_dep_graph_get_resource(ipa_rm_ctx->dep_graph,
-			resource_name,
-			&resource) != 0) {
-		IPA_RM_ERR("resource does not exist\n");
-		result = -EPERM;
-		goto bail;
-	}
-	result = ipa_rm_resource_producer_request(
-			(struct ipa_rm_resource_prod *)resource);
-
-bail:
-	spin_unlock_irqrestore(&ipa_rm_ctx->ipa_rm_lock, flags);
-
-	return result;
-}
-EXPORT_SYMBOL(ipa_rm_request_resource);
-
-void delayed_release_work_func(struct work_struct *work)
-{
-	unsigned long flags;
-	struct ipa_rm_resource *resource;
-	struct ipa_rm_delayed_release_work_type *rwork = container_of(
-			to_delayed_work(work),
-			struct ipa_rm_delayed_release_work_type,
-			work);
-
-	if (!IPA_RM_RESORCE_IS_CONS(rwork->resource_name)) {
-		IPA_RM_ERR("can be called on CONS only\n");
-		kfree(rwork);
-		return;
-	}
-	spin_lock_irqsave(&ipa_rm_ctx->ipa_rm_lock, flags);
-	if (ipa_rm_dep_graph_get_resource(ipa_rm_ctx->dep_graph,
-					rwork->resource_name,
-					&resource) != 0) {
-		IPA_RM_ERR("resource does not exists\n");
-		goto bail;
-	}
-
-	ipa_rm_resource_consumer_release(
-		(struct ipa_rm_resource_cons *)resource, rwork->needed_bw,
-		rwork->dec_usage_count);
-
-bail:
-	spin_unlock_irqrestore(&ipa_rm_ctx->ipa_rm_lock, flags);
-	kfree(rwork);
-
-}
-
-/**
- * ipa_rm_request_resource_with_timer() - requests the specified consumer
- * resource and releases it after 1 second
- * @resource_name: name of the requested resource
- *
- * Returns: 0 on success, negative on failure
- */
-int ipa_rm_request_resource_with_timer(enum ipa_rm_resource_name resource_name)
-{
-	unsigned long flags;
-	struct ipa_rm_resource *resource;
-	struct ipa_rm_delayed_release_work_type *release_work;
-	int result;
-
-	if (!IPA_RM_RESORCE_IS_CONS(resource_name)) {
-		IPA_RM_ERR("can be called on CONS only\n");
-		return -EINVAL;
-	}
-
-	spin_lock_irqsave(&ipa_rm_ctx->ipa_rm_lock, flags);
-	if (ipa_rm_dep_graph_get_resource(ipa_rm_ctx->dep_graph,
-			resource_name,
-			&resource) != 0) {
-		IPA_RM_ERR("resource does not exists\n");
-		result = -EPERM;
-		goto bail;
-	}
-	result = ipa_rm_resource_consumer_request(
-		(struct ipa_rm_resource_cons *)resource, 0, false, true);
-	if (result != 0 && result != -EINPROGRESS) {
-		IPA_RM_ERR("consumer request returned error %d\n", result);
-		result = -EPERM;
-		goto bail;
-	}
-
-	release_work = kzalloc(sizeof(*release_work), GFP_ATOMIC);
-	if (!release_work) {
-		result = -ENOMEM;
-		goto bail;
-	}
-	release_work->resource_name = resource->name;
-	release_work->needed_bw = 0;
-	release_work->dec_usage_count = false;
-	INIT_DELAYED_WORK(&release_work->work, delayed_release_work_func);
-	schedule_delayed_work(&release_work->work,
-			msecs_to_jiffies(IPA_RM_RELEASE_DELAY_IN_MSEC));
-	result = 0;
-bail:
-	spin_unlock_irqrestore(&ipa_rm_ctx->ipa_rm_lock, flags);
-
-	return result;
-}
-
-/**
- * ipa_rm_release_resource() - release resource
- * @resource_name: [in] name of the requested resource
- *
- * Returns: 0 on success, negative on failure
- *
- * All registered callbacks are called with IPA_RM_RESOURCE_RELEASED
- * on successful completion of this operation.
- */
-int ipa_rm_release_resource(enum ipa_rm_resource_name resource_name)
-{
-	unsigned long flags;
-	struct ipa_rm_resource *resource;
-	int result;
-
-	if (unlikely(!ipa_rm_ctx)) {
-		IPA_RM_ERR("IPA RM was not initialized\n");
-		return -EINVAL;
-	}
-
-	if (!IPA_RM_RESORCE_IS_PROD(resource_name)) {
-		IPA_RM_ERR("can be called on PROD only\n");
-		return -EINVAL;
-	}
-	spin_lock_irqsave(&ipa_rm_ctx->ipa_rm_lock, flags);
-	if (ipa_rm_dep_graph_get_resource(ipa_rm_ctx->dep_graph,
-					  resource_name,
-					  &resource) != 0) {
-		IPA_RM_ERR("resource does not exists\n");
-		result = -EPERM;
-		goto bail;
-	}
-	result = ipa_rm_resource_producer_release(
-		    (struct ipa_rm_resource_prod *)resource);
-
-bail:
-	spin_unlock_irqrestore(&ipa_rm_ctx->ipa_rm_lock, flags);
-
-	return result;
-}
-EXPORT_SYMBOL(ipa_rm_release_resource);
-
-/**
- * ipa_rm_register() - register for event
- * @resource_name: resource name
- * @reg_params: [in] registration parameters
- *
- * Returns: 0 on success, negative on failure
- *
- * Registration parameters provided here should be the same
- * as provided later in  ipa_rm_deregister() call.
- */
-int ipa_rm_register(enum ipa_rm_resource_name resource_name,
-			struct ipa_rm_register_params *reg_params)
-{
-	int result;
-	unsigned long flags;
-	struct ipa_rm_resource *resource;
-
-	IPA_RM_DBG("%s\n", ipa_rm_resource_str(resource_name));
-
-	if (!IPA_RM_RESORCE_IS_PROD(resource_name)) {
-		IPA_RM_ERR("can be called on PROD only\n");
-		return -EINVAL;
-	}
-	spin_lock_irqsave(&ipa_rm_ctx->ipa_rm_lock, flags);
-	if (ipa_rm_dep_graph_get_resource(ipa_rm_ctx->dep_graph,
-				resource_name,
-				&resource) != 0) {
-		IPA_RM_ERR("resource does not exists\n");
-		result = -EPERM;
-		goto bail;
-	}
-	result = ipa_rm_resource_producer_register(
-			(struct ipa_rm_resource_prod *)resource,
-			reg_params,
-			true);
-bail:
-	spin_unlock_irqrestore(&ipa_rm_ctx->ipa_rm_lock, flags);
-	IPA_RM_DBG("EXIT with %d\n", result);
-
-	return result;
-}
-EXPORT_SYMBOL(ipa_rm_register);
-
-/**
- * ipa_rm_deregister() - cancel the registration
- * @resource_name: resource name
- * @reg_params: [in] registration parameters
- *
- * Returns: 0 on success, negative on failure
- *
- * Registration parameters provided here should be the same
- * as provided in  ipa_rm_register() call.
- */
-int ipa_rm_deregister(enum ipa_rm_resource_name resource_name,
-			struct ipa_rm_register_params *reg_params)
-{
-	int result;
-	unsigned long flags;
-	struct ipa_rm_resource *resource;
-
-	IPA_RM_DBG("%s\n", ipa_rm_resource_str(resource_name));
-
-	if (!IPA_RM_RESORCE_IS_PROD(resource_name)) {
-		IPA_RM_ERR("can be called on PROD only\n");
-		return -EINVAL;
-	}
-	spin_lock_irqsave(&ipa_rm_ctx->ipa_rm_lock, flags);
-	if (ipa_rm_dep_graph_get_resource(ipa_rm_ctx->dep_graph,
-			resource_name,
-			&resource) != 0) {
-		IPA_RM_ERR("resource does not exists\n");
-		result = -EPERM;
-		goto bail;
-	}
-	result = ipa_rm_resource_producer_deregister(
-			(struct ipa_rm_resource_prod *)resource,
-			reg_params);
-bail:
-	spin_unlock_irqrestore(&ipa_rm_ctx->ipa_rm_lock, flags);
-	IPA_RM_DBG("EXIT with %d\n", result);
-
-	return result;
-}
-EXPORT_SYMBOL(ipa_rm_deregister);
-
-/**
- * ipa_rm_set_perf_profile() - set performance profile
- * @resource_name: resource name
- * @profile: [in] profile information.
- *
- * Returns: 0 on success, negative on failure
- *
- * Set resource performance profile.
- * Updates IPA driver if performance level changed.
- */
-int ipa_rm_set_perf_profile(enum ipa_rm_resource_name resource_name,
-			struct ipa_rm_perf_profile *profile)
-{
-	int result;
-	unsigned long flags;
-	struct ipa_rm_resource *resource;
-
-	if (unlikely(!ipa_rm_ctx)) {
-		IPA_RM_ERR("IPA RM was not initialized\n");
-		return -EINVAL;
-	}
-
-	IPA_RM_DBG("%s\n", ipa_rm_resource_str(resource_name));
-	if (profile)
-		IPA_RM_DBG("BW: %d\n", profile->max_supported_bandwidth_mbps);
-
-	spin_lock_irqsave(&ipa_rm_ctx->ipa_rm_lock, flags);
-	if (ipa_rm_dep_graph_get_resource(ipa_rm_ctx->dep_graph,
-				resource_name,
-				&resource) != 0) {
-		IPA_RM_ERR("resource does not exists\n");
-		result = -EPERM;
-		goto bail;
-	}
-	result = ipa_rm_resource_set_perf_profile(resource, profile);
-	if (result) {
-		IPA_RM_ERR("ipa_rm_resource_set_perf_profile failed %d\n",
-			result);
-		goto bail;
-	}
-
-	result = 0;
-bail:
-	spin_unlock_irqrestore(&ipa_rm_ctx->ipa_rm_lock, flags);
-	IPA_RM_DBG("EXIT with %d\n", result);
-
-	return result;
-}
-EXPORT_SYMBOL(ipa_rm_set_perf_profile);
-
-/**
- * ipa_rm_notify_completion() -
- *	consumer driver notification for
- *	request_resource / release_resource operations
- *	completion
- * @event: notified event
- * @resource_name: resource name
- *
- * Returns: 0 on success, negative on failure
- */
-int ipa_rm_notify_completion(enum ipa_rm_event event,
-		enum ipa_rm_resource_name resource_name)
-{
-	int result;
-
-	if (unlikely(!ipa_rm_ctx)) {
-		IPA_RM_ERR("IPA RM was not initialized\n");
-		return -EINVAL;
-	}
-
-	IPA_RM_DBG("event %d on %s\n", event,
-				ipa_rm_resource_str(resource_name));
-	if (!IPA_RM_RESORCE_IS_CONS(resource_name)) {
-		IPA_RM_ERR("can be called on CONS only\n");
-		result = -EINVAL;
-		goto bail;
-	}
-	ipa_rm_wq_send_cmd(IPA_RM_WQ_RESOURCE_CB,
-			resource_name,
-			event,
-			false);
-	result = 0;
-bail:
-	IPA_RM_DBG("EXIT with %d\n", result);
-
-	return result;
-}
-EXPORT_SYMBOL(ipa_rm_notify_completion);
-
-static void ipa_rm_wq_handler(struct work_struct *work)
-{
-	unsigned long flags;
-	struct ipa_rm_resource *resource;
-	struct ipa_rm_wq_work_type *ipa_rm_work =
-			container_of(work,
-					struct ipa_rm_wq_work_type,
-					work);
-	IPA_RM_DBG_LOW("%s cmd=%d event=%d notify_registered_only=%d\n",
-		ipa_rm_resource_str(ipa_rm_work->resource_name),
-		ipa_rm_work->wq_cmd,
-		ipa_rm_work->event,
-		ipa_rm_work->notify_registered_only);
-	switch (ipa_rm_work->wq_cmd) {
-	case IPA_RM_WQ_NOTIFY_PROD:
-		if (!IPA_RM_RESORCE_IS_PROD(ipa_rm_work->resource_name)) {
-			IPA_RM_ERR("resource is not PROD\n");
-			goto free_work;
-		}
-		spin_lock_irqsave(&ipa_rm_ctx->ipa_rm_lock, flags);
-		if (ipa_rm_dep_graph_get_resource(ipa_rm_ctx->dep_graph,
-						ipa_rm_work->resource_name,
-						&resource) != 0){
-			IPA_RM_ERR("resource does not exists\n");
-			spin_unlock_irqrestore(&ipa_rm_ctx->ipa_rm_lock, flags);
-			goto free_work;
-		}
-		ipa_rm_resource_producer_notify_clients(
-				(struct ipa_rm_resource_prod *)resource,
-				ipa_rm_work->event,
-				ipa_rm_work->notify_registered_only);
-		spin_unlock_irqrestore(&ipa_rm_ctx->ipa_rm_lock, flags);
-		break;
-	case IPA_RM_WQ_NOTIFY_CONS:
-		break;
-	case IPA_RM_WQ_RESOURCE_CB:
-		spin_lock_irqsave(&ipa_rm_ctx->ipa_rm_lock, flags);
-		if (ipa_rm_dep_graph_get_resource(ipa_rm_ctx->dep_graph,
-						ipa_rm_work->resource_name,
-						&resource) != 0){
-			IPA_RM_ERR("resource does not exists\n");
-			spin_unlock_irqrestore(&ipa_rm_ctx->ipa_rm_lock, flags);
-			goto free_work;
-		}
-		ipa_rm_resource_consumer_handle_cb(
-				(struct ipa_rm_resource_cons *)resource,
-				ipa_rm_work->event);
-		spin_unlock_irqrestore(&ipa_rm_ctx->ipa_rm_lock, flags);
-		break;
-	default:
-		break;
-	}
-
-free_work:
-	kfree((void *) work);
-}
-
-static void ipa_rm_wq_resume_handler(struct work_struct *work)
-{
-	unsigned long flags;
-	struct ipa_rm_resource *resource;
-	struct ipa_rm_wq_suspend_resume_work_type *ipa_rm_work =
-			container_of(work,
-			struct ipa_rm_wq_suspend_resume_work_type,
-			work);
-		IPA_RM_DBG_LOW("resume work handler: %s",
-		ipa_rm_resource_str(ipa_rm_work->resource_name));
-
-	if (!IPA_RM_RESORCE_IS_CONS(ipa_rm_work->resource_name)) {
-		IPA_RM_ERR("resource is not CONS\n");
-		return;
-	}
-	IPA_ACTIVE_CLIENTS_INC_RESOURCE(ipa_rm_resource_str(
-			ipa_rm_work->resource_name));
-	spin_lock_irqsave(&ipa_rm_ctx->ipa_rm_lock, flags);
-	if (ipa_rm_dep_graph_get_resource(ipa_rm_ctx->dep_graph,
-					ipa_rm_work->resource_name,
-					&resource) != 0){
-		IPA_RM_ERR("resource does not exists\n");
-		spin_unlock_irqrestore(&ipa_rm_ctx->ipa_rm_lock, flags);
-		IPA_ACTIVE_CLIENTS_DEC_RESOURCE(ipa_rm_resource_str(
-				ipa_rm_work->resource_name));
-		goto bail;
-	}
-	ipa_rm_resource_consumer_request_work(
-			(struct ipa_rm_resource_cons *)resource,
-			ipa_rm_work->prev_state, ipa_rm_work->needed_bw, true,
-			ipa_rm_work->inc_usage_count);
-	spin_unlock_irqrestore(&ipa_rm_ctx->ipa_rm_lock, flags);
-bail:
-	kfree(ipa_rm_work);
-}
-
-
-static void ipa_rm_wq_suspend_handler(struct work_struct *work)
-{
-	unsigned long flags;
-	struct ipa_rm_resource *resource;
-	struct ipa_rm_wq_suspend_resume_work_type *ipa_rm_work =
-			container_of(work,
-			struct ipa_rm_wq_suspend_resume_work_type,
-			work);
-		IPA_RM_DBG_LOW("suspend work handler: %s",
-		ipa_rm_resource_str(ipa_rm_work->resource_name));
-
-	if (!IPA_RM_RESORCE_IS_CONS(ipa_rm_work->resource_name)) {
-		IPA_RM_ERR("resource is not CONS\n");
-		return;
-	}
-	ipa_suspend_resource_sync(ipa_rm_work->resource_name);
-	spin_lock_irqsave(&ipa_rm_ctx->ipa_rm_lock, flags);
-	if (ipa_rm_dep_graph_get_resource(ipa_rm_ctx->dep_graph,
-					ipa_rm_work->resource_name,
-					&resource) != 0){
-		IPA_RM_ERR("resource does not exists\n");
-		spin_unlock_irqrestore(&ipa_rm_ctx->ipa_rm_lock, flags);
-		return;
-	}
-	ipa_rm_resource_consumer_release_work(
-			(struct ipa_rm_resource_cons *)resource,
-			ipa_rm_work->prev_state,
-			true);
-	spin_unlock_irqrestore(&ipa_rm_ctx->ipa_rm_lock, flags);
-
-	kfree(ipa_rm_work);
-}
-
-/**
- * ipa_rm_wq_send_cmd() - send a command for deferred work
- * @wq_cmd: command that should be executed
- * @resource_name: resource on which command should be executed
- * @notify_registered_only: notify only clients registered by
- *	ipa_rm_register()
- *
- * Returns: 0 on success, negative otherwise
- */
-int ipa_rm_wq_send_cmd(enum ipa_rm_wq_cmd wq_cmd,
-		enum ipa_rm_resource_name resource_name,
-		enum ipa_rm_event event,
-		bool notify_registered_only)
-{
-	int result = -ENOMEM;
-	struct ipa_rm_wq_work_type *work = kzalloc(sizeof(*work), GFP_ATOMIC);
-
-	if (work) {
-		INIT_WORK((struct work_struct *)work, ipa_rm_wq_handler);
-		work->wq_cmd = wq_cmd;
-		work->resource_name = resource_name;
-		work->event = event;
-		work->notify_registered_only = notify_registered_only;
-		result = queue_work(ipa_rm_ctx->ipa_rm_wq,
-				(struct work_struct *)work);
-	}
-
-	return result;
-}
-
-int ipa_rm_wq_send_suspend_cmd(enum ipa_rm_resource_name resource_name,
-		enum ipa_rm_resource_state prev_state,
-		u32 needed_bw)
-{
-	int result = -ENOMEM;
-	struct ipa_rm_wq_suspend_resume_work_type *work = kzalloc(sizeof(*work),
-			GFP_ATOMIC);
-	if (work) {
-		INIT_WORK((struct work_struct *)work,
-				ipa_rm_wq_suspend_handler);
-		work->resource_name = resource_name;
-		work->prev_state = prev_state;
-		work->needed_bw = needed_bw;
-		result = queue_work(ipa_rm_ctx->ipa_rm_wq,
-				(struct work_struct *)work);
-	}
-
-	return result;
-}
-
-int ipa_rm_wq_send_resume_cmd(enum ipa_rm_resource_name resource_name,
-		enum ipa_rm_resource_state prev_state,
-		u32 needed_bw,
-		bool inc_usage_count)
-{
-	int result = -ENOMEM;
-	struct ipa_rm_wq_suspend_resume_work_type *work = kzalloc(sizeof(*work),
-			GFP_ATOMIC);
-	if (work) {
-		INIT_WORK((struct work_struct *)work, ipa_rm_wq_resume_handler);
-		work->resource_name = resource_name;
-		work->prev_state = prev_state;
-		work->needed_bw = needed_bw;
-		work->inc_usage_count = inc_usage_count;
-		result = queue_work(ipa_rm_ctx->ipa_rm_wq,
-				(struct work_struct *)work);
-	} else {
-		IPA_RM_ERR("no mem\n");
-	}
-
-	return result;
-}
-/**
- * ipa_rm_initialize() - initialize IPA RM component
- *
- * Returns: 0 on success, negative otherwise
- */
-int ipa_rm_initialize(void)
-{
-	int result;
-
-	ipa_rm_ctx = kzalloc(sizeof(*ipa_rm_ctx), GFP_KERNEL);
-	if (!ipa_rm_ctx) {
-		IPA_RM_ERR("no mem\n");
-		result = -ENOMEM;
-		goto bail;
-	}
-	ipa_rm_ctx->ipa_rm_wq = create_singlethread_workqueue("ipa_rm_wq");
-	if (!ipa_rm_ctx->ipa_rm_wq) {
-		IPA_RM_ERR("create workqueue failed\n");
-		result = -ENOMEM;
-		goto create_wq_fail;
-	}
-	result = ipa_rm_dep_graph_create(&(ipa_rm_ctx->dep_graph));
-	if (result) {
-		IPA_RM_ERR("create dependency graph failed\n");
-		goto graph_alloc_fail;
-	}
-	spin_lock_init(&ipa_rm_ctx->ipa_rm_lock);
-	IPA_RM_DBG("SUCCESS\n");
-
-	return 0;
-graph_alloc_fail:
-	destroy_workqueue(ipa_rm_ctx->ipa_rm_wq);
-create_wq_fail:
-	kfree(ipa_rm_ctx);
-bail:
-	return result;
-}
-
-/**
- * ipa_rm_stat() - print RM stat
- * @buf: [in] The user buff used to print
- * @size: [in] The size of buf
- * Returns: number of bytes used on success, negative on failure
- *
- * This function is called by ipa_debugfs in order to receive
- * a full picture of the current state of the RM
- */
-
-int ipa_rm_stat(char *buf, int size)
-{
-	unsigned long flags;
-	int i, cnt = 0, result = EINVAL;
-	struct ipa_rm_resource *resource = NULL;
-	u32 sum_bw_prod = 0;
-	u32 sum_bw_cons = 0;
-
-	if (!buf || size < 0)
-		return result;
-
-	spin_lock_irqsave(&ipa_rm_ctx->ipa_rm_lock, flags);
-	for (i = 0; i < IPA_RM_RESOURCE_MAX; ++i) {
-		if (!IPA_RM_RESORCE_IS_PROD(i))
-			continue;
-		result = ipa_rm_dep_graph_get_resource(
-				ipa_rm_ctx->dep_graph,
-				i,
-				&resource);
-		if (!result) {
-			result = ipa_rm_resource_producer_print_stat(
-							resource, buf + cnt,
-							size-cnt);
-			if (result < 0)
-				goto bail;
-			cnt += result;
-		}
-	}
-
-	for (i = 0; i < IPA_RM_RESOURCE_MAX; i++) {
-		if (IPA_RM_RESORCE_IS_PROD(i))
-			sum_bw_prod += ipa_rm_ctx->prof_vote.bw_resources[i];
-		else
-			sum_bw_cons += ipa_rm_ctx->prof_vote.bw_resources[i];
-	}
-
-	result = scnprintf(buf + cnt, size - cnt,
-		"All prod bandwidth: %d, All cons bandwidth: %d\n",
-		sum_bw_prod, sum_bw_cons);
-	cnt += result;
-
-	result = scnprintf(buf + cnt, size - cnt,
-		"Voting: voltage %d, bandwidth %d\n",
-		ipa_rm_ctx->prof_vote.curr_volt,
-		ipa_rm_ctx->prof_vote.curr_bw);
-	cnt += result;
-
-	result = cnt;
-bail:
-	spin_unlock_irqrestore(&ipa_rm_ctx->ipa_rm_lock, flags);
-
-	return result;
-}
-
-/**
- * ipa_rm_resource_str() - returns string that represent the resource
- * @resource_name: [in] resource name
- */
-const char *ipa_rm_resource_str(enum ipa_rm_resource_name resource_name)
-{
-	if (resource_name < 0 || resource_name >= IPA_RM_RESOURCE_MAX)
-		return "INVALID RESOURCE";
-
-	return resource_name_to_str[resource_name];
-};
-
-static void ipa_rm_perf_profile_notify_to_ipa_work(struct work_struct *work)
-{
-	struct ipa_rm_notify_ipa_work_type *notify_work = container_of(work,
-				struct ipa_rm_notify_ipa_work_type,
-				work);
-	int res;
-
-	IPA_RM_DBG_LOW("calling to IPA driver. voltage %d bandwidth %d\n",
-		notify_work->volt, notify_work->bandwidth_mbps);
-
-	res = ipa_set_required_perf_profile(notify_work->volt,
-		notify_work->bandwidth_mbps);
-	if (res) {
-		IPA_RM_ERR("ipa_set_required_perf_profile failed %d\n", res);
-		goto bail;
-	}
-
-	IPA_RM_DBG_LOW("IPA driver notified\n");
-bail:
-	kfree(notify_work);
-}
-
-static void ipa_rm_perf_profile_notify_to_ipa(enum ipa_voltage_level volt,
-					      u32 bandwidth)
-{
-	struct ipa_rm_notify_ipa_work_type *work;
-
-	work = kzalloc(sizeof(*work), GFP_ATOMIC);
-	if (!work)
-		return;
-
-	INIT_WORK(&work->work, ipa_rm_perf_profile_notify_to_ipa_work);
-	work->volt = volt;
-	work->bandwidth_mbps = bandwidth;
-	queue_work(ipa_rm_ctx->ipa_rm_wq, &work->work);
-}
-
-/**
- * ipa_rm_perf_profile_change() - change performance profile vote for resource
- * @resource_name: [in] resource name
- *
- * change bandwidth and voltage vote based on resource state.
- */
-void ipa_rm_perf_profile_change(enum ipa_rm_resource_name resource_name)
-{
-	enum ipa_voltage_level old_volt;
-	u32 *bw_ptr;
-	u32 old_bw;
-	struct ipa_rm_resource *resource;
-	int i;
-	u32 sum_bw_prod = 0;
-	u32 sum_bw_cons = 0;
-
-	IPA_RM_DBG_LOW("%s\n", ipa_rm_resource_str(resource_name));
-
-	if (ipa_rm_dep_graph_get_resource(ipa_rm_ctx->dep_graph,
-					  resource_name,
-					  &resource) != 0) {
-		IPA_RM_ERR("resource does not exists\n");
-		WARN_ON(1);
-		return;
-	}
-
-	old_volt = ipa_rm_ctx->prof_vote.curr_volt;
-	old_bw = ipa_rm_ctx->prof_vote.curr_bw;
-
-	bw_ptr = &ipa_rm_ctx->prof_vote.bw_resources[resource_name];
-
-	switch (resource->state) {
-	case IPA_RM_GRANTED:
-	case IPA_RM_REQUEST_IN_PROGRESS:
-		IPA_RM_DBG_LOW("max_bw = %d, needed_bw = %d\n",
-			resource->max_bw, resource->needed_bw);
-		*bw_ptr = min(resource->max_bw, resource->needed_bw);
-		ipa_rm_ctx->prof_vote.volt[resource_name] =
-						resource->floor_voltage;
-		break;
-
-	case IPA_RM_RELEASE_IN_PROGRESS:
-	case IPA_RM_RELEASED:
-		*bw_ptr = 0;
-		ipa_rm_ctx->prof_vote.volt[resource_name] = 0;
-		break;
-
-	default:
-		IPA_RM_ERR("unknown state %d\n", resource->state);
-		WARN_ON(1);
-	return;
-	}
-	IPA_RM_DBG_LOW("resource bandwidth: %d voltage: %d\n", *bw_ptr,
-					resource->floor_voltage);
-
-	ipa_rm_ctx->prof_vote.curr_volt = IPA_VOLTAGE_UNSPECIFIED;
-	for (i = 0; i < IPA_RM_RESOURCE_MAX; i++) {
-		if (ipa_rm_ctx->prof_vote.volt[i] >
-				ipa_rm_ctx->prof_vote.curr_volt) {
-			ipa_rm_ctx->prof_vote.curr_volt =
-				ipa_rm_ctx->prof_vote.volt[i];
-		}
-	}
-
-	for (i = 0; i < IPA_RM_RESOURCE_MAX; i++) {
-		if (IPA_RM_RESORCE_IS_PROD(i))
-			sum_bw_prod += ipa_rm_ctx->prof_vote.bw_resources[i];
-		else
-			sum_bw_cons += ipa_rm_ctx->prof_vote.bw_resources[i];
-	}
-
-	IPA_RM_DBG_LOW("all prod bandwidth: %d all cons bandwidth: %d\n",
-		sum_bw_prod, sum_bw_cons);
-	ipa_rm_ctx->prof_vote.curr_bw = min(sum_bw_prod, sum_bw_cons);
-
-	if (ipa_rm_ctx->prof_vote.curr_volt == old_volt &&
-		ipa_rm_ctx->prof_vote.curr_bw == old_bw) {
-		IPA_RM_DBG_LOW("same voting\n");
-		return;
-	}
-
-	IPA_RM_DBG_LOW("new voting: voltage %d bandwidth %d\n",
-		ipa_rm_ctx->prof_vote.curr_volt,
-		ipa_rm_ctx->prof_vote.curr_bw);
-
-	ipa_rm_perf_profile_notify_to_ipa(ipa_rm_ctx->prof_vote.curr_volt,
-			ipa_rm_ctx->prof_vote.curr_bw);
-
-	return;
-};
-/**
- * ipa_rm_exit() - free all IPA RM resources
- */
-void ipa_rm_exit(void)
-{
-	IPA_RM_DBG("ENTER\n");
-	ipa_rm_dep_graph_delete(ipa_rm_ctx->dep_graph);
-	destroy_workqueue(ipa_rm_ctx->ipa_rm_wq);
-	kfree(ipa_rm_ctx);
-	ipa_rm_ctx = NULL;
-	IPA_RM_DBG("EXIT\n");
-}

ipa/ipa_rm_dependency_graph.c

@@ -1,240 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2013-2018, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/slab.h>
-#include "ipa_rm_dependency_graph.h"
-#include "ipa_rm_i.h"
-
-static int ipa_rm_dep_get_index(enum ipa_rm_resource_name resource_name)
-{
-	int resource_index = IPA_RM_INDEX_INVALID;
-
-	if (IPA_RM_RESORCE_IS_PROD(resource_name))
-		resource_index = ipa_rm_prod_index(resource_name);
-	else if (IPA_RM_RESORCE_IS_CONS(resource_name))
-		resource_index = ipa_rm_cons_index(resource_name);
-
-	return resource_index;
-}
-
-/**
- * ipa_rm_dep_graph_create() - creates graph
- * @dep_graph: [out] created dependency graph
- *
- * Returns: dependency graph on success, NULL on failure
- */
-int  ipa_rm_dep_graph_create(struct ipa_rm_dep_graph **dep_graph)
-{
-	int result = 0;
-
-	*dep_graph = kzalloc(sizeof(**dep_graph), GFP_KERNEL);
-	if (!*dep_graph)
-		result = -ENOMEM;
-	return result;
-}
-
-/**
- * ipa_rm_dep_graph_delete() - destroyes the graph
- * @graph: [in] dependency graph
- *
- * Frees all resources.
- */
-void ipa_rm_dep_graph_delete(struct ipa_rm_dep_graph *graph)
-{
-	int resource_index;
-
-	if (!graph) {
-		IPA_RM_ERR("invalid params\n");
-		return;
-	}
-	for (resource_index = 0;
-			resource_index < IPA_RM_RESOURCE_MAX;
-			resource_index++)
-		kfree(graph->resource_table[resource_index]);
-	memset(graph->resource_table, 0, sizeof(graph->resource_table));
-}
-
-/**
- * ipa_rm_dep_graph_get_resource() - provides a resource by name
- * @graph: [in] dependency graph
- * @name: [in] name of the resource
- * @resource: [out] resource in case of success
- *
- * Returns: 0 on success, negative on failure
- */
-int ipa_rm_dep_graph_get_resource(
-				struct ipa_rm_dep_graph *graph,
-				enum ipa_rm_resource_name resource_name,
-				struct ipa_rm_resource **resource)
-{
-	int result;
-	int resource_index;
-
-	if (!graph) {
-		result = -EINVAL;
-		goto bail;
-	}
-	resource_index = ipa_rm_dep_get_index(resource_name);
-	if (resource_index == IPA_RM_INDEX_INVALID) {
-		result = -EINVAL;
-		goto bail;
-	}
-	*resource = graph->resource_table[resource_index];
-	if (!*resource) {
-		result = -EINVAL;
-		goto bail;
-	}
-	result = 0;
-bail:
-	return result;
-}
-
-/**
- * ipa_rm_dep_graph_add() - adds resource to graph
- * @graph: [in] dependency graph
- * @resource: [in] resource to add
- *
- * Returns: 0 on success, negative on failure
- */
-int ipa_rm_dep_graph_add(struct ipa_rm_dep_graph *graph,
-			 struct ipa_rm_resource *resource)
-{
-	int result = 0;
-	int resource_index;
-
-	if (!graph || !resource) {
-		result = -EINVAL;
-		goto bail;
-	}
-	resource_index = ipa_rm_dep_get_index(resource->name);
-	if (resource_index == IPA_RM_INDEX_INVALID) {
-		result = -EINVAL;
-		goto bail;
-	}
-	graph->resource_table[resource_index] = resource;
-bail:
-	return result;
-}
-
-/**
- * ipa_rm_dep_graph_remove() - removes resource from graph
- * @graph: [in] dependency graph
- * @resource: [in] resource to add
- *
- * Returns: 0 on success, negative on failure
- */
-int ipa_rm_dep_graph_remove(struct ipa_rm_dep_graph *graph,
-		enum ipa_rm_resource_name resource_name)
-{
-	if (!graph)
-		return -EINVAL;
-	graph->resource_table[resource_name] = NULL;
-
-	return 0;
-}
-
-/**
- * ipa_rm_dep_graph_add_dependency() - adds dependency between
- *				two nodes in graph
- * @graph: [in] dependency graph
- * @resource_name: [in] resource to add
- * @depends_on_name: [in] resource to add
- * @userspace_dep: [in] operation requested by userspace ?
- *
- * Returns: 0 on success, negative on failure
- */
-int ipa_rm_dep_graph_add_dependency(struct ipa_rm_dep_graph *graph,
-				    enum ipa_rm_resource_name resource_name,
-				    enum ipa_rm_resource_name depends_on_name,
-				    bool userspace_dep)
-{
-	struct ipa_rm_resource *dependent = NULL;
-	struct ipa_rm_resource *dependency = NULL;
-	int result;
-
-	if (!graph ||
-		!IPA_RM_RESORCE_IS_PROD(resource_name) ||
-		!IPA_RM_RESORCE_IS_CONS(depends_on_name)) {
-		IPA_RM_ERR("invalid params\n");
-		result = -EINVAL;
-		goto bail;
-	}
-	if (ipa_rm_dep_graph_get_resource(graph,
-					  resource_name,
-					  &dependent)) {
-		IPA_RM_ERR("%s does not exist\n",
-					ipa_rm_resource_str(resource_name));
-		result = -EINVAL;
-		goto bail;
-	}
-	if (ipa_rm_dep_graph_get_resource(graph,
-					depends_on_name,
-					  &dependency)) {
-		IPA_RM_ERR("%s does not exist\n",
-					ipa_rm_resource_str(depends_on_name));
-		result = -EINVAL;
-		goto bail;
-	}
-	result = ipa_rm_resource_add_dependency(dependent, dependency,
-		userspace_dep);
-bail:
-	IPA_RM_DBG("EXIT with %d\n", result);
-
-	return result;
-}
-
-/**
- * ipa_rm_dep_graph_delete_dependency() - deleted dependency between
- *				two nodes in graph
- * @graph: [in] dependency graph
- * @resource_name: [in] resource to delete
- * @depends_on_name: [in] resource to delete
- * @userspace_dep: [in] operation requested by userspace ?
- *
- * Returns: 0 on success, negative on failure
- *
- */
-int ipa_rm_dep_graph_delete_dependency(struct ipa_rm_dep_graph *graph,
-				enum ipa_rm_resource_name resource_name,
-				enum ipa_rm_resource_name depends_on_name,
-				bool userspace_dep)
-{
-	struct ipa_rm_resource *dependent = NULL;
-	struct ipa_rm_resource *dependency = NULL;
-	int result;
-
-	if (!graph ||
-		!IPA_RM_RESORCE_IS_PROD(resource_name) ||
-		!IPA_RM_RESORCE_IS_CONS(depends_on_name)) {
-		IPA_RM_ERR("invalid params\n");
-		result = -EINVAL;
-		goto bail;
-	}
-
-	if (ipa_rm_dep_graph_get_resource(graph,
-					  resource_name,
-					  &dependent)) {
-		IPA_RM_DBG("%s does not exist\n",
-					ipa_rm_resource_str(resource_name));
-		result = -EINVAL;
-		goto bail;
-	}
-
-	if (ipa_rm_dep_graph_get_resource(graph,
-					  depends_on_name,
-					  &dependency)) {
-		IPA_RM_DBG("%s does not exist\n",
-					ipa_rm_resource_str(depends_on_name));
-		result = -EINVAL;
-		goto bail;
-	}
-
-	result = ipa_rm_resource_delete_dependency(dependent, dependency,
-		userspace_dep);
-bail:
-	IPA_RM_DBG("EXIT with %d\n", result);
-
-	return result;
-}

ipa/ipa_rm_dependency_graph.h

@@ -1,42 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2013-2018, The Linux Foundation. All rights reserved.
- */
-
-#ifndef _IPA_RM_DEPENDENCY_GRAPH_H_
-#define _IPA_RM_DEPENDENCY_GRAPH_H_
-
-#include <linux/list.h>
-#include <linux/ipa.h>
-#include "ipa_rm_resource.h"
-
-struct ipa_rm_dep_graph {
-	struct ipa_rm_resource *resource_table[IPA_RM_RESOURCE_MAX];
-};
-
-int ipa_rm_dep_graph_get_resource(
-				struct ipa_rm_dep_graph *graph,
-				enum ipa_rm_resource_name name,
-				struct ipa_rm_resource **resource);
-
-int ipa_rm_dep_graph_create(struct ipa_rm_dep_graph **dep_graph);
-
-void ipa_rm_dep_graph_delete(struct ipa_rm_dep_graph *graph);
-
-int ipa_rm_dep_graph_add(struct ipa_rm_dep_graph *graph,
-			 struct ipa_rm_resource *resource);
-
-int ipa_rm_dep_graph_remove(struct ipa_rm_dep_graph *graph,
-				enum ipa_rm_resource_name resource_name);
-
-int ipa_rm_dep_graph_add_dependency(struct ipa_rm_dep_graph *graph,
-				enum ipa_rm_resource_name resource_name,
-				enum ipa_rm_resource_name depends_on_name,
-				bool userspsace_dep);
-
-int ipa_rm_dep_graph_delete_dependency(struct ipa_rm_dep_graph *graph,
-				enum ipa_rm_resource_name resource_name,
-				enum ipa_rm_resource_name depends_on_name,
-				bool userspsace_dep);
-
-#endif /* _IPA_RM_DEPENDENCY_GRAPH_H_ */

ipa/ipa_rm_i.h

@@ -1,150 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2013-2018, The Linux Foundation. All rights reserved.
- */
-
-#ifndef _IPA_RM_I_H_
-#define _IPA_RM_I_H_
-
-#include <linux/workqueue.h>
-#include <linux/ipa.h>
-#include "ipa_rm_resource.h"
-#include "ipa_common_i.h"
-
-#define IPA_RM_DRV_NAME "ipa_rm"
-
-#define IPA_RM_DBG_LOW(fmt, args...) \
-	do { \
-		pr_debug(IPA_RM_DRV_NAME " %s:%d " fmt, __func__, __LINE__, \
-			## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			IPA_RM_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-#define IPA_RM_DBG(fmt, args...) \
-	do { \
-		pr_debug(IPA_RM_DRV_NAME " %s:%d " fmt, __func__, __LINE__, \
-			## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-			IPA_RM_DRV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			IPA_RM_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-#define IPA_RM_ERR(fmt, args...) \
-	do { \
-		pr_err(IPA_RM_DRV_NAME " %s:%d " fmt, __func__, __LINE__, \
-			## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-			IPA_RM_DRV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			IPA_RM_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-#define IPA_RM_RESORCE_IS_PROD(x) \
-	(x < IPA_RM_RESOURCE_MAX && (x & 0x1) == 0)
-#define IPA_RM_RESORCE_IS_CONS(x) \
-	(x < IPA_RM_RESOURCE_MAX && (x & 0x1) == 1)
-#define IPA_RM_INDEX_INVALID	(-1)
-#define IPA_RM_RELEASE_DELAY_IN_MSEC 1000
-
-int ipa_rm_prod_index(enum ipa_rm_resource_name resource_name);
-int ipa_rm_cons_index(enum ipa_rm_resource_name resource_name);
-
-/**
- * struct ipa_rm_delayed_release_work_type - IPA RM delayed resource release
- *				work type
- * @delayed_work: work struct
- * @ipa_rm_resource_name: name of the resource on which this work should be done
- * @needed_bw: bandwidth required for resource in Mbps
- * @dec_usage_count: decrease usage count on release ?
- */
-struct ipa_rm_delayed_release_work_type {
-	struct delayed_work		work;
-	enum ipa_rm_resource_name	resource_name;
-	u32				needed_bw;
-	bool				dec_usage_count;
-
-};
-
-/**
- * enum ipa_rm_wq_cmd - workqueue commands
- */
-enum ipa_rm_wq_cmd {
-	IPA_RM_WQ_NOTIFY_PROD,
-	IPA_RM_WQ_NOTIFY_CONS,
-	IPA_RM_WQ_RESOURCE_CB
-};
-
-/**
- * struct ipa_rm_wq_work_type - IPA RM worqueue specific
- *				work type
- * @work: work struct
- * @wq_cmd: command that should be processed in workqueue context
- * @resource_name: name of the resource on which this work
- *			should be done
- * @dep_graph: data structure to search for resource if exists
- * @event: event to notify
- * @notify_registered_only: notify only clients registered by
- *	ipa_rm_register()
- */
-struct ipa_rm_wq_work_type {
-	struct work_struct		work;
-	enum ipa_rm_wq_cmd		wq_cmd;
-	enum ipa_rm_resource_name	resource_name;
-	enum ipa_rm_event		event;
-	bool				notify_registered_only;
-};
-
-/**
- * struct ipa_rm_wq_suspend_resume_work_type - IPA RM worqueue resume or
- *				suspend work type
- * @work: work struct
- * @resource_name: name of the resource on which this work
- *			should be done
- * @prev_state:
- * @needed_bw:
- */
-struct ipa_rm_wq_suspend_resume_work_type {
-	struct work_struct		work;
-	enum ipa_rm_resource_name	resource_name;
-	enum ipa_rm_resource_state	prev_state;
-	u32				needed_bw;
-	bool				inc_usage_count;
-
-};
-
-int ipa_rm_wq_send_cmd(enum ipa_rm_wq_cmd wq_cmd,
-		enum ipa_rm_resource_name resource_name,
-		enum ipa_rm_event event,
-		bool notify_registered_only);
-
-int ipa_rm_wq_send_resume_cmd(enum ipa_rm_resource_name resource_name,
-		enum ipa_rm_resource_state prev_state,
-		u32 needed_bw,
-		bool inc_usage_count);
-
-int ipa_rm_wq_send_suspend_cmd(enum ipa_rm_resource_name resource_name,
-		enum ipa_rm_resource_state prev_state,
-		u32 needed_bw);
-
-int ipa_rm_initialize(void);
-
-int ipa_rm_stat(char *buf, int size);
-
-const char *ipa_rm_resource_str(enum ipa_rm_resource_name resource_name);
-
-void ipa_rm_perf_profile_change(enum ipa_rm_resource_name resource_name);
-
-int ipa_rm_request_resource_with_timer(enum ipa_rm_resource_name resource_name);
-
-void delayed_release_work_func(struct work_struct *work);
-
-int ipa_rm_add_dependency_from_ioctl(enum ipa_rm_resource_name resource_name,
-	enum ipa_rm_resource_name depends_on_name);
-
-int ipa_rm_delete_dependency_from_ioctl(enum ipa_rm_resource_name resource_name,
-	enum ipa_rm_resource_name depends_on_name);
-
-void ipa_rm_exit(void);
-
-#endif /* _IPA_RM_I_H_ */

ipa/ipa_rm_inactivity_timer.c

@@ -1,277 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2013-2018, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/jiffies.h>
-#include <linux/kernel.h>
-#include <linux/slab.h>
-#include <linux/spinlock.h>
-#include <linux/timer.h>
-#include <linux/unistd.h>
-#include <linux/workqueue.h>
-#include <linux/ipa.h>
-#include "ipa_rm_i.h"
-
-#define MAX_WS_NAME 20
-
-/**
- * struct ipa_rm_it_private - IPA RM Inactivity Timer private
- *	data
- * @initied: indicates if instance was initialized
- * @lock - spinlock for mutual exclusion
- * @resource_name - resource name
- * @work: delayed work object for running delayed releas
- *	function
- * @resource_requested: boolean flag indicates if resource was requested
- * @reschedule_work: boolean flag indicates to not release and to
- *	reschedule the release work.
- * @work_in_progress: boolean flag indicates is release work was scheduled.
- * @jiffies: number of jiffies for timeout
- *
- * WWAN private - holds all relevant info about WWAN driver
- */
-struct ipa_rm_it_private {
-	bool initied;
-	enum ipa_rm_resource_name resource_name;
-	spinlock_t lock;
-	struct delayed_work work;
-	bool resource_requested;
-	bool reschedule_work;
-	bool work_in_progress;
-	unsigned long jiffies;
-	struct wakeup_source w_lock;
-	char w_lock_name[MAX_WS_NAME];
-};
-
-static struct ipa_rm_it_private ipa_rm_it_handles[IPA_RM_RESOURCE_MAX];
-
-/**
- * ipa_rm_inactivity_timer_func() - called when timer expired in
- * the context of the shared workqueue. Checks internally if
- * reschedule_work flag is set. In case it is not set this function calls to
- * ipa_rm_release_resource(). In case reschedule_work is set this function
- * reschedule the work. This flag is cleared cleared when
- * calling to ipa_rm_inactivity_timer_release_resource().
- *
- * @work: work object provided by the work queue
- *
- * Return codes:
- * None
- */
-static void ipa_rm_inactivity_timer_func(struct work_struct *work)
-{
-
-	struct ipa_rm_it_private *me = container_of(to_delayed_work(work),
-						    struct ipa_rm_it_private,
-						    work);
-	unsigned long flags;
-
-	IPA_RM_DBG_LOW("timer expired for resource %d\n", me->resource_name);
-
-	spin_lock_irqsave(
-		&ipa_rm_it_handles[me->resource_name].lock, flags);
-	if (ipa_rm_it_handles[me->resource_name].reschedule_work) {
-		IPA_RM_DBG_LOW("setting delayed work\n");
-		ipa_rm_it_handles[me->resource_name].reschedule_work = false;
-		queue_delayed_work(system_unbound_wq,
-			&ipa_rm_it_handles[me->resource_name].work,
-			ipa_rm_it_handles[me->resource_name].jiffies);
-	} else if (ipa_rm_it_handles[me->resource_name].resource_requested) {
-		IPA_RM_DBG_LOW("not calling release\n");
-		ipa_rm_it_handles[me->resource_name].work_in_progress = false;
-	} else {
-		IPA_RM_DBG_LOW("calling release_resource on resource %d\n",
-			me->resource_name);
-		__pm_relax(&ipa_rm_it_handles[me->resource_name].w_lock);
-		ipa_rm_release_resource(me->resource_name);
-		ipa_rm_it_handles[me->resource_name].work_in_progress = false;
-	}
-	spin_unlock_irqrestore(
-		&ipa_rm_it_handles[me->resource_name].lock, flags);
-}
-
-/**
- * ipa_rm_inactivity_timer_init() - Init function for IPA RM
- * inactivity timer. This function shall be called prior calling
- * any other API of IPA RM inactivity timer.
- *
- * @resource_name: Resource name. @see ipa_rm.h
- * @msecs: time in miliseccond, that IPA RM inactivity timer
- * shall wait prior calling to ipa_rm_release_resource().
- *
- * Return codes:
- * 0: success
- * -EINVAL: invalid parameters
- */
-int ipa_rm_inactivity_timer_init(enum ipa_rm_resource_name resource_name,
-				 unsigned long msecs)
-{
-	struct wakeup_source *pwlock;
-	char *name;
-
-	IPA_RM_DBG_LOW("resource %d\n", resource_name);
-
-	if (resource_name < 0 ||
-	    resource_name >= IPA_RM_RESOURCE_MAX) {
-		IPA_RM_ERR("Invalid parameter\n");
-		return -EINVAL;
-	}
-
-	if (ipa_rm_it_handles[resource_name].initied) {
-		IPA_RM_ERR("resource %d already inited\n", resource_name);
-		return -EINVAL;
-	}
-
-	spin_lock_init(&ipa_rm_it_handles[resource_name].lock);
-	ipa_rm_it_handles[resource_name].resource_name = resource_name;
-	ipa_rm_it_handles[resource_name].jiffies = msecs_to_jiffies(msecs);
-	ipa_rm_it_handles[resource_name].resource_requested = false;
-	ipa_rm_it_handles[resource_name].reschedule_work = false;
-	ipa_rm_it_handles[resource_name].work_in_progress = false;
-	pwlock = &(ipa_rm_it_handles[resource_name].w_lock);
-	name = ipa_rm_it_handles[resource_name].w_lock_name;
-	snprintf(name, MAX_WS_NAME, "IPA_RM%d\n", resource_name);
-	wakeup_source_init(pwlock, name);
-	INIT_DELAYED_WORK(&ipa_rm_it_handles[resource_name].work,
-			  ipa_rm_inactivity_timer_func);
-	ipa_rm_it_handles[resource_name].initied = true;
-
-	return 0;
-}
-EXPORT_SYMBOL(ipa_rm_inactivity_timer_init);
-
-/**
- * ipa_rm_inactivity_timer_destroy() - De-Init function for IPA
- * RM inactivity timer.
- * @resource_name: Resource name. @see ipa_rm.h
- * Return codes:
- * 0: success
- * -EINVAL: invalid parameters
- */
-int ipa_rm_inactivity_timer_destroy(enum ipa_rm_resource_name resource_name)
-{
-	struct wakeup_source *pwlock;
-
-	IPA_RM_DBG_LOW("resource %d\n", resource_name);
-
-	if (resource_name < 0 ||
-	    resource_name >= IPA_RM_RESOURCE_MAX) {
-		IPA_RM_ERR("Invalid parameter\n");
-		return -EINVAL;
-	}
-
-	if (!ipa_rm_it_handles[resource_name].initied) {
-		IPA_RM_ERR("resource %d already inited\n",
-			resource_name);
-		return -EINVAL;
-	}
-
-	cancel_delayed_work_sync(&ipa_rm_it_handles[resource_name].work);
-	pwlock = &(ipa_rm_it_handles[resource_name].w_lock);
-	wakeup_source_trash(pwlock);
-
-	memset(&ipa_rm_it_handles[resource_name], 0,
-	       sizeof(struct ipa_rm_it_private));
-
-	return 0;
-}
-EXPORT_SYMBOL(ipa_rm_inactivity_timer_destroy);
-
-/**
- * ipa_rm_inactivity_timer_request_resource() - Same as
- * ipa_rm_request_resource(), with a difference that calling to
- * this function will also cancel the inactivity timer, if
- * ipa_rm_inactivity_timer_release_resource() was called earlier.
- *
- * @resource_name: Resource name. @see ipa_rm.h
- *
- * Return codes:
- * 0: success
- * -EINVAL: invalid parameters
- */
-int ipa_rm_inactivity_timer_request_resource(
-				enum ipa_rm_resource_name resource_name)
-{
-	int ret;
-	unsigned long flags;
-
-	IPA_RM_DBG_LOW("resource %d\n", resource_name);
-
-	if (resource_name < 0 ||
-	    resource_name >= IPA_RM_RESOURCE_MAX) {
-		IPA_RM_ERR("Invalid parameter\n");
-		return -EINVAL;
-	}
-
-	if (!ipa_rm_it_handles[resource_name].initied) {
-		IPA_RM_ERR("Not initialized\n");
-		return -EINVAL;
-	}
-
-	spin_lock_irqsave(&ipa_rm_it_handles[resource_name].lock, flags);
-	ipa_rm_it_handles[resource_name].resource_requested = true;
-	spin_unlock_irqrestore(&ipa_rm_it_handles[resource_name].lock, flags);
-	ret = ipa_rm_request_resource(resource_name);
-	IPA_RM_DBG_LOW("resource %d: returning %d\n", resource_name, ret);
-
-	return ret;
-}
-EXPORT_SYMBOL(ipa_rm_inactivity_timer_request_resource);
-
-/**
- * ipa_rm_inactivity_timer_release_resource() - Sets the
- * inactivity timer to the timeout set by
- * ipa_rm_inactivity_timer_init(). When the timeout expires, IPA
- * RM inactivity timer will call to ipa_rm_release_resource().
- * If a call to ipa_rm_inactivity_timer_request_resource() was
- * made BEFORE the timeout has expired, rge timer will be
- * cancelled.
- *
- * @resource_name: Resource name. @see ipa_rm.h
- *
- * Return codes:
- * 0: success
- * -EINVAL: invalid parameters
- */
-int ipa_rm_inactivity_timer_release_resource(
-				enum ipa_rm_resource_name resource_name)
-{
-	unsigned long flags;
-
-	IPA_RM_DBG_LOW("resource %d\n", resource_name);
-
-	if (resource_name < 0 ||
-	    resource_name >= IPA_RM_RESOURCE_MAX) {
-		IPA_RM_ERR("Invalid parameter\n");
-		return -EINVAL;
-	}
-
-	if (!ipa_rm_it_handles[resource_name].initied) {
-		IPA_RM_ERR("Not initialized\n");
-		return -EINVAL;
-	}
-
-	spin_lock_irqsave(&ipa_rm_it_handles[resource_name].lock, flags);
-	ipa_rm_it_handles[resource_name].resource_requested = false;
-	if (ipa_rm_it_handles[resource_name].work_in_progress) {
-		IPA_RM_DBG_LOW("Timer already set, no sched again %d\n",
-		    resource_name);
-		ipa_rm_it_handles[resource_name].reschedule_work = true;
-		spin_unlock_irqrestore(
-			&ipa_rm_it_handles[resource_name].lock, flags);
-		return 0;
-	}
-	ipa_rm_it_handles[resource_name].work_in_progress = true;
-	ipa_rm_it_handles[resource_name].reschedule_work = false;
-	__pm_stay_awake(&ipa_rm_it_handles[resource_name].w_lock);
-	IPA_RM_DBG_LOW("setting delayed work\n");
-	queue_delayed_work(system_unbound_wq,
-			      &ipa_rm_it_handles[resource_name].work,
-			      ipa_rm_it_handles[resource_name].jiffies);
-	spin_unlock_irqrestore(&ipa_rm_it_handles[resource_name].lock, flags);
-
-	return 0;
-}
-EXPORT_SYMBOL(ipa_rm_inactivity_timer_release_resource);
-

ipa/ipa_rm_peers_list.c

@@ -1,270 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2013-2018, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/slab.h>
-#include "ipa_rm_i.h"
-
-/**
- * ipa_rm_peers_list_get_resource_index() - resource name to index
- *	of this resource in corresponding peers list
- * @resource_name: [in] resource name
- *
- * Returns: resource index mapping, IPA_RM_INDEX_INVALID
- * in case provided resource name isn't contained in enum
- * ipa_rm_resource_name.
- *
- */
-static int ipa_rm_peers_list_get_resource_index(
-		enum ipa_rm_resource_name resource_name)
-{
-	int resource_index = IPA_RM_INDEX_INVALID;
-
-	if (IPA_RM_RESORCE_IS_PROD(resource_name))
-		resource_index = ipa_rm_prod_index(resource_name);
-	else if (IPA_RM_RESORCE_IS_CONS(resource_name))
-		resource_index = ipa_rm_cons_index(resource_name);
-
-	return resource_index;
-}
-
-static bool ipa_rm_peers_list_check_index(int index,
-		struct ipa_rm_peers_list *peers_list)
-{
-	return !(index > peers_list->max_peers || index < 0);
-}
-
-/**
- * ipa_rm_peers_list_create() - creates the peers list
- *
- * @max_peers: maximum number of peers in new list
- * @peers_list: [out] newly created peers list
- *
- * Returns: 0 in case of SUCCESS, negative otherwise
- */
-int ipa_rm_peers_list_create(int max_peers,
-		struct ipa_rm_peers_list **peers_list)
-{
-	int result;
-
-	*peers_list = kzalloc(sizeof(**peers_list), GFP_ATOMIC);
-	if (!*peers_list) {
-		IPA_RM_ERR("no mem\n");
-		result = -ENOMEM;
-		goto bail;
-	}
-
-	(*peers_list)->max_peers = max_peers;
-	(*peers_list)->peers = kzalloc((*peers_list)->max_peers *
-			sizeof(*((*peers_list)->peers)), GFP_ATOMIC);
-	if (!((*peers_list)->peers)) {
-		IPA_RM_ERR("no mem\n");
-		result = -ENOMEM;
-		goto list_alloc_fail;
-	}
-
-	return 0;
-
-list_alloc_fail:
-	kfree(*peers_list);
-bail:
-	return result;
-}
-
-/**
- * ipa_rm_peers_list_delete() - deletes the peers list
- *
- * @peers_list: peers list
- *
- */
-void ipa_rm_peers_list_delete(struct ipa_rm_peers_list *peers_list)
-{
-	if (peers_list) {
-		kfree(peers_list->peers);
-		kfree(peers_list);
-	}
-}
-
-/**
- * ipa_rm_peers_list_remove_peer() - removes peer from the list
- *
- * @peers_list: peers list
- * @resource_name: name of the resource to remove
- *
- */
-void ipa_rm_peers_list_remove_peer(
-		struct ipa_rm_peers_list *peers_list,
-		enum ipa_rm_resource_name resource_name)
-{
-	if (!peers_list)
-		return;
-
-	peers_list->peers[ipa_rm_peers_list_get_resource_index(
-			resource_name)].resource = NULL;
-	peers_list->peers[ipa_rm_peers_list_get_resource_index(
-			resource_name)].userspace_dep = false;
-	peers_list->peers_count--;
-}
-
-/**
- * ipa_rm_peers_list_add_peer() - adds peer to the list
- *
- * @peers_list: peers list
- * @resource: resource to add
- *
- */
-void ipa_rm_peers_list_add_peer(
-		struct ipa_rm_peers_list *peers_list,
-		struct ipa_rm_resource *resource,
-		bool userspace_dep)
-{
-	if (!peers_list || !resource)
-		return;
-
-	peers_list->peers[ipa_rm_peers_list_get_resource_index(
-			resource->name)].resource = resource;
-	peers_list->peers[ipa_rm_peers_list_get_resource_index(
-		resource->name)].userspace_dep = userspace_dep;
-	peers_list->peers_count++;
-}
-
-/**
- * ipa_rm_peers_list_is_empty() - checks
- *	if resource peers list is empty
- *
- * @peers_list: peers list
- *
- * Returns: true if the list is empty, false otherwise
- */
-bool ipa_rm_peers_list_is_empty(struct ipa_rm_peers_list *peers_list)
-{
-	bool result = true;
-
-	if (!peers_list)
-		goto bail;
-
-	if (peers_list->peers_count > 0)
-		result = false;
-bail:
-	return result;
-}
-
-/**
- * ipa_rm_peers_list_has_last_peer() - checks
- *	if resource peers list has exactly one peer
- *
- * @peers_list: peers list
- *
- * Returns: true if the list has exactly one peer, false otherwise
- */
-bool ipa_rm_peers_list_has_last_peer(
-		struct ipa_rm_peers_list *peers_list)
-{
-	bool result = false;
-
-	if (!peers_list)
-		goto bail;
-
-	if (peers_list->peers_count == 1)
-		result = true;
-bail:
-	return result;
-}
-
-/**
- * ipa_rm_peers_list_check_dependency() - check dependency
- *	between 2 peer lists
- * @resource_peers: first peers list
- * @resource_name: first peers list resource name
- * @depends_on_peers: second peers list
- * @depends_on_name: second peers list resource name
- * @userspace_dep: [out] dependency was created by userspace
- *
- * Returns: true if there is dependency, false otherwise
- *
- */
-bool ipa_rm_peers_list_check_dependency(
-		struct ipa_rm_peers_list *resource_peers,
-		enum ipa_rm_resource_name resource_name,
-		struct ipa_rm_peers_list *depends_on_peers,
-		enum ipa_rm_resource_name depends_on_name,
-		bool *userspace_dep)
-{
-	bool result = false;
-	int resource_index;
-	struct ipa_rm_resource_peer *peer_ptr;
-
-	if (!resource_peers || !depends_on_peers || !userspace_dep)
-		return result;
-
-	resource_index = ipa_rm_peers_list_get_resource_index(depends_on_name);
-	peer_ptr = &resource_peers->peers[resource_index];
-	if (peer_ptr->resource != NULL) {
-		result = true;
-		*userspace_dep = peer_ptr->userspace_dep;
-	}
-
-	resource_index = ipa_rm_peers_list_get_resource_index(resource_name);
-	peer_ptr = &depends_on_peers->peers[resource_index];
-	if (peer_ptr->resource != NULL) {
-		result = true;
-		*userspace_dep = peer_ptr->userspace_dep;
-	}
-
-	return result;
-}
-
-/**
- * ipa_rm_peers_list_get_resource() - get resource by
- *	resource index
- * @resource_index: resource index
- * @resource_peers: peers list
- *
- * Returns: the resource if found, NULL otherwise
- */
-struct ipa_rm_resource *ipa_rm_peers_list_get_resource(int resource_index,
-		struct ipa_rm_peers_list *resource_peers)
-{
-	struct ipa_rm_resource *result = NULL;
-
-	if (!ipa_rm_peers_list_check_index(resource_index, resource_peers))
-		goto bail;
-
-	result = resource_peers->peers[resource_index].resource;
-bail:
-	return result;
-}
-
-/**
- * ipa_rm_peers_list_get_userspace_dep() - returns whether resource dependency
- * was added by userspace
- * @resource_index: resource index
- * @resource_peers: peers list
- *
- * Returns: true if dependency was added by userspace, false by kernel
- */
-bool ipa_rm_peers_list_get_userspace_dep(int resource_index,
-		struct ipa_rm_peers_list *resource_peers)
-{
-	bool result = false;
-
-	if (!ipa_rm_peers_list_check_index(resource_index, resource_peers))
-		goto bail;
-
-	result = resource_peers->peers[resource_index].userspace_dep;
-bail:
-	return result;
-}
-
-/**
- * ipa_rm_peers_list_get_size() - get peers list sise
- *
- * @peers_list: peers list
- *
- * Returns: the size of the peers list
- */
-int ipa_rm_peers_list_get_size(struct ipa_rm_peers_list *peers_list)
-{
-	return peers_list->max_peers;
-}

ipa/ipa_rm_peers_list.h

@@ -1,55 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2013-2018, The Linux Foundation. All rights reserved.
- */
-
-#ifndef _IPA_RM_PEERS_LIST_H_
-#define _IPA_RM_PEERS_LIST_H_
-
-#include "ipa_rm_resource.h"
-
-struct ipa_rm_resource_peer {
-	struct ipa_rm_resource *resource;
-	bool userspace_dep;
-};
-
-/**
- * struct ipa_rm_peers_list - IPA RM resource peers list
- * @peers: the list of references to resources dependent on this resource
- *          in case of producer or list of dependencies in case of consumer
- * @max_peers: maximum number of peers for this resource
- * @peers_count: actual number of peers for this resource
- */
-struct ipa_rm_peers_list {
-	struct ipa_rm_resource_peer	*peers;
-	int				max_peers;
-	int				peers_count;
-};
-
-int ipa_rm_peers_list_create(int max_peers,
-		struct ipa_rm_peers_list **peers_list);
-void ipa_rm_peers_list_delete(struct ipa_rm_peers_list *peers_list);
-void ipa_rm_peers_list_remove_peer(
-		struct ipa_rm_peers_list *peers_list,
-		enum ipa_rm_resource_name resource_name);
-void ipa_rm_peers_list_add_peer(
-		struct ipa_rm_peers_list *peers_list,
-		struct ipa_rm_resource *resource,
-		bool userspace_dep);
-bool ipa_rm_peers_list_check_dependency(
-		struct ipa_rm_peers_list *resource_peers,
-		enum ipa_rm_resource_name resource_name,
-		struct ipa_rm_peers_list *depends_on_peers,
-		enum ipa_rm_resource_name depends_on_name,
-		bool *userspace_dep);
-struct ipa_rm_resource *ipa_rm_peers_list_get_resource(int resource_index,
-		struct ipa_rm_peers_list *peers_list);
-bool ipa_rm_peers_list_get_userspace_dep(int resource_index,
-		struct ipa_rm_peers_list *resource_peers);
-int ipa_rm_peers_list_get_size(struct ipa_rm_peers_list *peers_list);
-bool ipa_rm_peers_list_is_empty(struct ipa_rm_peers_list *peers_list);
-bool ipa_rm_peers_list_has_last_peer(
-		struct ipa_rm_peers_list *peers_list);
-
-
-#endif /* _IPA_RM_PEERS_LIST_H_ */

ipa/ipa_rm_resource.c

@@ -1,1204 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2013-2018, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/slab.h>
-#include "ipa_rm_resource.h"
-#include "ipa_rm_i.h"
-#include "ipa_common_i.h"
-/**
- * ipa_rm_dep_prod_index() - producer name to producer index mapping
- * @resource_name: [in] resource name (should be of producer)
- *
- * Returns: resource index mapping, IPA_RM_INDEX_INVALID
- *	in case provided resource name isn't contained
- *	in enum ipa_rm_resource_name or is not of producers.
- *
- */
-int ipa_rm_prod_index(enum ipa_rm_resource_name resource_name)
-{
-	int result = resource_name;
-
-	switch (resource_name) {
-	case IPA_RM_RESOURCE_Q6_PROD:
-	case IPA_RM_RESOURCE_USB_PROD:
-	case IPA_RM_RESOURCE_USB_DPL_DUMMY_PROD:
-	case IPA_RM_RESOURCE_HSIC_PROD:
-	case IPA_RM_RESOURCE_STD_ECM_PROD:
-	case IPA_RM_RESOURCE_RNDIS_PROD:
-	case IPA_RM_RESOURCE_WWAN_0_PROD:
-	case IPA_RM_RESOURCE_WLAN_PROD:
-	case IPA_RM_RESOURCE_ODU_ADAPT_PROD:
-	case IPA_RM_RESOURCE_MHI_PROD:
-	case IPA_RM_RESOURCE_ETHERNET_PROD:
-		break;
-	default:
-		result = IPA_RM_INDEX_INVALID;
-		break;
-	}
-
-	return result;
-}
-
-/**
- * ipa_rm_cons_index() - consumer name to consumer index mapping
- * @resource_name: [in] resource name (should be of consumer)
- *
- * Returns: resource index mapping, IPA_RM_INDEX_INVALID
- *	in case provided resource name isn't contained
- *	in enum ipa_rm_resource_name or is not of consumers.
- *
- */
-int ipa_rm_cons_index(enum ipa_rm_resource_name resource_name)
-{
-	int result = resource_name;
-
-	switch (resource_name) {
-	case IPA_RM_RESOURCE_Q6_CONS:
-	case IPA_RM_RESOURCE_USB_CONS:
-	case IPA_RM_RESOURCE_HSIC_CONS:
-	case IPA_RM_RESOURCE_WLAN_CONS:
-	case IPA_RM_RESOURCE_APPS_CONS:
-	case IPA_RM_RESOURCE_ODU_ADAPT_CONS:
-	case IPA_RM_RESOURCE_MHI_CONS:
-	case IPA_RM_RESOURCE_USB_DPL_CONS:
-	case IPA_RM_RESOURCE_ETHERNET_CONS:
-		break;
-	default:
-		result = IPA_RM_INDEX_INVALID;
-		break;
-	}
-
-	return result;
-}
-
-int ipa_rm_resource_consumer_release_work(
-		struct ipa_rm_resource_cons *consumer,
-		enum ipa_rm_resource_state prev_state,
-		bool notify_completion)
-{
-	int driver_result;
-
-	IPA_RM_DBG_LOW("calling driver CB\n");
-	driver_result = consumer->release_resource();
-	IPA_RM_DBG_LOW("driver CB returned with %d\n", driver_result);
-	/*
-	 * Treat IPA_RM_RELEASE_IN_PROGRESS as IPA_RM_RELEASED
-	 * for CONS which remains in RELEASE_IN_PROGRESS.
-	 */
-	if (driver_result == -EINPROGRESS)
-		driver_result = 0;
-	if (driver_result != 0 && driver_result != -EINPROGRESS) {
-		IPA_RM_ERR("driver CB returned error %d\n", driver_result);
-		consumer->resource.state = prev_state;
-		goto bail;
-	}
-	if (driver_result == 0) {
-		if (notify_completion)
-			ipa_rm_resource_consumer_handle_cb(consumer,
-					IPA_RM_RESOURCE_RELEASED);
-		else
-			consumer->resource.state = IPA_RM_RELEASED;
-	}
-	complete_all(&consumer->request_consumer_in_progress);
-
-	ipa_rm_perf_profile_change(consumer->resource.name);
-bail:
-	return driver_result;
-}
-
-int ipa_rm_resource_consumer_request_work(struct ipa_rm_resource_cons *consumer,
-		enum ipa_rm_resource_state prev_state,
-		u32 prod_needed_bw,
-		bool notify_completion,
-		bool dec_client_on_err)
-{
-	int driver_result;
-
-	IPA_RM_DBG_LOW("calling driver CB\n");
-	driver_result = consumer->request_resource();
-	IPA_RM_DBG_LOW("driver CB returned with %d\n", driver_result);
-	if (driver_result == 0) {
-		if (notify_completion) {
-			ipa_rm_resource_consumer_handle_cb(consumer,
-					IPA_RM_RESOURCE_GRANTED);
-		} else {
-			consumer->resource.state = IPA_RM_GRANTED;
-			ipa_rm_perf_profile_change(consumer->resource.name);
-			ipa_resume_resource(consumer->resource.name);
-		}
-	} else if (driver_result != -EINPROGRESS) {
-		consumer->resource.state = prev_state;
-		consumer->resource.needed_bw -= prod_needed_bw;
-		if (dec_client_on_err)
-			consumer->usage_count--;
-	}
-
-	return driver_result;
-}
-
-int ipa_rm_resource_consumer_request(
-		struct ipa_rm_resource_cons *consumer,
-		u32 prod_needed_bw,
-		bool inc_usage_count,
-		bool wake_client)
-{
-	int result = 0;
-	enum ipa_rm_resource_state prev_state;
-	struct ipa_active_client_logging_info log_info;
-
-	IPA_RM_DBG_LOW("%s state: %d\n",
-			ipa_rm_resource_str(consumer->resource.name),
-			consumer->resource.state);
-
-	prev_state = consumer->resource.state;
-	consumer->resource.needed_bw += prod_needed_bw;
-	switch (consumer->resource.state) {
-	case IPA_RM_RELEASED:
-	case IPA_RM_RELEASE_IN_PROGRESS:
-		reinit_completion(&consumer->request_consumer_in_progress);
-		consumer->resource.state = IPA_RM_REQUEST_IN_PROGRESS;
-		IPA_ACTIVE_CLIENTS_PREP_RESOURCE(log_info,
-				ipa_rm_resource_str(consumer->resource.name));
-		if (prev_state == IPA_RM_RELEASE_IN_PROGRESS ||
-			ipa_inc_client_enable_clks_no_block(&log_info) != 0) {
-			IPA_RM_DBG_LOW("async resume work for %s\n",
-				ipa_rm_resource_str(consumer->resource.name));
-			ipa_rm_wq_send_resume_cmd(consumer->resource.name,
-						prev_state,
-						prod_needed_bw,
-						inc_usage_count);
-			result = -EINPROGRESS;
-			break;
-		}
-		result = ipa_rm_resource_consumer_request_work(consumer,
-						prev_state,
-						prod_needed_bw,
-						false,
-						inc_usage_count);
-		break;
-	case IPA_RM_GRANTED:
-		if (wake_client) {
-			result = ipa_rm_resource_consumer_request_work(
-				consumer, prev_state, prod_needed_bw, false,
-				inc_usage_count);
-			break;
-		}
-		ipa_rm_perf_profile_change(consumer->resource.name);
-		break;
-	case IPA_RM_REQUEST_IN_PROGRESS:
-		result = -EINPROGRESS;
-		break;
-	default:
-		consumer->resource.needed_bw -= prod_needed_bw;
-		result = -EPERM;
-		goto bail;
-	}
-	if (inc_usage_count)
-		consumer->usage_count++;
-bail:
-	IPA_RM_DBG_LOW("%s new state: %d\n",
-		ipa_rm_resource_str(consumer->resource.name),
-		consumer->resource.state);
-	IPA_RM_DBG_LOW("EXIT with %d\n", result);
-
-	return result;
-}
-
-int ipa_rm_resource_consumer_release(
-		struct ipa_rm_resource_cons *consumer,
-		u32 prod_needed_bw,
-		bool dec_usage_count)
-{
-	int result = 0;
-	enum ipa_rm_resource_state save_state;
-
-	IPA_RM_DBG_LOW("%s state: %d\n",
-		ipa_rm_resource_str(consumer->resource.name),
-		consumer->resource.state);
-	save_state = consumer->resource.state;
-	consumer->resource.needed_bw -= prod_needed_bw;
-	switch (consumer->resource.state) {
-	case IPA_RM_RELEASED:
-		break;
-	case IPA_RM_GRANTED:
-	case IPA_RM_REQUEST_IN_PROGRESS:
-		if (dec_usage_count && consumer->usage_count > 0)
-			consumer->usage_count--;
-		if (consumer->usage_count == 0) {
-			consumer->resource.state = IPA_RM_RELEASE_IN_PROGRESS;
-			if (save_state == IPA_RM_REQUEST_IN_PROGRESS ||
-			    ipa_suspend_resource_no_block(
-						consumer->resource.name) != 0) {
-				ipa_rm_wq_send_suspend_cmd(
-						consumer->resource.name,
-						save_state,
-						prod_needed_bw);
-				result = -EINPROGRESS;
-				goto bail;
-			}
-			result = ipa_rm_resource_consumer_release_work(consumer,
-					save_state, false);
-			goto bail;
-		} else if (consumer->resource.state == IPA_RM_GRANTED) {
-			ipa_rm_perf_profile_change(consumer->resource.name);
-		}
-		break;
-	case IPA_RM_RELEASE_IN_PROGRESS:
-		if (dec_usage_count && consumer->usage_count > 0)
-			consumer->usage_count--;
-		result = -EINPROGRESS;
-		break;
-	default:
-		result = -EPERM;
-		goto bail;
-	}
-bail:
-	IPA_RM_DBG_LOW("%s new state: %d\n",
-		ipa_rm_resource_str(consumer->resource.name),
-		consumer->resource.state);
-	IPA_RM_DBG_LOW("EXIT with %d\n", result);
-
-	return result;
-}
-
-/**
- * ipa_rm_resource_producer_notify_clients() - notify
- *	all registered clients of given producer
- * @producer: producer
- * @event: event to notify
- * @notify_registered_only: notify only clients registered by
- *	ipa_rm_register()
- */
-void ipa_rm_resource_producer_notify_clients(
-				struct ipa_rm_resource_prod *producer,
-				enum ipa_rm_event event,
-				bool notify_registered_only)
-{
-	struct ipa_rm_notification_info *reg_info;
-
-	IPA_RM_DBG_LOW("%s event: %d notify_registered_only: %d\n",
-		ipa_rm_resource_str(producer->resource.name),
-		event,
-		notify_registered_only);
-
-	list_for_each_entry(reg_info, &(producer->event_listeners), link) {
-		if (notify_registered_only && !reg_info->explicit)
-			continue;
-
-		IPA_RM_DBG_LOW("Notifying %s event: %d\n",
-			   ipa_rm_resource_str(producer->resource.name), event);
-		reg_info->reg_params.notify_cb(reg_info->reg_params.user_data,
-					       event,
-					       0);
-		IPA_RM_DBG_LOW("back from client CB\n");
-	}
-}
-
-static int ipa_rm_resource_producer_create(struct ipa_rm_resource **resource,
-		struct ipa_rm_resource_prod **producer,
-		struct ipa_rm_create_params *create_params,
-		int *max_peers)
-{
-	int result = 0;
-
-	*producer = kzalloc(sizeof(**producer), GFP_ATOMIC);
-	if (*producer == NULL) {
-		result = -ENOMEM;
-		goto bail;
-	}
-
-	INIT_LIST_HEAD(&((*producer)->event_listeners));
-	result = ipa_rm_resource_producer_register(*producer,
-			&(create_params->reg_params),
-			false);
-	if (result) {
-		IPA_RM_ERR("ipa_rm_resource_producer_register() failed\n");
-		goto register_fail;
-	}
-
-	(*resource) = (struct ipa_rm_resource *) (*producer);
-	(*resource)->type = IPA_RM_PRODUCER;
-	*max_peers = IPA_RM_RESOURCE_MAX;
-	goto bail;
-register_fail:
-	kfree(*producer);
-bail:
-	return result;
-}
-
-static void ipa_rm_resource_producer_delete(
-				struct ipa_rm_resource_prod *producer)
-{
-	struct ipa_rm_notification_info *reg_info;
-	struct list_head *pos, *q;
-
-	ipa_rm_resource_producer_release(producer);
-	list_for_each_safe(pos, q, &(producer->event_listeners)) {
-		reg_info = list_entry(pos,
-				struct ipa_rm_notification_info,
-				link);
-		list_del(pos);
-		kfree(reg_info);
-	}
-}
-
-static int ipa_rm_resource_consumer_create(struct ipa_rm_resource **resource,
-		struct ipa_rm_resource_cons **consumer,
-		struct ipa_rm_create_params *create_params,
-		int *max_peers)
-{
-	int result = 0;
-
-	*consumer = kzalloc(sizeof(**consumer), GFP_ATOMIC);
-	if (*consumer == NULL) {
-		result = -ENOMEM;
-		goto bail;
-	}
-
-	(*consumer)->request_resource = create_params->request_resource;
-	(*consumer)->release_resource = create_params->release_resource;
-	(*resource) = (struct ipa_rm_resource *) (*consumer);
-	(*resource)->type = IPA_RM_CONSUMER;
-	init_completion(&((*consumer)->request_consumer_in_progress));
-	*max_peers = IPA_RM_RESOURCE_MAX;
-bail:
-	return result;
-}
-
-/**
- * ipa_rm_resource_create() - creates resource
- * @create_params: [in] parameters needed
- *			for resource initialization with IPA RM
- * @resource: [out] created resource
- *
- * Returns: 0 on success, negative on failure
- */
-int ipa_rm_resource_create(
-		struct ipa_rm_create_params *create_params,
-		struct ipa_rm_resource **resource)
-{
-	struct ipa_rm_resource_cons *consumer;
-	struct ipa_rm_resource_prod *producer;
-	int max_peers;
-	int result = 0;
-
-	if (!create_params) {
-		result = -EINVAL;
-		goto bail;
-	}
-
-	if (IPA_RM_RESORCE_IS_PROD(create_params->name)) {
-		result = ipa_rm_resource_producer_create(resource,
-				&producer,
-				create_params,
-				&max_peers);
-		if (result) {
-			IPA_RM_ERR("ipa_rm_resource_producer_create failed\n");
-			goto bail;
-		}
-	} else if (IPA_RM_RESORCE_IS_CONS(create_params->name)) {
-		result = ipa_rm_resource_consumer_create(resource,
-				&consumer,
-				create_params,
-				&max_peers);
-		if (result) {
-			IPA_RM_ERR("ipa_rm_resource_producer_create failed\n");
-			goto bail;
-		}
-	} else {
-		IPA_RM_ERR("invalid resource\n");
-		result = -EPERM;
-		goto bail;
-	}
-
-	result = ipa_rm_peers_list_create(max_peers,
-			&((*resource)->peers_list));
-	if (result) {
-		IPA_RM_ERR("ipa_rm_peers_list_create failed\n");
-		goto peers_alloc_fail;
-	}
-	(*resource)->name = create_params->name;
-	(*resource)->floor_voltage = create_params->floor_voltage;
-	(*resource)->state = IPA_RM_RELEASED;
-	goto bail;
-
-peers_alloc_fail:
-	ipa_rm_resource_delete(*resource);
-bail:
-	return result;
-}
-
-/**
- * ipa_rm_resource_delete() - deletes resource
- * @resource: [in] resource
- *			for resource initialization with IPA RM
- *
- * Returns: 0 on success, negative on failure
- */
-int ipa_rm_resource_delete(struct ipa_rm_resource *resource)
-{
-	struct ipa_rm_resource *consumer;
-	struct ipa_rm_resource *producer;
-	int peers_index;
-	int result = 0;
-	int list_size;
-	bool userspace_dep;
-
-	if (!resource) {
-		IPA_RM_ERR("invalid params\n");
-		return -EINVAL;
-	}
-
-	IPA_RM_DBG("ENTER with resource %d\n", resource->name);
-	if (resource->type == IPA_RM_PRODUCER) {
-		if (resource->peers_list) {
-			list_size = ipa_rm_peers_list_get_size(
-				resource->peers_list);
-			for (peers_index = 0;
-				peers_index < list_size;
-				peers_index++) {
-				consumer = ipa_rm_peers_list_get_resource(
-						peers_index,
-						resource->peers_list);
-				if (consumer) {
-					userspace_dep =
-					ipa_rm_peers_list_get_userspace_dep(
-							peers_index,
-							resource->peers_list);
-					ipa_rm_resource_delete_dependency(
-						resource,
-						consumer,
-						userspace_dep);
-				}
-			}
-		}
-
-		ipa_rm_resource_producer_delete(
-				(struct ipa_rm_resource_prod *) resource);
-	} else if (resource->type == IPA_RM_CONSUMER) {
-		if (resource->peers_list) {
-			list_size = ipa_rm_peers_list_get_size(
-				resource->peers_list);
-			for (peers_index = 0;
-					peers_index < list_size;
-					peers_index++){
-				producer = ipa_rm_peers_list_get_resource(
-							peers_index,
-							resource->peers_list);
-				if (producer) {
-					userspace_dep =
-					ipa_rm_peers_list_get_userspace_dep(
-						peers_index,
-						resource->peers_list);
-					ipa_rm_resource_delete_dependency(
-							producer,
-							resource,
-							userspace_dep);
-				}
-			}
-		}
-	}
-	ipa_rm_peers_list_delete(resource->peers_list);
-	kfree(resource);
-	return result;
-}
-
-/**
- * ipa_rm_resource_register() - register resource
- * @resource: [in] resource
- * @reg_params: [in] registration parameters
- * @explicit: [in] registered explicitly by ipa_rm_register()
- *
- * Returns: 0 on success, negative on failure
- *
- * Producer resource is expected for this call.
- *
- */
-int ipa_rm_resource_producer_register(struct ipa_rm_resource_prod *producer,
-		struct ipa_rm_register_params *reg_params,
-		bool explicit)
-{
-	int result = 0;
-	struct ipa_rm_notification_info *reg_info;
-	struct list_head *pos;
-
-	if (!producer || !reg_params) {
-		IPA_RM_ERR("invalid params\n");
-		result = -EPERM;
-		goto bail;
-	}
-
-	list_for_each(pos, &(producer->event_listeners)) {
-		reg_info = list_entry(pos,
-					struct ipa_rm_notification_info,
-					link);
-		if (reg_info->reg_params.notify_cb ==
-						reg_params->notify_cb) {
-			IPA_RM_ERR("already registered\n");
-			result = -EPERM;
-			goto bail;
-		}
-
-	}
-
-	reg_info = kzalloc(sizeof(*reg_info), GFP_ATOMIC);
-	if (reg_info == NULL) {
-		result = -ENOMEM;
-		goto bail;
-	}
-
-	reg_info->reg_params.user_data = reg_params->user_data;
-	reg_info->reg_params.notify_cb = reg_params->notify_cb;
-	reg_info->explicit = explicit;
-	INIT_LIST_HEAD(&reg_info->link);
-	list_add(&reg_info->link, &producer->event_listeners);
-bail:
-	return result;
-}
-
-/**
- * ipa_rm_resource_deregister() - register resource
- * @resource: [in] resource
- * @reg_params: [in] registration parameters
- *
- * Returns: 0 on success, negative on failure
- *
- * Producer resource is expected for this call.
- * This function deleted only single instance of
- * registration info.
- *
- */
-int ipa_rm_resource_producer_deregister(struct ipa_rm_resource_prod *producer,
-		struct ipa_rm_register_params *reg_params)
-{
-	int result = -EINVAL;
-	struct ipa_rm_notification_info *reg_info;
-	struct list_head *pos, *q;
-
-	if (!producer || !reg_params) {
-		IPA_RM_ERR("invalid params\n");
-		return -EINVAL;
-	}
-
-	list_for_each_safe(pos, q, &(producer->event_listeners)) {
-		reg_info = list_entry(pos,
-				struct ipa_rm_notification_info,
-				link);
-		if (reg_info->reg_params.notify_cb ==
-						reg_params->notify_cb) {
-			list_del(pos);
-			kfree(reg_info);
-			result = 0;
-			goto bail;
-		}
-	}
-bail:
-	return result;
-}
-
-/**
- * ipa_rm_resource_add_dependency() - add dependency between two
- *				given resources
- * @resource: [in] resource resource
- * @depends_on: [in] depends_on resource
- *
- * Returns: 0 on success, negative on failure
- */
-int ipa_rm_resource_add_dependency(struct ipa_rm_resource *resource,
-				   struct ipa_rm_resource *depends_on,
-				   bool userspace_dep)
-{
-	int result = 0;
-	int consumer_result;
-	bool add_dep_by_userspace;
-
-	if (!resource || !depends_on) {
-		IPA_RM_ERR("invalid params\n");
-		return -EINVAL;
-	}
-
-	if (ipa_rm_peers_list_check_dependency(resource->peers_list,
-			resource->name,
-			depends_on->peers_list,
-			depends_on->name,
-			&add_dep_by_userspace)) {
-		IPA_RM_ERR("dependency already exists, added by %s\n",
-			add_dep_by_userspace ? "userspace" : "kernel");
-		return -EEXIST;
-	}
-
-	ipa_rm_peers_list_add_peer(resource->peers_list, depends_on,
-		userspace_dep);
-	ipa_rm_peers_list_add_peer(depends_on->peers_list, resource,
-		userspace_dep);
-	IPA_RM_DBG("%s state: %d\n", ipa_rm_resource_str(resource->name),
-				resource->state);
-
-	resource->needed_bw += depends_on->max_bw;
-	switch (resource->state) {
-	case IPA_RM_RELEASED:
-	case IPA_RM_RELEASE_IN_PROGRESS:
-		break;
-	case IPA_RM_GRANTED:
-	case IPA_RM_REQUEST_IN_PROGRESS:
-	{
-		enum ipa_rm_resource_state prev_state = resource->state;
-
-		resource->state = IPA_RM_REQUEST_IN_PROGRESS;
-		((struct ipa_rm_resource_prod *)
-					resource)->pending_request++;
-		consumer_result = ipa_rm_resource_consumer_request(
-				(struct ipa_rm_resource_cons *)depends_on,
-				resource->max_bw,
-				true, false);
-		if (consumer_result != -EINPROGRESS) {
-			resource->state = prev_state;
-			((struct ipa_rm_resource_prod *)
-					resource)->pending_request--;
-			ipa_rm_perf_profile_change(resource->name);
-		}
-		result = consumer_result;
-		break;
-	}
-	default:
-		IPA_RM_ERR("invalid state\n");
-		result = -EPERM;
-		goto bail;
-	}
-bail:
-	IPA_RM_DBG("%s new state: %d\n", ipa_rm_resource_str(resource->name),
-					resource->state);
-	IPA_RM_DBG("EXIT with %d\n", result);
-
-	return result;
-}
-
-/**
- * ipa_rm_resource_delete_dependency() - add dependency between two
- *				given resources
- * @resource: [in] resource resource
- * @depends_on: [in] depends_on resource
- *
- * Returns: 0 on success, negative on failure
- * In case the resource state was changed, a notification
- * will be sent to the RM client
- */
-int ipa_rm_resource_delete_dependency(struct ipa_rm_resource *resource,
-				   struct ipa_rm_resource *depends_on,
-				   bool userspace_dep)
-{
-	int result = 0;
-	bool state_changed = false;
-	bool release_consumer = false;
-	enum ipa_rm_event evt;
-	bool add_dep_by_userspace;
-
-	if (!resource || !depends_on) {
-		IPA_RM_ERR("invalid params\n");
-		return -EINVAL;
-	}
-
-	if (!ipa_rm_peers_list_check_dependency(resource->peers_list,
-			resource->name,
-			depends_on->peers_list,
-			depends_on->name,
-			&add_dep_by_userspace)) {
-		IPA_RM_ERR("dependency does not exist\n");
-		return -EINVAL;
-	}
-
-	/*
-	 * to avoid race conditions between kernel and userspace
-	 * need to check that the dependency was added by same entity
-	 */
-	if (add_dep_by_userspace != userspace_dep) {
-		IPA_RM_DBG("dependency was added by %s\n",
-			add_dep_by_userspace ? "userspace" : "kernel");
-		IPA_RM_DBG("ignore request to delete dependency by %s\n",
-			userspace_dep ? "userspace" : "kernel");
-		return 0;
-	}
-
-	IPA_RM_DBG("%s state: %d\n", ipa_rm_resource_str(resource->name),
-				resource->state);
-
-	resource->needed_bw -= depends_on->max_bw;
-	switch (resource->state) {
-	case IPA_RM_RELEASED:
-		break;
-	case IPA_RM_GRANTED:
-		ipa_rm_perf_profile_change(resource->name);
-		release_consumer = true;
-		break;
-	case IPA_RM_RELEASE_IN_PROGRESS:
-		if (((struct ipa_rm_resource_prod *)
-			resource)->pending_release > 0)
-			((struct ipa_rm_resource_prod *)
-				resource)->pending_release--;
-		if (depends_on->state == IPA_RM_RELEASE_IN_PROGRESS &&
-			((struct ipa_rm_resource_prod *)
-			resource)->pending_release == 0) {
-			resource->state = IPA_RM_RELEASED;
-			state_changed = true;
-			evt = IPA_RM_RESOURCE_RELEASED;
-			ipa_rm_perf_profile_change(resource->name);
-		}
-		break;
-	case IPA_RM_REQUEST_IN_PROGRESS:
-		release_consumer = true;
-		if (((struct ipa_rm_resource_prod *)
-			resource)->pending_request > 0)
-			((struct ipa_rm_resource_prod *)
-				resource)->pending_request--;
-		if (depends_on->state == IPA_RM_REQUEST_IN_PROGRESS &&
-			((struct ipa_rm_resource_prod *)
-				resource)->pending_request == 0) {
-			resource->state = IPA_RM_GRANTED;
-			state_changed = true;
-			evt = IPA_RM_RESOURCE_GRANTED;
-			ipa_rm_perf_profile_change(resource->name);
-		}
-		break;
-	default:
-		result = -EINVAL;
-		goto bail;
-	}
-	if (state_changed) {
-		(void) ipa_rm_wq_send_cmd(IPA_RM_WQ_NOTIFY_PROD,
-				resource->name,
-				evt,
-				false);
-	}
-	IPA_RM_DBG("%s new state: %d\n", ipa_rm_resource_str(resource->name),
-					resource->state);
-	ipa_rm_peers_list_remove_peer(resource->peers_list,
-			depends_on->name);
-	ipa_rm_peers_list_remove_peer(depends_on->peers_list,
-			resource->name);
-	if (release_consumer)
-		(void) ipa_rm_resource_consumer_release(
-				(struct ipa_rm_resource_cons *)depends_on,
-				resource->max_bw,
-				true);
-bail:
-	IPA_RM_DBG("EXIT with %d\n", result);
-
-	return result;
-}
-
-/**
- * ipa_rm_resource_producer_request() - producer resource request
- * @producer: [in] producer
- *
- * Returns: 0 on success, negative on failure
- */
-int ipa_rm_resource_producer_request(struct ipa_rm_resource_prod *producer)
-{
-	int peers_index;
-	int result = 0;
-	struct ipa_rm_resource *consumer;
-	int consumer_result;
-	enum ipa_rm_resource_state state;
-
-	state = producer->resource.state;
-	switch (producer->resource.state) {
-	case IPA_RM_RELEASED:
-	case IPA_RM_RELEASE_IN_PROGRESS:
-		producer->resource.state = IPA_RM_REQUEST_IN_PROGRESS;
-		break;
-	case IPA_RM_GRANTED:
-		goto unlock_and_bail;
-	case IPA_RM_REQUEST_IN_PROGRESS:
-		result = -EINPROGRESS;
-		goto unlock_and_bail;
-	default:
-		result = -EINVAL;
-		goto unlock_and_bail;
-	}
-
-	producer->pending_request = 0;
-	for (peers_index = 0;
-		peers_index < ipa_rm_peers_list_get_size(
-				producer->resource.peers_list);
-		peers_index++) {
-		consumer = ipa_rm_peers_list_get_resource(peers_index,
-				producer->resource.peers_list);
-		if (consumer) {
-			producer->pending_request++;
-			consumer_result = ipa_rm_resource_consumer_request(
-				(struct ipa_rm_resource_cons *)consumer,
-				producer->resource.max_bw,
-				true, false);
-			if (consumer_result == -EINPROGRESS) {
-				result = -EINPROGRESS;
-			} else {
-				producer->pending_request--;
-				if (consumer_result != 0) {
-					result = consumer_result;
-					goto bail;
-				}
-			}
-		}
-	}
-
-	if (producer->pending_request == 0) {
-		producer->resource.state = IPA_RM_GRANTED;
-		ipa_rm_perf_profile_change(producer->resource.name);
-		(void) ipa_rm_wq_send_cmd(IPA_RM_WQ_NOTIFY_PROD,
-			producer->resource.name,
-			IPA_RM_RESOURCE_GRANTED,
-			true);
-		result = 0;
-	}
-unlock_and_bail:
-	if (state != producer->resource.state)
-		IPA_RM_DBG_LOW("%s state changed %d->%d\n",
-			ipa_rm_resource_str(producer->resource.name),
-			state,
-			producer->resource.state);
-bail:
-	return result;
-}
-
-/**
- * ipa_rm_resource_producer_release() - producer resource release
- * producer: [in] producer resource
- *
- * Returns: 0 on success, negative on failure
- *
- */
-int ipa_rm_resource_producer_release(struct ipa_rm_resource_prod *producer)
-{
-	int peers_index;
-	int result = 0;
-	struct ipa_rm_resource *consumer;
-	int consumer_result;
-	enum ipa_rm_resource_state state;
-
-	state = producer->resource.state;
-	switch (producer->resource.state) {
-	case IPA_RM_RELEASED:
-		goto bail;
-	case IPA_RM_GRANTED:
-	case IPA_RM_REQUEST_IN_PROGRESS:
-		producer->resource.state = IPA_RM_RELEASE_IN_PROGRESS;
-		break;
-	case IPA_RM_RELEASE_IN_PROGRESS:
-		result = -EINPROGRESS;
-		goto bail;
-	default:
-		result = -EPERM;
-		goto bail;
-	}
-
-	producer->pending_release = 0;
-	for (peers_index = 0;
-		peers_index < ipa_rm_peers_list_get_size(
-				producer->resource.peers_list);
-		peers_index++) {
-		consumer = ipa_rm_peers_list_get_resource(peers_index,
-				producer->resource.peers_list);
-		if (consumer) {
-			producer->pending_release++;
-			consumer_result = ipa_rm_resource_consumer_release(
-				(struct ipa_rm_resource_cons *)consumer,
-				producer->resource.max_bw,
-				true);
-			producer->pending_release--;
-		}
-	}
-
-	if (producer->pending_release == 0) {
-		producer->resource.state = IPA_RM_RELEASED;
-		ipa_rm_perf_profile_change(producer->resource.name);
-		(void) ipa_rm_wq_send_cmd(IPA_RM_WQ_NOTIFY_PROD,
-			producer->resource.name,
-			IPA_RM_RESOURCE_RELEASED,
-			true);
-	}
-bail:
-	if (state != producer->resource.state)
-		IPA_RM_DBG_LOW("%s state changed %d->%d\n",
-		ipa_rm_resource_str(producer->resource.name),
-		state,
-		producer->resource.state);
-
-	return result;
-}
-
-static void ipa_rm_resource_producer_handle_cb(
-		struct ipa_rm_resource_prod *producer,
-		enum ipa_rm_event event)
-{
-	IPA_RM_DBG_LOW("%s state: %d event: %d pending_request: %d\n",
-		ipa_rm_resource_str(producer->resource.name),
-		producer->resource.state,
-		event,
-		producer->pending_request);
-
-	switch (producer->resource.state) {
-	case IPA_RM_REQUEST_IN_PROGRESS:
-		if (event != IPA_RM_RESOURCE_GRANTED)
-			goto unlock_and_bail;
-		if (producer->pending_request > 0) {
-			producer->pending_request--;
-			if (producer->pending_request == 0) {
-				producer->resource.state =
-						IPA_RM_GRANTED;
-				ipa_rm_perf_profile_change(
-					producer->resource.name);
-				ipa_rm_resource_producer_notify_clients(
-						producer,
-						IPA_RM_RESOURCE_GRANTED,
-						false);
-				goto bail;
-			}
-		}
-		break;
-	case IPA_RM_RELEASE_IN_PROGRESS:
-		if (event != IPA_RM_RESOURCE_RELEASED)
-			goto unlock_and_bail;
-		if (producer->pending_release > 0) {
-			producer->pending_release--;
-			if (producer->pending_release == 0) {
-				producer->resource.state =
-						IPA_RM_RELEASED;
-				ipa_rm_perf_profile_change(
-					producer->resource.name);
-				ipa_rm_resource_producer_notify_clients(
-						producer,
-						IPA_RM_RESOURCE_RELEASED,
-						false);
-				goto bail;
-			}
-		}
-		break;
-	case IPA_RM_GRANTED:
-	case IPA_RM_RELEASED:
-	default:
-		goto unlock_and_bail;
-	}
-unlock_and_bail:
-	IPA_RM_DBG_LOW("%s new state: %d\n",
-		ipa_rm_resource_str(producer->resource.name),
-		producer->resource.state);
-bail:
-	return;
-}
-
-/**
- * ipa_rm_resource_consumer_handle_cb() - propagates resource
- *	notification to all dependent producers
- * @consumer: [in] notifying resource
- *
- */
-void ipa_rm_resource_consumer_handle_cb(struct ipa_rm_resource_cons *consumer,
-				enum ipa_rm_event event)
-{
-	int peers_index;
-	struct ipa_rm_resource *producer;
-
-	if (!consumer) {
-		IPA_RM_ERR("invalid params\n");
-		return;
-	}
-	IPA_RM_DBG_LOW("%s state: %d event: %d\n",
-		ipa_rm_resource_str(consumer->resource.name),
-		consumer->resource.state,
-		event);
-
-	switch (consumer->resource.state) {
-	case IPA_RM_REQUEST_IN_PROGRESS:
-		if (event == IPA_RM_RESOURCE_RELEASED)
-			goto bail;
-		consumer->resource.state = IPA_RM_GRANTED;
-		ipa_rm_perf_profile_change(consumer->resource.name);
-		ipa_resume_resource(consumer->resource.name);
-		complete_all(&consumer->request_consumer_in_progress);
-		break;
-	case IPA_RM_RELEASE_IN_PROGRESS:
-		if (event == IPA_RM_RESOURCE_GRANTED)
-			goto bail;
-		consumer->resource.state = IPA_RM_RELEASED;
-		break;
-	case IPA_RM_GRANTED:
-	case IPA_RM_RELEASED:
-	default:
-		goto bail;
-	}
-
-	for (peers_index = 0;
-		peers_index < ipa_rm_peers_list_get_size(
-				consumer->resource.peers_list);
-		peers_index++) {
-		producer = ipa_rm_peers_list_get_resource(peers_index,
-				consumer->resource.peers_list);
-		if (producer)
-			ipa_rm_resource_producer_handle_cb(
-					(struct ipa_rm_resource_prod *)
-						producer,
-						event);
-	}
-
-	return;
-bail:
-	IPA_RM_DBG_LOW("%s new state: %d\n",
-		ipa_rm_resource_str(consumer->resource.name),
-		consumer->resource.state);
-}
-
-/*
- * ipa_rm_resource_set_perf_profile() - sets the performance profile to
- *					resource.
- *
- * @resource: [in] resource
- * @profile: [in] profile to be set
- *
- * sets the profile to the given resource, In case the resource is
- * granted, update bandwidth vote of the resource
- */
-int ipa_rm_resource_set_perf_profile(struct ipa_rm_resource *resource,
-				     struct ipa_rm_perf_profile *profile)
-{
-	int peers_index;
-	struct ipa_rm_resource *peer;
-
-	if (!resource || !profile) {
-		IPA_RM_ERR("invalid params\n");
-		return -EINVAL;
-	}
-
-	if (profile->max_supported_bandwidth_mbps == resource->max_bw) {
-		IPA_RM_DBG_LOW("same profile\n");
-		return 0;
-	}
-
-	if ((resource->type == IPA_RM_PRODUCER &&
-	    (resource->state == IPA_RM_GRANTED ||
-	    resource->state == IPA_RM_REQUEST_IN_PROGRESS)) ||
-	    resource->type == IPA_RM_CONSUMER) {
-		for (peers_index = 0;
-		     peers_index < ipa_rm_peers_list_get_size(
-		     resource->peers_list);
-		     peers_index++) {
-			peer = ipa_rm_peers_list_get_resource(peers_index,
-				resource->peers_list);
-			if (!peer)
-				continue;
-			peer->needed_bw -= resource->max_bw;
-			peer->needed_bw +=
-				profile->max_supported_bandwidth_mbps;
-			if (peer->state == IPA_RM_GRANTED)
-				ipa_rm_perf_profile_change(peer->name);
-		}
-	}
-
-	resource->max_bw = profile->max_supported_bandwidth_mbps;
-	if (resource->state == IPA_RM_GRANTED)
-		ipa_rm_perf_profile_change(resource->name);
-
-	return 0;
-}
-
-
-/*
- * ipa_rm_resource_producer_print_stat() - print the
- * resource status and all his dependencies
- *
- * @resource: [in] Resource resource
- * @buff: [in] The buf used to print
- * @size: [in] Buf size
- *
- * Returns: number of bytes used on success, negative on failure
- */
-int ipa_rm_resource_producer_print_stat(
-				struct ipa_rm_resource *resource,
-				char *buf,
-				int size)
-{
-
-	int i;
-	int nbytes;
-	int cnt = 0;
-	struct ipa_rm_resource *consumer;
-
-	if (!buf || size < 0)
-		return -EINVAL;
-
-	nbytes = scnprintf(buf + cnt, size - cnt,
-		ipa_rm_resource_str(resource->name));
-	cnt += nbytes;
-	nbytes = scnprintf(buf + cnt, size - cnt, "[%d, ", resource->max_bw);
-	cnt += nbytes;
-
-	switch (resource->state) {
-	case IPA_RM_RELEASED:
-		nbytes = scnprintf(buf + cnt, size - cnt,
-			"Released] -> ");
-		cnt += nbytes;
-		break;
-	case IPA_RM_REQUEST_IN_PROGRESS:
-		nbytes = scnprintf(buf + cnt, size - cnt,
-			"Request In Progress] -> ");
-		cnt += nbytes;
-		break;
-	case IPA_RM_GRANTED:
-		nbytes = scnprintf(buf + cnt, size - cnt,
-			"Granted] -> ");
-		cnt += nbytes;
-		break;
-	case IPA_RM_RELEASE_IN_PROGRESS:
-		nbytes = scnprintf(buf + cnt, size - cnt,
-			"Release In Progress] -> ");
-		cnt += nbytes;
-		break;
-	default:
-		return -EPERM;
-	}
-
-	for (i = 0; i < resource->peers_list->max_peers; ++i) {
-		consumer =
-			ipa_rm_peers_list_get_resource(
-			i,
-			resource->peers_list);
-		if (consumer) {
-			nbytes = scnprintf(buf + cnt, size - cnt,
-				ipa_rm_resource_str(consumer->name));
-			cnt += nbytes;
-			nbytes = scnprintf(buf + cnt, size - cnt, "[%d, ",
-				consumer->max_bw);
-			cnt += nbytes;
-
-			switch (consumer->state) {
-			case IPA_RM_RELEASED:
-				nbytes = scnprintf(buf + cnt, size - cnt,
-					"Released], ");
-				cnt += nbytes;
-				break;
-			case IPA_RM_REQUEST_IN_PROGRESS:
-				nbytes = scnprintf(buf + cnt, size - cnt,
-						"Request In Progress], ");
-				cnt += nbytes;
-					break;
-			case IPA_RM_GRANTED:
-				nbytes = scnprintf(buf + cnt, size - cnt,
-						"Granted], ");
-				cnt += nbytes;
-				break;
-			case IPA_RM_RELEASE_IN_PROGRESS:
-				nbytes = scnprintf(buf + cnt, size - cnt,
-						"Release In Progress], ");
-				cnt += nbytes;
-				break;
-			default:
-				return -EPERM;
-			}
-		}
-	}
-	nbytes = scnprintf(buf + cnt, size - cnt, "\n");
-	cnt += nbytes;
-
-	return cnt;
-}

ipa/ipa_rm_resource.h

@@ -1,159 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2013-2018, The Linux Foundation. All rights reserved.
- */
-
-#ifndef _IPA_RM_RESOURCE_H_
-#define _IPA_RM_RESOURCE_H_
-
-#include <linux/list.h>
-#include <linux/ipa.h>
-#include "ipa_rm_peers_list.h"
-
-/**
- * enum ipa_rm_resource_state - resource state
- */
-enum ipa_rm_resource_state {
-	IPA_RM_RELEASED,
-	IPA_RM_REQUEST_IN_PROGRESS,
-	IPA_RM_GRANTED,
-	IPA_RM_RELEASE_IN_PROGRESS
-};
-
-/**
- * enum ipa_rm_resource_type - IPA resource manager resource type
- */
-enum ipa_rm_resource_type {
-	IPA_RM_PRODUCER,
-	IPA_RM_CONSUMER
-};
-
-/**
- * struct ipa_rm_notification_info - notification information
- *				of IPA RM client
- * @reg_params: registration parameters
- * @explicit: registered explicitly by ipa_rm_register()
- * @link: link to the list of all registered clients information
- */
-struct ipa_rm_notification_info {
-	struct ipa_rm_register_params	reg_params;
-	bool				explicit;
-	struct list_head		link;
-};
-
-/**
- * struct ipa_rm_resource - IPA RM resource
- * @name: name identifying resource
- * @type: type of resource (PRODUCER or CONSUMER)
- * @floor_voltage: minimum voltage level for operation
- * @max_bw: maximum bandwidth required for resource in Mbps
- * @state: state of the resource
- * @peers_list: list of the peers of the resource
- */
-struct ipa_rm_resource {
-	enum ipa_rm_resource_name	name;
-	enum ipa_rm_resource_type	type;
-	enum ipa_voltage_level		floor_voltage;
-	u32				max_bw;
-	u32				needed_bw;
-	enum ipa_rm_resource_state	state;
-	struct ipa_rm_peers_list	*peers_list;
-};
-
-/**
- * struct ipa_rm_resource_cons - IPA RM consumer
- * @resource: resource
- * @usage_count: number of producers in GRANTED / REQUESTED state
- *		using this consumer
- * @request_consumer_in_progress: when set, the consumer is during its request
- *		phase
- * @request_resource: function which should be called to request resource
- *			from resource manager
- * @release_resource: function which should be called to release resource
- *			from resource manager
- * Add new fields after @resource only.
- */
-struct ipa_rm_resource_cons {
-	struct ipa_rm_resource resource;
-	int usage_count;
-	struct completion request_consumer_in_progress;
-	int (*request_resource)(void);
-	int (*release_resource)(void);
-};
-
-/**
- * struct ipa_rm_resource_prod - IPA RM producer
- * @resource: resource
- * @event_listeners: clients registered with this producer
- *		for notifications in resource state
- * list Add new fields after @resource only.
- */
-struct ipa_rm_resource_prod {
-	struct ipa_rm_resource	resource;
-	struct list_head	event_listeners;
-	int			pending_request;
-	int			pending_release;
-};
-
-int ipa_rm_resource_create(
-		struct ipa_rm_create_params *create_params,
-		struct ipa_rm_resource **resource);
-
-int ipa_rm_resource_delete(struct ipa_rm_resource *resource);
-
-int ipa_rm_resource_producer_register(struct ipa_rm_resource_prod *producer,
-				struct ipa_rm_register_params *reg_params,
-				bool explicit);
-
-int ipa_rm_resource_producer_deregister(struct ipa_rm_resource_prod *producer,
-				struct ipa_rm_register_params *reg_params);
-
-int ipa_rm_resource_add_dependency(struct ipa_rm_resource *resource,
-				   struct ipa_rm_resource *depends_on,
-				   bool userspace_dep);
-
-int ipa_rm_resource_delete_dependency(struct ipa_rm_resource *resource,
-				      struct ipa_rm_resource *depends_on,
-				      bool userspace_dep);
-
-int ipa_rm_resource_producer_request(struct ipa_rm_resource_prod *producer);
-
-int ipa_rm_resource_producer_release(struct ipa_rm_resource_prod *producer);
-
-int ipa_rm_resource_consumer_request(struct ipa_rm_resource_cons *consumer,
-				u32 needed_bw,
-				bool inc_usage_count,
-				bool wake_client);
-
-int ipa_rm_resource_consumer_release(struct ipa_rm_resource_cons *consumer,
-				u32 needed_bw,
-				bool dec_usage_count);
-
-int ipa_rm_resource_set_perf_profile(struct ipa_rm_resource *resource,
-				     struct ipa_rm_perf_profile *profile);
-
-void ipa_rm_resource_consumer_handle_cb(struct ipa_rm_resource_cons *consumer,
-				enum ipa_rm_event event);
-
-void ipa_rm_resource_producer_notify_clients(
-				struct ipa_rm_resource_prod *producer,
-				enum ipa_rm_event event,
-				bool notify_registered_only);
-
-int ipa_rm_resource_producer_print_stat(
-		struct ipa_rm_resource *resource,
-		char *buf,
-		int size);
-
-int ipa_rm_resource_consumer_request_work(struct ipa_rm_resource_cons *consumer,
-		enum ipa_rm_resource_state prev_state,
-		u32 needed_bw,
-		bool notify_completion,
-		bool dec_client_on_err);
-
-int ipa_rm_resource_consumer_release_work(
-		struct ipa_rm_resource_cons *consumer,
-		enum ipa_rm_resource_state prev_state,
-		bool notify_completion);
-
-#endif /* _IPA_RM_RESOURCE_H_ */

ipa/ipa_uc_offload_common_i.h

@@ -1,22 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/ipa_mhi.h>
-#include <linux/ipa_qmi_service_v01.h>
-
-#ifndef _IPA_UC_OFFLOAD_COMMON_I_H_
-#define _IPA_UC_OFFLOAD_COMMON_I_H_
-
-int ipa_setup_uc_ntn_pipes(struct ipa_ntn_conn_in_params *in,
-	ipa_notify_cb notify, void *priv, u8 hdr_len,
-	struct ipa_ntn_conn_out_params *outp);
-
-int ipa_tear_down_uc_offload_pipes(int ipa_ep_idx_ul, int ipa_ep_idx_dl,
-	struct ipa_ntn_conn_in_params *params);
-
-int ipa_ntn_uc_reg_rdyCB(void (*ipauc_ready_cb)(void *user_data),
-			      void *user_data);
-void ipa_ntn_uc_dereg_rdyCB(void);
-#endif /* _IPA_UC_OFFLOAD_COMMON_I_H_ */

ipa/ipa_v3/Makefile

@@ -1,23 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-
-obj-$(CONFIG_IPA3) += ipahal/
-
-obj-$(CONFIG_IPA3) += ipat.o
-ipat-y := ipa.o ipa_debugfs.o ipa_hdr.o ipa_flt.o ipa_rt.o ipa_dp.o ipa_client.o \
-	ipa_utils.o ipa_nat.o ipa_intf.o teth_bridge.o ipa_interrupts.o \
-	ipa_uc.o ipa_uc_wdi.o ipa_dma.o ipa_uc_mhi.o ipa_mhi.o ipa_uc_ntn.o \
-	ipa_hw_stats.o ipa_pm.o ipa_wdi3_i.o ipa_odl.o ipa_wigig_i.o
-
-ipat-$(CONFIG_IPA_EMULATION) += ipa_dt_replacement.o
-
-obj-$(CONFIG_RMNET_IPA3) += rmnet_ipa.o ipa_qmi_service_v01.o ipa_qmi_service.o rmnet_ipa_fd_ioctl.o
-
-obj-$(CONFIG_IPA3_MHI_PROXY) += ipa_mhi_proxy.o
-
-obj-$(CONFIG_IPA3_MHI_PRIME_MANAGER) += ipa_mpm.o
-
-ipat-$(CONFIG_IPA3_REGDUMP) += dump/ipa_reg_dump.o
-
-ccflags-$(CONFIG_IPA3_REGDUMP) += -Itechpack/dataipa/ipa/ipa_v3/dump
-
-ccflags-$(CONFIG_IPA3_REGDUMP_IPA_4_5) += -Itechpack/dataipa/ipa/ipa_v3/dump/ipa4.5

ipa/ipa_v3/dump/ipa4.5/gsi_hwio.h

@@ -1,2392 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2019, The Linux Foundation. All rights reserved.
- */
-#if !defined(_GSI_HWIO_H_)
-#define _GSI_HWIO_H_
-/*
- *
- * HWIO register definitions to follow:
- *
- */
-#define GSI_REG_BASE (IPA_0_IPA_WRAPPER_BASE + 0x00004000)
-#define GSI_REG_BASE_PHYS (IPA_0_IPA_WRAPPER_BASE_PHYS + 0x00004000)
-#define GSI_REG_BASE_OFFS 0x00004000
-#define HWIO_GSI_CFG_ADDR (GSI_REG_BASE + 0x00000000)
-#define HWIO_GSI_CFG_PHYS (GSI_REG_BASE_PHYS + 0x00000000)
-#define HWIO_GSI_CFG_OFFS (GSI_REG_BASE_OFFS + 0x00000000)
-#define HWIO_GSI_CFG_RMSK 0xf3f
-#define HWIO_GSI_CFG_ATTR 0x3
-#define HWIO_GSI_CFG_IN in_dword_masked(HWIO_GSI_CFG_ADDR, \
-					HWIO_GSI_CFG_RMSK)
-#define HWIO_GSI_CFG_INM(m) in_dword_masked(HWIO_GSI_CFG_ADDR, m)
-#define HWIO_GSI_CFG_OUT(v) out_dword(HWIO_GSI_CFG_ADDR, v)
-#define HWIO_GSI_CFG_OUTM(m, v) out_dword_masked_ns(HWIO_GSI_CFG_ADDR, \
-						    m, \
-						    v, \
-						    HWIO_GSI_CFG_IN)
-#define HWIO_GSI_CFG_SLEEP_CLK_DIV_BMSK 0xf00
-#define HWIO_GSI_CFG_SLEEP_CLK_DIV_SHFT 0x8
-#define HWIO_GSI_CFG_BP_MTRIX_DISABLE_BMSK 0x20
-#define HWIO_GSI_CFG_BP_MTRIX_DISABLE_SHFT 0x5
-#define HWIO_GSI_CFG_GSI_PWR_CLPS_BMSK 0x10
-#define HWIO_GSI_CFG_GSI_PWR_CLPS_SHFT 0x4
-#define HWIO_GSI_CFG_UC_IS_MCS_BMSK 0x8
-#define HWIO_GSI_CFG_UC_IS_MCS_SHFT 0x3
-#define HWIO_GSI_CFG_DOUBLE_MCS_CLK_FREQ_BMSK 0x4
-#define HWIO_GSI_CFG_DOUBLE_MCS_CLK_FREQ_SHFT 0x2
-#define HWIO_GSI_CFG_MCS_ENABLE_BMSK 0x2
-#define HWIO_GSI_CFG_MCS_ENABLE_SHFT 0x1
-#define HWIO_GSI_CFG_GSI_ENABLE_BMSK 0x1
-#define HWIO_GSI_CFG_GSI_ENABLE_SHFT 0x0
-#define HWIO_GSI_MANAGER_MCS_CODE_VER_ADDR (GSI_REG_BASE + 0x00000008)
-#define HWIO_GSI_MANAGER_MCS_CODE_VER_PHYS (GSI_REG_BASE_PHYS + 0x00000008)
-#define HWIO_GSI_MANAGER_MCS_CODE_VER_OFFS (GSI_REG_BASE_OFFS + 0x00000008)
-#define HWIO_GSI_ZEROS_ADDR (GSI_REG_BASE + 0x00000010)
-#define HWIO_GSI_ZEROS_PHYS (GSI_REG_BASE_PHYS + 0x00000010)
-#define HWIO_GSI_ZEROS_OFFS (GSI_REG_BASE_OFFS + 0x00000010)
-#define HWIO_GSI_PERIPH_BASE_ADDR_LSB_ADDR (GSI_REG_BASE + 0x00000018)
-#define HWIO_GSI_PERIPH_BASE_ADDR_LSB_PHYS (GSI_REG_BASE_PHYS + 0x00000018)
-#define HWIO_GSI_PERIPH_BASE_ADDR_LSB_OFFS (GSI_REG_BASE_OFFS + 0x00000018)
-#define HWIO_GSI_PERIPH_BASE_ADDR_MSB_ADDR (GSI_REG_BASE + 0x0000001c)
-#define HWIO_GSI_PERIPH_BASE_ADDR_MSB_PHYS (GSI_REG_BASE_PHYS + 0x0000001c)
-#define HWIO_GSI_PERIPH_BASE_ADDR_MSB_OFFS (GSI_REG_BASE_OFFS + 0x0000001c)
-#define HWIO_GSI_PERIPH_PENDING_ADDR (GSI_REG_BASE + 0x00000020)
-#define HWIO_GSI_PERIPH_PENDING_PHYS (GSI_REG_BASE_PHYS + 0x00000020)
-#define HWIO_GSI_PERIPH_PENDING_OFFS (GSI_REG_BASE_OFFS + 0x00000020)
-#define HWIO_GSI_MOQA_CFG_ADDR (GSI_REG_BASE + 0x00000030)
-#define HWIO_GSI_MOQA_CFG_PHYS (GSI_REG_BASE_PHYS + 0x00000030)
-#define HWIO_GSI_MOQA_CFG_OFFS (GSI_REG_BASE_OFFS + 0x00000030)
-#define HWIO_GSI_REE_CFG_ADDR (GSI_REG_BASE + 0x00000038)
-#define HWIO_GSI_REE_CFG_PHYS (GSI_REG_BASE_PHYS + 0x00000038)
-#define HWIO_GSI_REE_CFG_OFFS (GSI_REG_BASE_OFFS + 0x00000038)
-#define HWIO_GSI_REE_CFG_RMSK 0xff03
-#define HWIO_GSI_REE_CFG_ATTR 0x3
-#define HWIO_GSI_REE_CFG_IN in_dword_masked(HWIO_GSI_REE_CFG_ADDR, \
-					    HWIO_GSI_REE_CFG_RMSK)
-#define HWIO_GSI_REE_CFG_INM(m) in_dword_masked(HWIO_GSI_REE_CFG_ADDR, m)
-#define HWIO_GSI_REE_CFG_OUT(v) out_dword(HWIO_GSI_REE_CFG_ADDR, v)
-#define HWIO_GSI_REE_CFG_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_GSI_REE_CFG_ADDR, \
-		m, \
-		v, \
-		HWIO_GSI_REE_CFG_IN)
-#define HWIO_GSI_REE_CFG_MAX_BURST_SIZE_BMSK 0xff00
-#define HWIO_GSI_REE_CFG_MAX_BURST_SIZE_SHFT 0x8
-#define HWIO_GSI_REE_CFG_CHANNEL_EMPTY_INT_ENABLE_BMSK 0x2
-#define HWIO_GSI_REE_CFG_CHANNEL_EMPTY_INT_ENABLE_SHFT 0x1
-#define HWIO_GSI_REE_CFG_MOVE_TO_ESC_CLR_MODE_TRSH_BMSK 0x1
-#define HWIO_GSI_REE_CFG_MOVE_TO_ESC_CLR_MODE_TRSH_SHFT 0x0
-#define HWIO_GSI_CGC_CTRL_ADDR (GSI_REG_BASE + 0x00000060)
-#define HWIO_GSI_CGC_CTRL_PHYS (GSI_REG_BASE_PHYS + 0x00000060)
-#define HWIO_GSI_CGC_CTRL_OFFS (GSI_REG_BASE_OFFS + 0x00000060)
-#define HWIO_GSI_MSI_CACHEATTR_ADDR (GSI_REG_BASE + 0x00000080)
-#define HWIO_GSI_MSI_CACHEATTR_PHYS (GSI_REG_BASE_PHYS + 0x00000080)
-#define HWIO_GSI_MSI_CACHEATTR_OFFS (GSI_REG_BASE_OFFS + 0x00000080)
-#define HWIO_GSI_EVENT_CACHEATTR_ADDR (GSI_REG_BASE + 0x00000084)
-#define HWIO_GSI_EVENT_CACHEATTR_PHYS (GSI_REG_BASE_PHYS + 0x00000084)
-#define HWIO_GSI_EVENT_CACHEATTR_OFFS (GSI_REG_BASE_OFFS + 0x00000084)
-#define HWIO_GSI_DATA_CACHEATTR_ADDR (GSI_REG_BASE + 0x00000088)
-#define HWIO_GSI_DATA_CACHEATTR_PHYS (GSI_REG_BASE_PHYS + 0x00000088)
-#define HWIO_GSI_DATA_CACHEATTR_OFFS (GSI_REG_BASE_OFFS + 0x00000088)
-#define HWIO_GSI_TRE_CACHEATTR_ADDR (GSI_REG_BASE + 0x00000090)
-#define HWIO_GSI_TRE_CACHEATTR_PHYS (GSI_REG_BASE_PHYS + 0x00000090)
-#define HWIO_GSI_TRE_CACHEATTR_OFFS (GSI_REG_BASE_OFFS + 0x00000090)
-#define HWIO_IC_DISABLE_CHNL_BCK_PRS_LSB_ADDR (GSI_REG_BASE + 0x000000a0)
-#define HWIO_IC_DISABLE_CHNL_BCK_PRS_LSB_PHYS (GSI_REG_BASE_PHYS + \
-					       0x000000a0)
-#define HWIO_IC_DISABLE_CHNL_BCK_PRS_LSB_OFFS (GSI_REG_BASE_OFFS + \
-					       0x000000a0)
-#define HWIO_IC_DISABLE_CHNL_BCK_PRS_MSB_ADDR (GSI_REG_BASE + 0x000000a4)
-#define HWIO_IC_DISABLE_CHNL_BCK_PRS_MSB_PHYS (GSI_REG_BASE_PHYS + \
-					       0x000000a4)
-#define HWIO_IC_DISABLE_CHNL_BCK_PRS_MSB_OFFS (GSI_REG_BASE_OFFS + \
-					       0x000000a4)
-#define HWIO_IC_GEN_EVNT_BCK_PRS_LSB_ADDR (GSI_REG_BASE + 0x000000a8)
-#define HWIO_IC_GEN_EVNT_BCK_PRS_LSB_PHYS (GSI_REG_BASE_PHYS + 0x000000a8)
-#define HWIO_IC_GEN_EVNT_BCK_PRS_LSB_OFFS (GSI_REG_BASE_OFFS + 0x000000a8)
-#define HWIO_IC_GEN_EVNT_BCK_PRS_MSB_ADDR (GSI_REG_BASE + 0x000000ac)
-#define HWIO_IC_GEN_EVNT_BCK_PRS_MSB_PHYS (GSI_REG_BASE_PHYS + 0x000000ac)
-#define HWIO_IC_GEN_EVNT_BCK_PRS_MSB_OFFS (GSI_REG_BASE_OFFS + 0x000000ac)
-#define HWIO_IC_GEN_INT_BCK_PRS_LSB_ADDR (GSI_REG_BASE + 0x000000b0)
-#define HWIO_IC_GEN_INT_BCK_PRS_LSB_PHYS (GSI_REG_BASE_PHYS + 0x000000b0)
-#define HWIO_IC_GEN_INT_BCK_PRS_LSB_OFFS (GSI_REG_BASE_OFFS + 0x000000b0)
-#define HWIO_IC_GEN_INT_BCK_PRS_MSB_ADDR (GSI_REG_BASE + 0x000000b4)
-#define HWIO_IC_GEN_INT_BCK_PRS_MSB_PHYS (GSI_REG_BASE_PHYS + 0x000000b4)
-#define HWIO_IC_GEN_INT_BCK_PRS_MSB_OFFS (GSI_REG_BASE_OFFS + 0x000000b4)
-#define HWIO_IC_STOP_INT_MOD_BCK_PRS_LSB_ADDR (GSI_REG_BASE + 0x000000b8)
-#define HWIO_IC_STOP_INT_MOD_BCK_PRS_LSB_PHYS (GSI_REG_BASE_PHYS + \
-					       0x000000b8)
-#define HWIO_IC_STOP_INT_MOD_BCK_PRS_LSB_OFFS (GSI_REG_BASE_OFFS + \
-					       0x000000b8)
-#define HWIO_IC_STOP_INT_MOD_BCK_PRS_MSB_ADDR (GSI_REG_BASE + 0x000000bc)
-#define HWIO_IC_STOP_INT_MOD_BCK_PRS_MSB_PHYS (GSI_REG_BASE_PHYS + \
-					       0x000000bc)
-#define HWIO_IC_STOP_INT_MOD_BCK_PRS_MSB_OFFS (GSI_REG_BASE_OFFS + \
-					       0x000000bc)
-#define HWIO_IC_PROCESS_DESC_BCK_PRS_LSB_ADDR (GSI_REG_BASE + 0x000000c0)
-#define HWIO_IC_PROCESS_DESC_BCK_PRS_LSB_PHYS (GSI_REG_BASE_PHYS + \
-					       0x000000c0)
-#define HWIO_IC_PROCESS_DESC_BCK_PRS_LSB_OFFS (GSI_REG_BASE_OFFS + \
-					       0x000000c0)
-#define HWIO_IC_PROCESS_DESC_BCK_PRS_MSB_ADDR (GSI_REG_BASE + 0x000000c4)
-#define HWIO_IC_PROCESS_DESC_BCK_PRS_MSB_PHYS (GSI_REG_BASE_PHYS + \
-					       0x000000c4)
-#define HWIO_IC_PROCESS_DESC_BCK_PRS_MSB_OFFS (GSI_REG_BASE_OFFS + \
-					       0x000000c4)
-#define HWIO_IC_TLV_STOP_BCK_PRS_LSB_ADDR (GSI_REG_BASE + 0x000000c8)
-#define HWIO_IC_TLV_STOP_BCK_PRS_LSB_PHYS (GSI_REG_BASE_PHYS + 0x000000c8)
-#define HWIO_IC_TLV_STOP_BCK_PRS_LSB_OFFS (GSI_REG_BASE_OFFS + 0x000000c8)
-#define HWIO_IC_TLV_STOP_BCK_PRS_MSB_ADDR (GSI_REG_BASE + 0x000000cc)
-#define HWIO_IC_TLV_STOP_BCK_PRS_MSB_PHYS (GSI_REG_BASE_PHYS + 0x000000cc)
-#define HWIO_IC_TLV_STOP_BCK_PRS_MSB_OFFS (GSI_REG_BASE_OFFS + 0x000000cc)
-#define HWIO_IC_TLV_RESET_BCK_PRS_LSB_ADDR (GSI_REG_BASE + 0x000000d0)
-#define HWIO_IC_TLV_RESET_BCK_PRS_LSB_PHYS (GSI_REG_BASE_PHYS + 0x000000d0)
-#define HWIO_IC_TLV_RESET_BCK_PRS_LSB_OFFS (GSI_REG_BASE_OFFS + 0x000000d0)
-#define HWIO_IC_TLV_RESET_BCK_PRS_MSB_ADDR (GSI_REG_BASE + 0x000000d4)
-#define HWIO_IC_TLV_RESET_BCK_PRS_MSB_PHYS (GSI_REG_BASE_PHYS + 0x000000d4)
-#define HWIO_IC_TLV_RESET_BCK_PRS_MSB_OFFS (GSI_REG_BASE_OFFS + 0x000000d4)
-#define HWIO_IC_RGSTR_TIMER_BCK_PRS_LSB_ADDR (GSI_REG_BASE + 0x000000d8)
-#define HWIO_IC_RGSTR_TIMER_BCK_PRS_LSB_PHYS (GSI_REG_BASE_PHYS + \
-					      0x000000d8)
-#define HWIO_IC_RGSTR_TIMER_BCK_PRS_LSB_OFFS (GSI_REG_BASE_OFFS + \
-					      0x000000d8)
-#define HWIO_IC_RGSTR_TIMER_BCK_PRS_MSB_ADDR (GSI_REG_BASE + 0x000000dc)
-#define HWIO_IC_RGSTR_TIMER_BCK_PRS_MSB_PHYS (GSI_REG_BASE_PHYS + \
-					      0x000000dc)
-#define HWIO_IC_RGSTR_TIMER_BCK_PRS_MSB_OFFS (GSI_REG_BASE_OFFS + \
-					      0x000000dc)
-#define HWIO_IC_READ_BCK_PRS_LSB_ADDR (GSI_REG_BASE + 0x000000e0)
-#define HWIO_IC_READ_BCK_PRS_LSB_PHYS (GSI_REG_BASE_PHYS + 0x000000e0)
-#define HWIO_IC_READ_BCK_PRS_LSB_OFFS (GSI_REG_BASE_OFFS + 0x000000e0)
-#define HWIO_IC_READ_BCK_PRS_MSB_ADDR (GSI_REG_BASE + 0x000000e4)
-#define HWIO_IC_READ_BCK_PRS_MSB_PHYS (GSI_REG_BASE_PHYS + 0x000000e4)
-#define HWIO_IC_READ_BCK_PRS_MSB_OFFS (GSI_REG_BASE_OFFS + 0x000000e4)
-#define HWIO_IC_WRITE_BCK_PRS_LSB_ADDR (GSI_REG_BASE + 0x000000e8)
-#define HWIO_IC_WRITE_BCK_PRS_LSB_PHYS (GSI_REG_BASE_PHYS + 0x000000e8)
-#define HWIO_IC_WRITE_BCK_PRS_LSB_OFFS (GSI_REG_BASE_OFFS + 0x000000e8)
-#define HWIO_IC_WRITE_BCK_PRS_MSB_ADDR (GSI_REG_BASE + 0x000000ec)
-#define HWIO_IC_WRITE_BCK_PRS_MSB_PHYS (GSI_REG_BASE_PHYS + 0x000000ec)
-#define HWIO_IC_WRITE_BCK_PRS_MSB_OFFS (GSI_REG_BASE_OFFS + 0x000000ec)
-#define HWIO_IC_UCONTROLLER_GPR_BCK_PRS_LSB_ADDR (GSI_REG_BASE + \
-						  0x000000f0)
-#define HWIO_IC_UCONTROLLER_GPR_BCK_PRS_LSB_PHYS (GSI_REG_BASE_PHYS + \
-						  0x000000f0)
-#define HWIO_IC_UCONTROLLER_GPR_BCK_PRS_LSB_OFFS (GSI_REG_BASE_OFFS + \
-						  0x000000f0)
-#define HWIO_IC_UCONTROLLER_GPR_BCK_PRS_MSB_ADDR (GSI_REG_BASE + \
-						  0x000000f4)
-#define HWIO_IC_UCONTROLLER_GPR_BCK_PRS_MSB_PHYS (GSI_REG_BASE_PHYS + \
-						  0x000000f4)
-#define HWIO_IC_UCONTROLLER_GPR_BCK_PRS_MSB_OFFS (GSI_REG_BASE_OFFS + \
-						  0x000000f4)
-#define HWIO_IC_INT_WEIGHT_REE_ADDR (GSI_REG_BASE + 0x00000100)
-#define HWIO_IC_INT_WEIGHT_REE_PHYS (GSI_REG_BASE_PHYS + 0x00000100)
-#define HWIO_IC_INT_WEIGHT_REE_OFFS (GSI_REG_BASE_OFFS + 0x00000100)
-#define HWIO_IC_INT_WEIGHT_EVT_ENG_ADDR (GSI_REG_BASE + 0x00000104)
-#define HWIO_IC_INT_WEIGHT_EVT_ENG_PHYS (GSI_REG_BASE_PHYS + 0x00000104)
-#define HWIO_IC_INT_WEIGHT_EVT_ENG_OFFS (GSI_REG_BASE_OFFS + 0x00000104)
-#define HWIO_IC_INT_WEIGHT_INT_ENG_ADDR (GSI_REG_BASE + 0x00000108)
-#define HWIO_IC_INT_WEIGHT_INT_ENG_PHYS (GSI_REG_BASE_PHYS + 0x00000108)
-#define HWIO_IC_INT_WEIGHT_INT_ENG_OFFS (GSI_REG_BASE_OFFS + 0x00000108)
-#define HWIO_IC_INT_WEIGHT_CSR_ADDR (GSI_REG_BASE + 0x0000010c)
-#define HWIO_IC_INT_WEIGHT_CSR_PHYS (GSI_REG_BASE_PHYS + 0x0000010c)
-#define HWIO_IC_INT_WEIGHT_CSR_OFFS (GSI_REG_BASE_OFFS + 0x0000010c)
-#define HWIO_IC_INT_WEIGHT_TLV_ENG_ADDR (GSI_REG_BASE + 0x00000110)
-#define HWIO_IC_INT_WEIGHT_TLV_ENG_PHYS (GSI_REG_BASE_PHYS + 0x00000110)
-#define HWIO_IC_INT_WEIGHT_TLV_ENG_OFFS (GSI_REG_BASE_OFFS + 0x00000110)
-#define HWIO_IC_INT_WEIGHT_TIMER_ENG_ADDR (GSI_REG_BASE + 0x00000114)
-#define HWIO_IC_INT_WEIGHT_TIMER_ENG_PHYS (GSI_REG_BASE_PHYS + 0x00000114)
-#define HWIO_IC_INT_WEIGHT_TIMER_ENG_OFFS (GSI_REG_BASE_OFFS + 0x00000114)
-#define HWIO_IC_INT_WEIGHT_DB_ENG_ADDR (GSI_REG_BASE + 0x00000118)
-#define HWIO_IC_INT_WEIGHT_DB_ENG_PHYS (GSI_REG_BASE_PHYS + 0x00000118)
-#define HWIO_IC_INT_WEIGHT_DB_ENG_OFFS (GSI_REG_BASE_OFFS + 0x00000118)
-#define HWIO_IC_INT_WEIGHT_RD_WR_ENG_ADDR (GSI_REG_BASE + 0x0000011c)
-#define HWIO_IC_INT_WEIGHT_RD_WR_ENG_PHYS (GSI_REG_BASE_PHYS + 0x0000011c)
-#define HWIO_IC_INT_WEIGHT_RD_WR_ENG_OFFS (GSI_REG_BASE_OFFS + 0x0000011c)
-#define HWIO_IC_INT_WEIGHT_UCONTROLLER_ENG_ADDR (GSI_REG_BASE + 0x00000120)
-#define HWIO_IC_INT_WEIGHT_UCONTROLLER_ENG_PHYS (GSI_REG_BASE_PHYS + \
-						 0x00000120)
-#define HWIO_IC_INT_WEIGHT_UCONTROLLER_ENG_OFFS (GSI_REG_BASE_OFFS + \
-						 0x00000120)
-#define HWIO_IC_INT_WEIGHT_SDMA_ADDR (GSI_REG_BASE + 0x00000124)
-#define HWIO_IC_INT_WEIGHT_SDMA_PHYS (GSI_REG_BASE_PHYS + 0x00000124)
-#define HWIO_IC_INT_WEIGHT_SDMA_OFFS (GSI_REG_BASE_OFFS + 0x00000124)
-#define HWIO_GSI_SDMA_CFG_ADDR (GSI_REG_BASE + 0x0000003c)
-#define HWIO_GSI_SDMA_CFG_PHYS (GSI_REG_BASE_PHYS + 0x0000003c)
-#define HWIO_GSI_SDMA_CFG_OFFS (GSI_REG_BASE_OFFS + 0x0000003c)
-#define HWIO_GSI_SDMA_CACHEATTR_ADDR (GSI_REG_BASE + 0x00000094)
-#define HWIO_GSI_SDMA_CACHEATTR_PHYS (GSI_REG_BASE_PHYS + 0x00000094)
-#define HWIO_GSI_SDMA_CACHEATTR_OFFS (GSI_REG_BASE_OFFS + 0x00000094)
-#define HWIO_GSI_SDMA_SG_IOVEC_LSB_n_ADDR(n) (GSI_REG_BASE + 0x00000140 + \
-					      0x8 * (n))
-#define HWIO_GSI_SDMA_SG_IOVEC_LSB_n_PHYS(n) (GSI_REG_BASE_PHYS + \
-					      0x00000140 + 0x8 * (n))
-#define HWIO_GSI_SDMA_SG_IOVEC_LSB_n_OFFS(n) (GSI_REG_BASE_OFFS + \
-					      0x00000140 + 0x8 * (n))
-#define HWIO_GSI_SDMA_SG_IOVEC_MSB_n_ADDR(n) (GSI_REG_BASE + 0x00000144 + \
-					      0x8 * (n))
-#define HWIO_GSI_SDMA_SG_IOVEC_MSB_n_PHYS(n) (GSI_REG_BASE_PHYS + \
-					      0x00000144 + 0x8 * (n))
-#define HWIO_GSI_SDMA_SG_IOVEC_MSB_n_OFFS(n) (GSI_REG_BASE_OFFS + \
-					      0x00000144 + 0x8 * (n))
-#define HWIO_GSI_MANAGER_EE_QOS_n_ADDR(n) (GSI_REG_BASE + 0x00000300 + \
-					   0x4 * (n))
-#define HWIO_GSI_MANAGER_EE_QOS_n_PHYS(n) (GSI_REG_BASE_PHYS + \
-					   0x00000300 + 0x4 * (n))
-#define HWIO_GSI_MANAGER_EE_QOS_n_OFFS(n) (GSI_REG_BASE_OFFS + \
-					   0x00000300 + 0x4 * (n))
-#define HWIO_GSI_SHRAM_PTR_CH_CNTXT_BASE_ADDR_ADDR (GSI_REG_BASE + \
-						    0x00000200)
-#define HWIO_GSI_SHRAM_PTR_CH_CNTXT_BASE_ADDR_PHYS (GSI_REG_BASE_PHYS +	\
-						    0x00000200)
-#define HWIO_GSI_SHRAM_PTR_CH_CNTXT_BASE_ADDR_OFFS (GSI_REG_BASE_OFFS +	\
-						    0x00000200)
-#define HWIO_GSI_SHRAM_PTR_EV_CNTXT_BASE_ADDR_ADDR (GSI_REG_BASE + \
-						    0x00000204)
-#define HWIO_GSI_SHRAM_PTR_EV_CNTXT_BASE_ADDR_PHYS (GSI_REG_BASE_PHYS +	\
-						    0x00000204)
-#define HWIO_GSI_SHRAM_PTR_EV_CNTXT_BASE_ADDR_OFFS (GSI_REG_BASE_OFFS +	\
-						    0x00000204)
-#define HWIO_GSI_SHRAM_PTR_RE_STORAGE_BASE_ADDR_ADDR (GSI_REG_BASE + \
-						      0x00000208)
-#define HWIO_GSI_SHRAM_PTR_RE_STORAGE_BASE_ADDR_PHYS (GSI_REG_BASE_PHYS + \
-						      0x00000208)
-#define HWIO_GSI_SHRAM_PTR_RE_STORAGE_BASE_ADDR_OFFS (GSI_REG_BASE_OFFS + \
-						      0x00000208)
-#define HWIO_GSI_SHRAM_PTR_RE_ESC_BUF_BASE_ADDR_ADDR (GSI_REG_BASE + \
-						      0x0000020c)
-#define HWIO_GSI_SHRAM_PTR_RE_ESC_BUF_BASE_ADDR_PHYS (GSI_REG_BASE_PHYS + \
-						      0x0000020c)
-#define HWIO_GSI_SHRAM_PTR_RE_ESC_BUF_BASE_ADDR_OFFS (GSI_REG_BASE_OFFS + \
-						      0x0000020c)
-#define HWIO_GSI_SHRAM_PTR_EE_SCRACH_BASE_ADDR_ADDR (GSI_REG_BASE + \
-						     0x00000240)
-#define HWIO_GSI_SHRAM_PTR_EE_SCRACH_BASE_ADDR_PHYS (GSI_REG_BASE_PHYS + \
-						     0x00000240)
-#define HWIO_GSI_SHRAM_PTR_EE_SCRACH_BASE_ADDR_OFFS (GSI_REG_BASE_OFFS + \
-						     0x00000240)
-#define HWIO_GSI_SHRAM_PTR_FUNC_STACK_BASE_ADDR_ADDR (GSI_REG_BASE + \
-						      0x00000244)
-#define HWIO_GSI_SHRAM_PTR_FUNC_STACK_BASE_ADDR_PHYS (GSI_REG_BASE_PHYS + \
-						      0x00000244)
-#define HWIO_GSI_SHRAM_PTR_FUNC_STACK_BASE_ADDR_OFFS (GSI_REG_BASE_OFFS + \
-						      0x00000244)
-#define HWIO_GSI_SHRAM_PTR_MCS_SCRATCH_BASE_ADDR_ADDR (GSI_REG_BASE + \
-						       0x00000248)
-#define HWIO_GSI_SHRAM_PTR_MCS_SCRATCH_BASE_ADDR_PHYS (GSI_REG_BASE_PHYS + \
-						       0x00000248)
-#define HWIO_GSI_SHRAM_PTR_MCS_SCRATCH_BASE_ADDR_OFFS (GSI_REG_BASE_OFFS + \
-						       0x00000248)
-#define HWIO_GSI_SHRAM_PTR_MCS_SCRATCH1_BASE_ADDR_ADDR (GSI_REG_BASE + \
-							0x0000024c)
-#define HWIO_GSI_SHRAM_PTR_MCS_SCRATCH1_BASE_ADDR_PHYS (GSI_REG_BASE_PHYS \
-							+ 0x0000024c)
-#define HWIO_GSI_SHRAM_PTR_MCS_SCRATCH1_BASE_ADDR_OFFS (GSI_REG_BASE_OFFS \
-							+ 0x0000024c)
-#define HWIO_GSI_SHRAM_PTR_MCS_SCRATCH2_BASE_ADDR_ADDR (GSI_REG_BASE + \
-							0x00000250)
-#define HWIO_GSI_SHRAM_PTR_MCS_SCRATCH2_BASE_ADDR_PHYS (GSI_REG_BASE_PHYS \
-							+ 0x00000250)
-#define HWIO_GSI_SHRAM_PTR_MCS_SCRATCH2_BASE_ADDR_OFFS (GSI_REG_BASE_OFFS \
-							+ 0x00000250)
-#define HWIO_GSI_SHRAM_PTR_CH_VP_TRANS_TABLE_BASE_ADDR_ADDR (GSI_REG_BASE \
-							     + 0x00000254)
-#define HWIO_GSI_SHRAM_PTR_CH_VP_TRANS_TABLE_BASE_ADDR_PHYS ( \
-		GSI_REG_BASE_PHYS + 0x00000254)
-#define HWIO_GSI_SHRAM_PTR_CH_VP_TRANS_TABLE_BASE_ADDR_OFFS ( \
-		GSI_REG_BASE_OFFS + 0x00000254)
-#define HWIO_GSI_SHRAM_PTR_EV_VP_TRANS_TABLE_BASE_ADDR_ADDR (GSI_REG_BASE \
-							     + 0x00000258)
-#define HWIO_GSI_SHRAM_PTR_EV_VP_TRANS_TABLE_BASE_ADDR_PHYS ( \
-		GSI_REG_BASE_PHYS + 0x00000258)
-#define HWIO_GSI_SHRAM_PTR_EV_VP_TRANS_TABLE_BASE_ADDR_OFFS ( \
-		GSI_REG_BASE_OFFS + 0x00000258)
-#define HWIO_GSI_SHRAM_PTR_USER_INFO_DATA_BASE_ADDR_ADDR (GSI_REG_BASE + \
-							  0x0000025c)
-#define HWIO_GSI_SHRAM_PTR_USER_INFO_DATA_BASE_ADDR_PHYS ( \
-		GSI_REG_BASE_PHYS + 0x0000025c)
-#define HWIO_GSI_SHRAM_PTR_USER_INFO_DATA_BASE_ADDR_OFFS ( \
-		GSI_REG_BASE_OFFS + 0x0000025c)
-#define HWIO_GSI_SHRAM_PTR_EE_CMD_FIFO_BASE_ADDR_ADDR (GSI_REG_BASE + \
-						       0x00000260)
-#define HWIO_GSI_SHRAM_PTR_EE_CMD_FIFO_BASE_ADDR_PHYS (GSI_REG_BASE_PHYS + \
-						       0x00000260)
-#define HWIO_GSI_SHRAM_PTR_EE_CMD_FIFO_BASE_ADDR_OFFS (GSI_REG_BASE_OFFS + \
-						       0x00000260)
-#define HWIO_GSI_SHRAM_PTR_CH_CMD_FIFO_BASE_ADDR_ADDR (GSI_REG_BASE + \
-						       0x00000264)
-#define HWIO_GSI_SHRAM_PTR_CH_CMD_FIFO_BASE_ADDR_PHYS (GSI_REG_BASE_PHYS + \
-						       0x00000264)
-#define HWIO_GSI_SHRAM_PTR_CH_CMD_FIFO_BASE_ADDR_OFFS (GSI_REG_BASE_OFFS + \
-						       0x00000264)
-#define HWIO_GSI_IRAM_PTR_CH_CMD_ADDR (GSI_REG_BASE + 0x00000400)
-#define HWIO_GSI_IRAM_PTR_CH_CMD_PHYS (GSI_REG_BASE_PHYS + 0x00000400)
-#define HWIO_GSI_IRAM_PTR_CH_CMD_OFFS (GSI_REG_BASE_OFFS + 0x00000400)
-#define HWIO_GSI_IRAM_PTR_EE_GENERIC_CMD_ADDR (GSI_REG_BASE + 0x00000404)
-#define HWIO_GSI_IRAM_PTR_EE_GENERIC_CMD_PHYS (GSI_REG_BASE_PHYS + \
-					       0x00000404)
-#define HWIO_GSI_IRAM_PTR_EE_GENERIC_CMD_OFFS (GSI_REG_BASE_OFFS + \
-					       0x00000404)
-#define HWIO_GSI_IRAM_PTR_TLV_CH_NOT_FULL_ADDR (GSI_REG_BASE + 0x00000408)
-#define HWIO_GSI_IRAM_PTR_TLV_CH_NOT_FULL_PHYS (GSI_REG_BASE_PHYS + \
-						0x00000408)
-#define HWIO_GSI_IRAM_PTR_TLV_CH_NOT_FULL_OFFS (GSI_REG_BASE_OFFS + \
-						0x00000408)
-#define HWIO_GSI_IRAM_PTR_CH_DB_ADDR (GSI_REG_BASE + 0x00000418)
-#define HWIO_GSI_IRAM_PTR_CH_DB_PHYS (GSI_REG_BASE_PHYS + 0x00000418)
-#define HWIO_GSI_IRAM_PTR_CH_DB_OFFS (GSI_REG_BASE_OFFS + 0x00000418)
-#define HWIO_GSI_IRAM_PTR_EV_DB_ADDR (GSI_REG_BASE + 0x0000041c)
-#define HWIO_GSI_IRAM_PTR_EV_DB_PHYS (GSI_REG_BASE_PHYS + 0x0000041c)
-#define HWIO_GSI_IRAM_PTR_EV_DB_OFFS (GSI_REG_BASE_OFFS + 0x0000041c)
-#define HWIO_GSI_IRAM_PTR_NEW_RE_ADDR (GSI_REG_BASE + 0x00000420)
-#define HWIO_GSI_IRAM_PTR_NEW_RE_PHYS (GSI_REG_BASE_PHYS + 0x00000420)
-#define HWIO_GSI_IRAM_PTR_NEW_RE_OFFS (GSI_REG_BASE_OFFS + 0x00000420)
-#define HWIO_GSI_IRAM_PTR_CH_DIS_COMP_ADDR (GSI_REG_BASE + 0x00000424)
-#define HWIO_GSI_IRAM_PTR_CH_DIS_COMP_PHYS (GSI_REG_BASE_PHYS + 0x00000424)
-#define HWIO_GSI_IRAM_PTR_CH_DIS_COMP_OFFS (GSI_REG_BASE_OFFS + 0x00000424)
-#define HWIO_GSI_IRAM_PTR_CH_EMPTY_ADDR (GSI_REG_BASE + 0x00000428)
-#define HWIO_GSI_IRAM_PTR_CH_EMPTY_PHYS (GSI_REG_BASE_PHYS + 0x00000428)
-#define HWIO_GSI_IRAM_PTR_CH_EMPTY_OFFS (GSI_REG_BASE_OFFS + 0x00000428)
-#define HWIO_GSI_IRAM_PTR_EVENT_GEN_COMP_ADDR (GSI_REG_BASE + 0x0000042c)
-#define HWIO_GSI_IRAM_PTR_EVENT_GEN_COMP_PHYS (GSI_REG_BASE_PHYS + \
-					       0x0000042c)
-#define HWIO_GSI_IRAM_PTR_EVENT_GEN_COMP_OFFS (GSI_REG_BASE_OFFS + \
-					       0x0000042c)
-#define HWIO_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_0_ADDR (GSI_REG_BASE + \
-						   0x00000430)
-#define HWIO_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_0_PHYS (GSI_REG_BASE_PHYS + \
-						   0x00000430)
-#define HWIO_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_0_OFFS (GSI_REG_BASE_OFFS + \
-						   0x00000430)
-#define HWIO_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_2_ADDR (GSI_REG_BASE + \
-						   0x00000434)
-#define HWIO_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_2_PHYS (GSI_REG_BASE_PHYS + \
-						   0x00000434)
-#define HWIO_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_2_OFFS (GSI_REG_BASE_OFFS + \
-						   0x00000434)
-#define HWIO_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_1_ADDR (GSI_REG_BASE + \
-						   0x00000438)
-#define HWIO_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_1_PHYS (GSI_REG_BASE_PHYS + \
-						   0x00000438)
-#define HWIO_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_1_OFFS (GSI_REG_BASE_OFFS + \
-						   0x00000438)
-#define HWIO_GSI_IRAM_PTR_TIMER_EXPIRED_ADDR (GSI_REG_BASE + 0x0000043c)
-#define HWIO_GSI_IRAM_PTR_TIMER_EXPIRED_PHYS (GSI_REG_BASE_PHYS + \
-					      0x0000043c)
-#define HWIO_GSI_IRAM_PTR_TIMER_EXPIRED_OFFS (GSI_REG_BASE_OFFS + \
-					      0x0000043c)
-#define HWIO_GSI_IRAM_PTR_WRITE_ENG_COMP_ADDR (GSI_REG_BASE + 0x00000440)
-#define HWIO_GSI_IRAM_PTR_WRITE_ENG_COMP_PHYS (GSI_REG_BASE_PHYS + \
-					       0x00000440)
-#define HWIO_GSI_IRAM_PTR_WRITE_ENG_COMP_OFFS (GSI_REG_BASE_OFFS + \
-					       0x00000440)
-#define HWIO_GSI_IRAM_PTR_READ_ENG_COMP_ADDR (GSI_REG_BASE + 0x00000444)
-#define HWIO_GSI_IRAM_PTR_READ_ENG_COMP_PHYS (GSI_REG_BASE_PHYS + \
-					      0x00000444)
-#define HWIO_GSI_IRAM_PTR_READ_ENG_COMP_OFFS (GSI_REG_BASE_OFFS + \
-					      0x00000444)
-#define HWIO_GSI_IRAM_PTR_UC_GP_INT_ADDR (GSI_REG_BASE + 0x00000448)
-#define HWIO_GSI_IRAM_PTR_UC_GP_INT_PHYS (GSI_REG_BASE_PHYS + 0x00000448)
-#define HWIO_GSI_IRAM_PTR_UC_GP_INT_OFFS (GSI_REG_BASE_OFFS + 0x00000448)
-#define HWIO_GSI_IRAM_PTR_INT_MOD_STOPPED_ADDR (GSI_REG_BASE + 0x0000044c)
-#define HWIO_GSI_IRAM_PTR_INT_MOD_STOPPED_PHYS (GSI_REG_BASE_PHYS + \
-						0x0000044c)
-#define HWIO_GSI_IRAM_PTR_INT_MOD_STOPPED_OFFS (GSI_REG_BASE_OFFS + \
-						0x0000044c)
-#define HWIO_GSI_IRAM_PTR_SDMA_INT_n_ADDR(n) (GSI_REG_BASE + 0x00000450 + \
-					      0x4 * (n))
-#define HWIO_GSI_IRAM_PTR_SDMA_INT_n_PHYS(n) (GSI_REG_BASE_PHYS + \
-					      0x00000450 + 0x4 * (n))
-#define HWIO_GSI_IRAM_PTR_SDMA_INT_n_OFFS(n) (GSI_REG_BASE_OFFS + \
-					      0x00000450 + 0x4 * (n))
-#define HWIO_GSI_INST_RAM_n_ADDR(n) (GSI_REG_BASE + 0x0001b000 + 0x4 * (n))
-#define HWIO_GSI_INST_RAM_n_PHYS(n) (GSI_REG_BASE_PHYS + 0x0001b000 + \
-				     0x4 * (n))
-#define HWIO_GSI_INST_RAM_n_OFFS(n) (GSI_REG_BASE_OFFS + 0x0001b000 + \
-				     0x4 * (n))
-#define HWIO_GSI_SHRAM_n_ADDR(n) (GSI_REG_BASE + 0x00002000 + 0x4 * (n))
-#define HWIO_GSI_SHRAM_n_PHYS(n) (GSI_REG_BASE_PHYS + 0x00002000 + 0x4 * \
-				  (n))
-#define HWIO_GSI_SHRAM_n_OFFS(n) (GSI_REG_BASE_OFFS + 0x00002000 + 0x4 * \
-				  (n))
-#define HWIO_GSI_SHRAM_n_RMSK 0xffffffff
-#define HWIO_GSI_SHRAM_n_MAXn 1343
-#define HWIO_GSI_SHRAM_n_ATTR 0x3
-#define HWIO_GSI_SHRAM_n_INI(n) in_dword_masked(HWIO_GSI_SHRAM_n_ADDR( \
-							n), \
-						HWIO_GSI_SHRAM_n_RMSK)
-#define HWIO_GSI_SHRAM_n_INMI(n, mask) in_dword_masked(	\
-		HWIO_GSI_SHRAM_n_ADDR(n), \
-		mask)
-#define HWIO_GSI_SHRAM_n_OUTI(n, val) out_dword(HWIO_GSI_SHRAM_n_ADDR( \
-							n), val)
-#define HWIO_GSI_SHRAM_n_OUTMI(n, mask, val) out_dword_masked_ns( \
-		HWIO_GSI_SHRAM_n_ADDR(n), \
-		mask, \
-		val, \
-		HWIO_GSI_SHRAM_n_INI(n))
-#define HWIO_GSI_SHRAM_n_SHRAM_BMSK 0xffffffff
-#define HWIO_GSI_SHRAM_n_SHRAM_SHFT 0x0
-#define HWIO_GSI_MAP_EE_n_CH_k_VP_TABLE_ADDR(n, k) (GSI_REG_BASE + \
-						    0x00003800 + 0x80 *	\
-						    (n) + 0x4 * (k))
-#define HWIO_GSI_MAP_EE_n_CH_k_VP_TABLE_PHYS(n, k) (GSI_REG_BASE_PHYS +	\
-						    0x00003800 + 0x80 *	\
-						    (n) + 0x4 * (k))
-#define HWIO_GSI_MAP_EE_n_CH_k_VP_TABLE_OFFS(n, k) (GSI_REG_BASE_OFFS +	\
-						    0x00003800 + 0x80 *	\
-						    (n) + 0x4 * (k))
-#define HWIO_GSI_MAP_EE_n_CH_k_VP_TABLE_RMSK 0x3f
-#define HWIO_GSI_MAP_EE_n_CH_k_VP_TABLE_MAXn 2
-#define HWIO_GSI_MAP_EE_n_CH_k_VP_TABLE_MAXk 22
-#define HWIO_GSI_MAP_EE_n_CH_k_VP_TABLE_ATTR 0x3
-#define HWIO_GSI_MAP_EE_n_CH_k_VP_TABLE_INI2(n, k) in_dword_masked( \
-		HWIO_GSI_MAP_EE_n_CH_k_VP_TABLE_ADDR(n, k), \
-		HWIO_GSI_MAP_EE_n_CH_k_VP_TABLE_RMSK)
-#define HWIO_GSI_MAP_EE_n_CH_k_VP_TABLE_INMI2(n, k, mask) in_dword_masked( \
-		HWIO_GSI_MAP_EE_n_CH_k_VP_TABLE_ADDR(n, k), \
-		mask)
-#define HWIO_GSI_MAP_EE_n_CH_k_VP_TABLE_OUTI2(n, k, val) out_dword( \
-		HWIO_GSI_MAP_EE_n_CH_k_VP_TABLE_ADDR(n, k), \
-		val)
-#define HWIO_GSI_MAP_EE_n_CH_k_VP_TABLE_OUTMI2(n, k, mask, \
-					       val) out_dword_masked_ns( \
-		HWIO_GSI_MAP_EE_n_CH_k_VP_TABLE_ADDR(n,	\
-						     k), \
-		mask, \
-		val, \
-		HWIO_GSI_MAP_EE_n_CH_k_VP_TABLE_INI2(n, k))
-#define HWIO_GSI_MAP_EE_n_CH_k_VP_TABLE_VALID_BMSK 0x20
-#define HWIO_GSI_MAP_EE_n_CH_k_VP_TABLE_VALID_SHFT 0x5
-#define HWIO_GSI_MAP_EE_n_CH_k_VP_TABLE_PHY_CH_BMSK 0x1f
-#define HWIO_GSI_MAP_EE_n_CH_k_VP_TABLE_PHY_CH_SHFT 0x0
-#define HWIO_GSI_TEST_BUS_SEL_ADDR (GSI_REG_BASE + 0x00001000)
-#define HWIO_GSI_TEST_BUS_SEL_PHYS (GSI_REG_BASE_PHYS + 0x00001000)
-#define HWIO_GSI_TEST_BUS_SEL_OFFS (GSI_REG_BASE_OFFS + 0x00001000)
-#define HWIO_GSI_TEST_BUS_SEL_RMSK 0xf00ff
-#define HWIO_GSI_TEST_BUS_SEL_ATTR 0x3
-#define HWIO_GSI_TEST_BUS_SEL_IN in_dword_masked( \
-		HWIO_GSI_TEST_BUS_SEL_ADDR, \
-		HWIO_GSI_TEST_BUS_SEL_RMSK)
-#define HWIO_GSI_TEST_BUS_SEL_INM(m) in_dword_masked( \
-		HWIO_GSI_TEST_BUS_SEL_ADDR, \
-		m)
-#define HWIO_GSI_TEST_BUS_SEL_OUT(v) out_dword(HWIO_GSI_TEST_BUS_SEL_ADDR, \
-					       v)
-#define HWIO_GSI_TEST_BUS_SEL_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_GSI_TEST_BUS_SEL_ADDR, \
-		m, \
-		v, \
-		HWIO_GSI_TEST_BUS_SEL_IN)
-#define HWIO_GSI_TEST_BUS_SEL_GSI_HW_EVENTS_SEL_BMSK 0xf0000
-#define HWIO_GSI_TEST_BUS_SEL_GSI_HW_EVENTS_SEL_SHFT 0x10
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_BMSK 0xff
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_SHFT 0x0
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_ZEROS_FVAL 0x0
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_MCS_0_FVAL 0x1
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_MCS_1_FVAL 0x2
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_MCS_2_FVAL 0x3
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_MCS_3_FVAL 0x4
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_MCS_4_FVAL 0x5
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_DB_ENG_FVAL 0x9
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_REE_0_FVAL 0xb
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_REE_1_FVAL 0xc
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_REE_2_FVAL 0xd
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_REE_3_FVAL 0xe
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_REE_4_FVAL 0xf
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_REE_5_FVAL 0x10
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_REE_6_FVAL 0x11
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_REE_7_FVAL 0x12
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_EVE_0_FVAL 0x13
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_EVE_1_FVAL 0x14
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_EVE_2_FVAL 0x15
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_EVE_3_FVAL 0x16
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_EVE_4_FVAL 0x17
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_EVE_5_FVAL 0x18
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_IE_0_FVAL 0x1b
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_IE_1_FVAL 0x1c
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_IE_2_FVAL 0x1d
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_IC_0_FVAL 0x1f
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_IC_1_FVAL 0x20
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_IC_2_FVAL 0x21
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_IC_3_FVAL 0x22
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_IC_4_FVAL 0x23
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_MOQA_0_FVAL 0x27
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_MOQA_1_FVAL 0x28
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_MOQA_2_FVAL 0x29
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_MOQA_3_FVAL 0x2a
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_TMR_0_FVAL 0x2b
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_TMR_1_FVAL 0x2c
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_TMR_2_FVAL 0x2d
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_TMR_3_FVAL 0x2e
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_RD_WR_0_FVAL 0x33
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_RD_WR_1_FVAL 0x34
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_RD_WR_2_FVAL 0x35
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_RD_WR_3_FVAL 0x36
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_CSR_FVAL 0x3a
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_SDMA_0_FVAL 0x3c
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_SMDA_1_FVAL 0x3d
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_CSR_1_FVAL 0x3e
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_CSR_2_FVAL 0x3f
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_MCS_5_FVAL 0x40
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_IC_5_FVAL 0x41
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_CSR_3_FVAL 0x42
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_TLV_0_FVAL 0x43
-#define HWIO_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_REE_8_FVAL 0x44
-#define HWIO_GSI_TEST_BUS_REG_ADDR (GSI_REG_BASE + 0x00001008)
-#define HWIO_GSI_TEST_BUS_REG_PHYS (GSI_REG_BASE_PHYS + 0x00001008)
-#define HWIO_GSI_TEST_BUS_REG_OFFS (GSI_REG_BASE_OFFS + 0x00001008)
-#define HWIO_GSI_TEST_BUS_REG_RMSK 0xffffffff
-#define HWIO_GSI_TEST_BUS_REG_ATTR 0x1
-#define HWIO_GSI_TEST_BUS_REG_IN in_dword_masked( \
-		HWIO_GSI_TEST_BUS_REG_ADDR, \
-		HWIO_GSI_TEST_BUS_REG_RMSK)
-#define HWIO_GSI_TEST_BUS_REG_INM(m) in_dword_masked( \
-		HWIO_GSI_TEST_BUS_REG_ADDR, \
-		m)
-#define HWIO_GSI_TEST_BUS_REG_GSI_TESTBUS_REG_BMSK 0xffffffff
-#define HWIO_GSI_TEST_BUS_REG_GSI_TESTBUS_REG_SHFT 0x0
-#define HWIO_GSI_DEBUG_BUSY_REG_ADDR (GSI_REG_BASE + 0x00001010)
-#define HWIO_GSI_DEBUG_BUSY_REG_PHYS (GSI_REG_BASE_PHYS + 0x00001010)
-#define HWIO_GSI_DEBUG_BUSY_REG_OFFS (GSI_REG_BASE_OFFS + 0x00001010)
-#define HWIO_GSI_DEBUG_EVENT_PENDING_ADDR (GSI_REG_BASE + 0x00001014)
-#define HWIO_GSI_DEBUG_EVENT_PENDING_PHYS (GSI_REG_BASE_PHYS + 0x00001014)
-#define HWIO_GSI_DEBUG_EVENT_PENDING_OFFS (GSI_REG_BASE_OFFS + 0x00001014)
-#define HWIO_GSI_DEBUG_TIMER_PENDING_ADDR (GSI_REG_BASE + 0x00001018)
-#define HWIO_GSI_DEBUG_TIMER_PENDING_PHYS (GSI_REG_BASE_PHYS + 0x00001018)
-#define HWIO_GSI_DEBUG_TIMER_PENDING_OFFS (GSI_REG_BASE_OFFS + 0x00001018)
-#define HWIO_GSI_DEBUG_RD_WR_PENDING_ADDR (GSI_REG_BASE + 0x0000101c)
-#define HWIO_GSI_DEBUG_RD_WR_PENDING_PHYS (GSI_REG_BASE_PHYS + 0x0000101c)
-#define HWIO_GSI_DEBUG_RD_WR_PENDING_OFFS (GSI_REG_BASE_OFFS + 0x0000101c)
-#define HWIO_GSI_DEBUG_COUNTER_CFGn_ADDR(n) (GSI_REG_BASE + 0x00001200 + \
-					     0x4 * (n))
-#define HWIO_GSI_DEBUG_COUNTER_CFGn_PHYS(n) (GSI_REG_BASE_PHYS + \
-					     0x00001200 + 0x4 * (n))
-#define HWIO_GSI_DEBUG_COUNTER_CFGn_OFFS(n) (GSI_REG_BASE_OFFS + \
-					     0x00001200 + 0x4 * (n))
-#define HWIO_GSI_DEBUG_COUNTERn_ADDR(n) (GSI_REG_BASE + 0x00001240 + 0x4 * \
-					 (n))
-#define HWIO_GSI_DEBUG_COUNTERn_PHYS(n) (GSI_REG_BASE_PHYS + 0x00001240 + \
-					 0x4 * (n))
-#define HWIO_GSI_DEBUG_COUNTERn_OFFS(n) (GSI_REG_BASE_OFFS + 0x00001240 + \
-					 0x4 * (n))
-#define HWIO_GSI_DEBUG_COUNTERn_RMSK 0xffff
-#define HWIO_GSI_DEBUG_COUNTERn_MAXn 7
-#define HWIO_GSI_DEBUG_COUNTERn_ATTR 0x1
-#define HWIO_GSI_DEBUG_COUNTERn_INI(n) in_dword_masked(	\
-		HWIO_GSI_DEBUG_COUNTERn_ADDR(n), \
-		HWIO_GSI_DEBUG_COUNTERn_RMSK)
-#define HWIO_GSI_DEBUG_COUNTERn_INMI(n, mask) in_dword_masked( \
-		HWIO_GSI_DEBUG_COUNTERn_ADDR(n), \
-		mask)
-#define HWIO_GSI_DEBUG_COUNTERn_COUNTER_VALUE_BMSK 0xffff
-#define HWIO_GSI_DEBUG_COUNTERn_COUNTER_VALUE_SHFT 0x0
-#define HWIO_GSI_DEBUG_PC_FROM_SW_ADDR (GSI_REG_BASE + 0x00001040)
-#define HWIO_GSI_DEBUG_PC_FROM_SW_PHYS (GSI_REG_BASE_PHYS + 0x00001040)
-#define HWIO_GSI_DEBUG_PC_FROM_SW_OFFS (GSI_REG_BASE_OFFS + 0x00001040)
-#define HWIO_GSI_DEBUG_SW_STALL_ADDR (GSI_REG_BASE + 0x00001044)
-#define HWIO_GSI_DEBUG_SW_STALL_PHYS (GSI_REG_BASE_PHYS + 0x00001044)
-#define HWIO_GSI_DEBUG_SW_STALL_OFFS (GSI_REG_BASE_OFFS + 0x00001044)
-#define HWIO_GSI_DEBUG_PC_FOR_DEBUG_ADDR (GSI_REG_BASE + 0x00001048)
-#define HWIO_GSI_DEBUG_PC_FOR_DEBUG_PHYS (GSI_REG_BASE_PHYS + 0x00001048)
-#define HWIO_GSI_DEBUG_PC_FOR_DEBUG_OFFS (GSI_REG_BASE_OFFS + 0x00001048)
-#define HWIO_GSI_DEBUG_QSB_LOG_SEL_ADDR (GSI_REG_BASE + 0x00001050)
-#define HWIO_GSI_DEBUG_QSB_LOG_SEL_PHYS (GSI_REG_BASE_PHYS + 0x00001050)
-#define HWIO_GSI_DEBUG_QSB_LOG_SEL_OFFS (GSI_REG_BASE_OFFS + 0x00001050)
-#define HWIO_GSI_DEBUG_QSB_LOG_CLR_ADDR (GSI_REG_BASE + 0x00001058)
-#define HWIO_GSI_DEBUG_QSB_LOG_CLR_PHYS (GSI_REG_BASE_PHYS + 0x00001058)
-#define HWIO_GSI_DEBUG_QSB_LOG_CLR_OFFS (GSI_REG_BASE_OFFS + 0x00001058)
-#define HWIO_GSI_DEBUG_QSB_LOG_ERR_TRNS_ID_ADDR (GSI_REG_BASE + 0x00001060)
-#define HWIO_GSI_DEBUG_QSB_LOG_ERR_TRNS_ID_PHYS (GSI_REG_BASE_PHYS + \
-						 0x00001060)
-#define HWIO_GSI_DEBUG_QSB_LOG_ERR_TRNS_ID_OFFS (GSI_REG_BASE_OFFS + \
-						 0x00001060)
-#define HWIO_GSI_DEBUG_QSB_LOG_0_ADDR (GSI_REG_BASE + 0x00001064)
-#define HWIO_GSI_DEBUG_QSB_LOG_0_PHYS (GSI_REG_BASE_PHYS + 0x00001064)
-#define HWIO_GSI_DEBUG_QSB_LOG_0_OFFS (GSI_REG_BASE_OFFS + 0x00001064)
-#define HWIO_GSI_DEBUG_QSB_LOG_1_ADDR (GSI_REG_BASE + 0x00001068)
-#define HWIO_GSI_DEBUG_QSB_LOG_1_PHYS (GSI_REG_BASE_PHYS + 0x00001068)
-#define HWIO_GSI_DEBUG_QSB_LOG_1_OFFS (GSI_REG_BASE_OFFS + 0x00001068)
-#define HWIO_GSI_DEBUG_QSB_LOG_2_ADDR (GSI_REG_BASE + 0x0000106c)
-#define HWIO_GSI_DEBUG_QSB_LOG_2_PHYS (GSI_REG_BASE_PHYS + 0x0000106c)
-#define HWIO_GSI_DEBUG_QSB_LOG_2_OFFS (GSI_REG_BASE_OFFS + 0x0000106c)
-#define HWIO_GSI_DEBUG_QSB_LOG_LAST_MISC_IDn_ADDR(n) (GSI_REG_BASE + \
-						      0x00001070 + 0x4 * \
-						      (n))
-#define HWIO_GSI_DEBUG_QSB_LOG_LAST_MISC_IDn_PHYS(n) (GSI_REG_BASE_PHYS + \
-						      0x00001070 + 0x4 * \
-						      (n))
-#define HWIO_GSI_DEBUG_QSB_LOG_LAST_MISC_IDn_OFFS(n) (GSI_REG_BASE_OFFS + \
-						      0x00001070 + 0x4 * \
-						      (n))
-#define HWIO_GSI_DEBUG_SW_RF_n_WRITE_ADDR(n) (GSI_REG_BASE + 0x00001080 + \
-					      0x4 * (n))
-#define HWIO_GSI_DEBUG_SW_RF_n_WRITE_PHYS(n) (GSI_REG_BASE_PHYS + \
-					      0x00001080 + 0x4 * (n))
-#define HWIO_GSI_DEBUG_SW_RF_n_WRITE_OFFS(n) (GSI_REG_BASE_OFFS + \
-					      0x00001080 + 0x4 * (n))
-#define HWIO_GSI_DEBUG_SW_RF_n_READ_ADDR(n) (GSI_REG_BASE + 0x00001100 + \
-					     0x4 * (n))
-#define HWIO_GSI_DEBUG_SW_RF_n_READ_PHYS(n) (GSI_REG_BASE_PHYS + \
-					     0x00001100 + 0x4 * (n))
-#define HWIO_GSI_DEBUG_SW_RF_n_READ_OFFS(n) (GSI_REG_BASE_OFFS + \
-					     0x00001100 + 0x4 * (n))
-#define HWIO_GSI_DEBUG_SW_RF_n_READ_RMSK 0xffffffff
-#define HWIO_GSI_DEBUG_SW_RF_n_READ_MAXn 31
-#define HWIO_GSI_DEBUG_SW_RF_n_READ_ATTR 0x1
-#define HWIO_GSI_DEBUG_SW_RF_n_READ_INI(n) in_dword_masked( \
-		HWIO_GSI_DEBUG_SW_RF_n_READ_ADDR(n), \
-		HWIO_GSI_DEBUG_SW_RF_n_READ_RMSK)
-#define HWIO_GSI_DEBUG_SW_RF_n_READ_INMI(n, mask) in_dword_masked( \
-		HWIO_GSI_DEBUG_SW_RF_n_READ_ADDR(n), \
-		mask)
-#define HWIO_GSI_DEBUG_SW_RF_n_READ_RF_REG_BMSK 0xffffffff
-#define HWIO_GSI_DEBUG_SW_RF_n_READ_RF_REG_SHFT 0x0
-#define HWIO_GSI_DEBUG_EE_n_CH_k_VP_TABLE_ADDR(n, k) (GSI_REG_BASE + \
-						      0x00001400 + 0x80 * \
-						      (n) + 0x4 * (k))
-#define HWIO_GSI_DEBUG_EE_n_CH_k_VP_TABLE_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-						      0x00001400 + 0x80 * \
-						      (n) + 0x4 * (k))
-#define HWIO_GSI_DEBUG_EE_n_CH_k_VP_TABLE_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-						      0x00001400 + 0x80 * \
-						      (n) + 0x4 * (k))
-#define HWIO_GSI_DEBUG_EE_n_EV_k_VP_TABLE_ADDR(n, k) (GSI_REG_BASE + \
-						      0x00001600 + 0x80 * \
-						      (n) + 0x4 * (k))
-#define HWIO_GSI_DEBUG_EE_n_EV_k_VP_TABLE_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-						      0x00001600 + 0x80 * \
-						      (n) + 0x4 * (k))
-#define HWIO_GSI_DEBUG_EE_n_EV_k_VP_TABLE_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-						      0x00001600 + 0x80 * \
-						      (n) + 0x4 * (k))
-#define HWIO_GSI_DEBUG_EE_n_EV_k_VP_TABLE_RMSK 0x3f
-#define HWIO_GSI_DEBUG_EE_n_EV_k_VP_TABLE_MAXn 3
-#define HWIO_GSI_DEBUG_EE_n_EV_k_VP_TABLE_MAXk 19
-#define HWIO_GSI_DEBUG_EE_n_EV_k_VP_TABLE_ATTR 0x1
-#define HWIO_GSI_DEBUG_EE_n_EV_k_VP_TABLE_INI2(n, k) in_dword_masked( \
-		HWIO_GSI_DEBUG_EE_n_EV_k_VP_TABLE_ADDR(n, k), \
-		HWIO_GSI_DEBUG_EE_n_EV_k_VP_TABLE_RMSK)
-#define HWIO_GSI_DEBUG_EE_n_EV_k_VP_TABLE_INMI2(n, k, \
-						mask) in_dword_masked( \
-		HWIO_GSI_DEBUG_EE_n_EV_k_VP_TABLE_ADDR(n, \
-						       k), \
-		mask)
-#define HWIO_GSI_DEBUG_EE_n_EV_k_VP_TABLE_VALID_BMSK 0x20
-#define HWIO_GSI_DEBUG_EE_n_EV_k_VP_TABLE_VALID_SHFT 0x5
-#define HWIO_GSI_DEBUG_EE_n_EV_k_VP_TABLE_PHY_EV_CH_BMSK 0x1f
-#define HWIO_GSI_DEBUG_EE_n_EV_k_VP_TABLE_PHY_EV_CH_SHFT 0x0
-#define HWIO_GSI_DEBUG_SDMA_TRANS_DB_n_ADDR(n) (GSI_REG_BASE + \
-						0x00001800 + 0x4 * (n))
-#define HWIO_GSI_DEBUG_SDMA_TRANS_DB_n_PHYS(n) (GSI_REG_BASE_PHYS + \
-						0x00001800 + 0x4 * (n))
-#define HWIO_GSI_DEBUG_SDMA_TRANS_DB_n_OFFS(n) (GSI_REG_BASE_OFFS + \
-						0x00001800 + 0x4 * (n))
-#define HWIO_GSI_UC_SRC_IRQ_ADDR (GSI_REG_BASE + 0x00000500)
-#define HWIO_GSI_UC_SRC_IRQ_PHYS (GSI_REG_BASE_PHYS + 0x00000500)
-#define HWIO_GSI_UC_SRC_IRQ_OFFS (GSI_REG_BASE_OFFS + 0x00000500)
-#define HWIO_GSI_UC_SRC_IRQ_MSK_ADDR (GSI_REG_BASE + 0x00000504)
-#define HWIO_GSI_UC_SRC_IRQ_MSK_PHYS (GSI_REG_BASE_PHYS + 0x00000504)
-#define HWIO_GSI_UC_SRC_IRQ_MSK_OFFS (GSI_REG_BASE_OFFS + 0x00000504)
-#define HWIO_GSI_UC_SRC_IRQ_CLR_ADDR (GSI_REG_BASE + 0x00000508)
-#define HWIO_GSI_UC_SRC_IRQ_CLR_PHYS (GSI_REG_BASE_PHYS + 0x00000508)
-#define HWIO_GSI_UC_SRC_IRQ_CLR_OFFS (GSI_REG_BASE_OFFS + 0x00000508)
-#define HWIO_GSI_ACC_ARGS_n_ADDR(n) (GSI_REG_BASE + 0x0000050c + 0x4 * (n))
-#define HWIO_GSI_ACC_ARGS_n_PHYS(n) (GSI_REG_BASE_PHYS + 0x0000050c + \
-				     0x4 * (n))
-#define HWIO_GSI_ACC_ARGS_n_OFFS(n) (GSI_REG_BASE_OFFS + 0x0000050c + \
-				     0x4 * (n))
-#define HWIO_GSI_ACC_ROUTINE_ADDR (GSI_REG_BASE + 0x00000524)
-#define HWIO_GSI_ACC_ROUTINE_PHYS (GSI_REG_BASE_PHYS + 0x00000524)
-#define HWIO_GSI_ACC_ROUTINE_OFFS (GSI_REG_BASE_OFFS + 0x00000524)
-#define HWIO_GSI_ACC_GO_ADDR (GSI_REG_BASE + 0x00000528)
-#define HWIO_GSI_ACC_GO_PHYS (GSI_REG_BASE_PHYS + 0x00000528)
-#define HWIO_GSI_ACC_GO_OFFS (GSI_REG_BASE_OFFS + 0x00000528)
-#define HWIO_GSI_ACC_2_UC_MCS_STTS_ADDR (GSI_REG_BASE + 0x0000052c)
-#define HWIO_GSI_ACC_2_UC_MCS_STTS_PHYS (GSI_REG_BASE_PHYS + 0x0000052c)
-#define HWIO_GSI_ACC_2_UC_MCS_STTS_OFFS (GSI_REG_BASE_OFFS + 0x0000052c)
-#define HWIO_GSI_ACC_2_UC_MCS_RET_VAL_LSB_ADDR (GSI_REG_BASE + 0x00000530)
-#define HWIO_GSI_ACC_2_UC_MCS_RET_VAL_LSB_PHYS (GSI_REG_BASE_PHYS + \
-						0x00000530)
-#define HWIO_GSI_ACC_2_UC_MCS_RET_VAL_LSB_OFFS (GSI_REG_BASE_OFFS + \
-						0x00000530)
-#define HWIO_GSI_ACC_2_UC_MCS_RET_VAL_MSB_ADDR (GSI_REG_BASE + 0x00000534)
-#define HWIO_GSI_ACC_2_UC_MCS_RET_VAL_MSB_PHYS (GSI_REG_BASE_PHYS + \
-						0x00000534)
-#define HWIO_GSI_ACC_2_UC_MCS_RET_VAL_MSB_OFFS (GSI_REG_BASE_OFFS + \
-						0x00000534)
-#define HWIO_GSI_IC_2_UC_MCS_VLD_ADDR (GSI_REG_BASE + 0x00000538)
-#define HWIO_GSI_IC_2_UC_MCS_VLD_PHYS (GSI_REG_BASE_PHYS + 0x00000538)
-#define HWIO_GSI_IC_2_UC_MCS_VLD_OFFS (GSI_REG_BASE_OFFS + 0x00000538)
-#define HWIO_GSI_IC_2_UC_MCS_PC_ADDR (GSI_REG_BASE + 0x0000053c)
-#define HWIO_GSI_IC_2_UC_MCS_PC_PHYS (GSI_REG_BASE_PHYS + 0x0000053c)
-#define HWIO_GSI_IC_2_UC_MCS_PC_OFFS (GSI_REG_BASE_OFFS + 0x0000053c)
-#define HWIO_GSI_IC_2_UC_MCS_ARGS_n_ADDR(n) (GSI_REG_BASE + 0x00000540 + \
-					     0x4 * (n))
-#define HWIO_GSI_IC_2_UC_MCS_ARGS_n_PHYS(n) (GSI_REG_BASE_PHYS + \
-					     0x00000540 + 0x4 * (n))
-#define HWIO_GSI_IC_2_UC_MCS_ARGS_n_OFFS(n) (GSI_REG_BASE_OFFS + \
-					     0x00000540 + 0x4 * (n))
-#define HWIO_GSI_UC_TLV_IN_VLD_ADDR (GSI_REG_BASE + 0x00000558)
-#define HWIO_GSI_UC_TLV_IN_VLD_PHYS (GSI_REG_BASE_PHYS + 0x00000558)
-#define HWIO_GSI_UC_TLV_IN_VLD_OFFS (GSI_REG_BASE_OFFS + 0x00000558)
-#define HWIO_GSI_UC_TLV_IN_ROUTINE_ADDR (GSI_REG_BASE + 0x0000055c)
-#define HWIO_GSI_UC_TLV_IN_ROUTINE_PHYS (GSI_REG_BASE_PHYS + 0x0000055c)
-#define HWIO_GSI_UC_TLV_IN_ROUTINE_OFFS (GSI_REG_BASE_OFFS + 0x0000055c)
-#define HWIO_GSI_UC_TLV_IN_ARGS_n_ADDR(n) (GSI_REG_BASE + 0x00000560 + \
-					   0x4 * (n))
-#define HWIO_GSI_UC_TLV_IN_ARGS_n_PHYS(n) (GSI_REG_BASE_PHYS + \
-					   0x00000560 + 0x4 * (n))
-#define HWIO_GSI_UC_TLV_IN_ARGS_n_OFFS(n) (GSI_REG_BASE_OFFS + \
-					   0x00000560 + 0x4 * (n))
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_ADDR(n, k) (GSI_REG_BASE + 0x0000f000 + \
-					       0x4000 * (n) + 0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-					       0x0000f000 + 0x4000 * (n) + \
-					       0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-					       0x0000f000 + 0x4000 * (n) + \
-					       0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_RMSK 0xfff7ffff
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_MAXn 2
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_MAXk 22
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_ATTR 0x3
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_INI2(n, k) in_dword_masked( \
-		HWIO_EE_n_GSI_CH_k_CNTXT_0_ADDR(n, k), \
-		HWIO_EE_n_GSI_CH_k_CNTXT_0_RMSK)
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_INMI2(n, k, mask) in_dword_masked( \
-		HWIO_EE_n_GSI_CH_k_CNTXT_0_ADDR(n, k), \
-		mask)
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_OUTI2(n, k, val) out_dword( \
-		HWIO_EE_n_GSI_CH_k_CNTXT_0_ADDR(n, k), \
-		val)
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_OUTMI2(n, k, mask, \
-					  val) out_dword_masked_ns( \
-		HWIO_EE_n_GSI_CH_k_CNTXT_0_ADDR(n, \
-						k), \
-		mask, \
-		val, \
-		HWIO_EE_n_GSI_CH_k_CNTXT_0_INI2(n, k))
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_ELEMENT_SIZE_BMSK 0xff000000
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_ELEMENT_SIZE_SHFT 0x18
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_CHSTATE_BMSK 0xf00000
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_CHSTATE_SHFT 0x14
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_CHSTATE_NOT_ALLOCATED_FVAL 0x0
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_CHSTATE_ALLOCATED_FVAL 0x1
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_CHSTATE_STARTED_FVAL 0x2
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_CHSTATE_STOPPED_FVAL 0x3
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_CHSTATE_STOP_IN_PROC_FVAL 0x4
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_CHSTATE_ERROR_FVAL 0xf
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_ERINDEX_BMSK 0x7c000
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_ERINDEX_SHFT 0xe
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_PROTOCOL_MSB_BMSK 0x2000
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_PROTOCOL_MSB_SHFT 0xd
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_CHID_BMSK 0x1f00
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_CHID_SHFT 0x8
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_EE_BMSK 0xf0
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_EE_SHFT 0x4
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_DIR_BMSK 0x8
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_DIR_SHFT 0x3
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_DIR_INBOUND_FVAL 0x0
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_DIR_OUTBOUND_FVAL 0x1
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_PROTOCOL_BMSK 0x7
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_PROTOCOL_SHFT 0x0
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_PROTOCOL_MHI_FVAL 0x0
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_PROTOCOL_XHCI_FVAL 0x1
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_PROTOCOL_GPI_FVAL 0x2
-#define HWIO_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_PROTOCOL_XDCI_FVAL 0x3
-#define HWIO_EE_n_GSI_CH_k_CNTXT_1_ADDR(n, k) (GSI_REG_BASE + 0x0000f004 + \
-					       0x4000 * (n) + 0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_CNTXT_1_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-					       0x0000f004 + 0x4000 * (n) + \
-					       0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_CNTXT_1_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-					       0x0000f004 + 0x4000 * (n) + \
-					       0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_CNTXT_1_RMSK 0xffff
-#define HWIO_EE_n_GSI_CH_k_CNTXT_1_MAXn 2
-#define HWIO_EE_n_GSI_CH_k_CNTXT_1_MAXk 22
-#define HWIO_EE_n_GSI_CH_k_CNTXT_1_ATTR 0x3
-#define HWIO_EE_n_GSI_CH_k_CNTXT_1_INI2(n, k) in_dword_masked( \
-		HWIO_EE_n_GSI_CH_k_CNTXT_1_ADDR(n, k), \
-		HWIO_EE_n_GSI_CH_k_CNTXT_1_RMSK)
-#define HWIO_EE_n_GSI_CH_k_CNTXT_1_INMI2(n, k, mask) in_dword_masked( \
-		HWIO_EE_n_GSI_CH_k_CNTXT_1_ADDR(n, k), \
-		mask)
-#define HWIO_EE_n_GSI_CH_k_CNTXT_1_OUTI2(n, k, val) out_dword( \
-		HWIO_EE_n_GSI_CH_k_CNTXT_1_ADDR(n, k), \
-		val)
-#define HWIO_EE_n_GSI_CH_k_CNTXT_1_OUTMI2(n, k, mask, \
-					  val) out_dword_masked_ns( \
-		HWIO_EE_n_GSI_CH_k_CNTXT_1_ADDR(n, \
-						k), \
-		mask, \
-		val, \
-		HWIO_EE_n_GSI_CH_k_CNTXT_1_INI2(n, k))
-#define HWIO_EE_n_GSI_CH_k_CNTXT_1_R_LENGTH_BMSK 0xffff
-#define HWIO_EE_n_GSI_CH_k_CNTXT_1_R_LENGTH_SHFT 0x0
-#define HWIO_EE_n_GSI_CH_k_CNTXT_2_ADDR(n, k) (GSI_REG_BASE + 0x0000f008 + \
-					       0x4000 * (n) + 0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_CNTXT_2_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-					       0x0000f008 + 0x4000 * (n) + \
-					       0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_CNTXT_2_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-					       0x0000f008 + 0x4000 * (n) + \
-					       0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_CNTXT_2_RMSK 0xffffffff
-#define HWIO_EE_n_GSI_CH_k_CNTXT_2_MAXn 2
-#define HWIO_EE_n_GSI_CH_k_CNTXT_2_MAXk 22
-#define HWIO_EE_n_GSI_CH_k_CNTXT_2_ATTR 0x3
-#define HWIO_EE_n_GSI_CH_k_CNTXT_2_INI2(n, k) in_dword_masked( \
-		HWIO_EE_n_GSI_CH_k_CNTXT_2_ADDR(n, k), \
-		HWIO_EE_n_GSI_CH_k_CNTXT_2_RMSK)
-#define HWIO_EE_n_GSI_CH_k_CNTXT_2_INMI2(n, k, mask) in_dword_masked( \
-		HWIO_EE_n_GSI_CH_k_CNTXT_2_ADDR(n, k), \
-		mask)
-#define HWIO_EE_n_GSI_CH_k_CNTXT_2_OUTI2(n, k, val) out_dword( \
-		HWIO_EE_n_GSI_CH_k_CNTXT_2_ADDR(n, k), \
-		val)
-#define HWIO_EE_n_GSI_CH_k_CNTXT_2_OUTMI2(n, k, mask, \
-					  val) out_dword_masked_ns( \
-		HWIO_EE_n_GSI_CH_k_CNTXT_2_ADDR(n, \
-						k), \
-		mask, \
-		val, \
-		HWIO_EE_n_GSI_CH_k_CNTXT_2_INI2(n, k))
-#define HWIO_EE_n_GSI_CH_k_CNTXT_2_R_BASE_ADDR_LSBS_BMSK 0xffffffff
-#define HWIO_EE_n_GSI_CH_k_CNTXT_2_R_BASE_ADDR_LSBS_SHFT 0x0
-#define HWIO_EE_n_GSI_CH_k_CNTXT_3_ADDR(n, k) (GSI_REG_BASE + 0x0000f00c + \
-					       0x4000 * (n) + 0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_CNTXT_3_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-					       0x0000f00c + 0x4000 * (n) + \
-					       0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_CNTXT_3_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-					       0x0000f00c + 0x4000 * (n) + \
-					       0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_CNTXT_3_RMSK 0xffffffff
-#define HWIO_EE_n_GSI_CH_k_CNTXT_3_MAXn 2
-#define HWIO_EE_n_GSI_CH_k_CNTXT_3_MAXk 22
-#define HWIO_EE_n_GSI_CH_k_CNTXT_3_ATTR 0x3
-#define HWIO_EE_n_GSI_CH_k_CNTXT_3_INI2(n, k) in_dword_masked( \
-		HWIO_EE_n_GSI_CH_k_CNTXT_3_ADDR(n, k), \
-		HWIO_EE_n_GSI_CH_k_CNTXT_3_RMSK)
-#define HWIO_EE_n_GSI_CH_k_CNTXT_3_INMI2(n, k, mask) in_dword_masked( \
-		HWIO_EE_n_GSI_CH_k_CNTXT_3_ADDR(n, k), \
-		mask)
-#define HWIO_EE_n_GSI_CH_k_CNTXT_3_OUTI2(n, k, val) out_dword( \
-		HWIO_EE_n_GSI_CH_k_CNTXT_3_ADDR(n, k), \
-		val)
-#define HWIO_EE_n_GSI_CH_k_CNTXT_3_OUTMI2(n, k, mask, \
-					  val) out_dword_masked_ns( \
-		HWIO_EE_n_GSI_CH_k_CNTXT_3_ADDR(n, \
-						k), \
-		mask, \
-		val, \
-		HWIO_EE_n_GSI_CH_k_CNTXT_3_INI2(n, k))
-#define HWIO_EE_n_GSI_CH_k_CNTXT_3_R_BASE_ADDR_MSBS_BMSK 0xffffffff
-#define HWIO_EE_n_GSI_CH_k_CNTXT_3_R_BASE_ADDR_MSBS_SHFT 0x0
-#define HWIO_EE_n_GSI_CH_k_CNTXT_4_ADDR(n, k) (GSI_REG_BASE + 0x0000f010 + \
-					       0x4000 * (n) + 0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_CNTXT_4_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-					       0x0000f010 + 0x4000 * (n) + \
-					       0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_CNTXT_4_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-					       0x0000f010 + 0x4000 * (n) + \
-					       0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_CNTXT_4_RMSK 0xffffffff
-#define HWIO_EE_n_GSI_CH_k_CNTXT_4_MAXn 2
-#define HWIO_EE_n_GSI_CH_k_CNTXT_4_MAXk 22
-#define HWIO_EE_n_GSI_CH_k_CNTXT_4_ATTR 0x3
-#define HWIO_EE_n_GSI_CH_k_CNTXT_4_INI2(n, k) in_dword_masked( \
-		HWIO_EE_n_GSI_CH_k_CNTXT_4_ADDR(n, k), \
-		HWIO_EE_n_GSI_CH_k_CNTXT_4_RMSK)
-#define HWIO_EE_n_GSI_CH_k_CNTXT_4_INMI2(n, k, mask) in_dword_masked( \
-		HWIO_EE_n_GSI_CH_k_CNTXT_4_ADDR(n, k), \
-		mask)
-#define HWIO_EE_n_GSI_CH_k_CNTXT_4_OUTI2(n, k, val) out_dword( \
-		HWIO_EE_n_GSI_CH_k_CNTXT_4_ADDR(n, k), \
-		val)
-#define HWIO_EE_n_GSI_CH_k_CNTXT_4_OUTMI2(n, k, mask, \
-					  val) out_dword_masked_ns( \
-		HWIO_EE_n_GSI_CH_k_CNTXT_4_ADDR(n, \
-						k), \
-		mask, \
-		val, \
-		HWIO_EE_n_GSI_CH_k_CNTXT_4_INI2(n, k))
-#define HWIO_EE_n_GSI_CH_k_CNTXT_4_READ_PTR_LSB_BMSK 0xffffffff
-#define HWIO_EE_n_GSI_CH_k_CNTXT_4_READ_PTR_LSB_SHFT 0x0
-#define HWIO_EE_n_GSI_CH_k_CNTXT_5_ADDR(n, k) (GSI_REG_BASE + 0x0000f014 + \
-					       0x4000 * (n) + 0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_CNTXT_5_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-					       0x0000f014 + 0x4000 * (n) + \
-					       0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_CNTXT_5_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-					       0x0000f014 + 0x4000 * (n) + \
-					       0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_CNTXT_5_RMSK 0xffffffff
-#define HWIO_EE_n_GSI_CH_k_CNTXT_5_MAXn 2
-#define HWIO_EE_n_GSI_CH_k_CNTXT_5_MAXk 22
-#define HWIO_EE_n_GSI_CH_k_CNTXT_5_ATTR 0x1
-#define HWIO_EE_n_GSI_CH_k_CNTXT_5_INI2(n, k) in_dword_masked( \
-		HWIO_EE_n_GSI_CH_k_CNTXT_5_ADDR(n, k), \
-		HWIO_EE_n_GSI_CH_k_CNTXT_5_RMSK)
-#define HWIO_EE_n_GSI_CH_k_CNTXT_5_INMI2(n, k, mask) in_dword_masked( \
-		HWIO_EE_n_GSI_CH_k_CNTXT_5_ADDR(n, k), \
-		mask)
-#define HWIO_EE_n_GSI_CH_k_CNTXT_5_READ_PTR_MSB_BMSK 0xffffffff
-#define HWIO_EE_n_GSI_CH_k_CNTXT_5_READ_PTR_MSB_SHFT 0x0
-#define HWIO_EE_n_GSI_CH_k_CNTXT_6_ADDR(n, k) (GSI_REG_BASE + 0x0000f018 + \
-					       0x4000 * (n) + 0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_CNTXT_6_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-					       0x0000f018 + 0x4000 * (n) + \
-					       0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_CNTXT_6_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-					       0x0000f018 + 0x4000 * (n) + \
-					       0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_CNTXT_6_RMSK 0xffffffff
-#define HWIO_EE_n_GSI_CH_k_CNTXT_6_MAXn 2
-#define HWIO_EE_n_GSI_CH_k_CNTXT_6_MAXk 22
-#define HWIO_EE_n_GSI_CH_k_CNTXT_6_ATTR 0x1
-#define HWIO_EE_n_GSI_CH_k_CNTXT_6_INI2(n, k) in_dword_masked( \
-		HWIO_EE_n_GSI_CH_k_CNTXT_6_ADDR(n, k), \
-		HWIO_EE_n_GSI_CH_k_CNTXT_6_RMSK)
-#define HWIO_EE_n_GSI_CH_k_CNTXT_6_INMI2(n, k, mask) in_dword_masked( \
-		HWIO_EE_n_GSI_CH_k_CNTXT_6_ADDR(n, k), \
-		mask)
-#define HWIO_EE_n_GSI_CH_k_CNTXT_6_WRITE_PTR_LSB_BMSK 0xffffffff
-#define HWIO_EE_n_GSI_CH_k_CNTXT_6_WRITE_PTR_LSB_SHFT 0x0
-#define HWIO_EE_n_GSI_CH_k_CNTXT_7_ADDR(n, k) (GSI_REG_BASE + 0x0000f01c + \
-					       0x4000 * (n) + 0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_CNTXT_7_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-					       0x0000f01c + 0x4000 * (n) + \
-					       0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_CNTXT_7_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-					       0x0000f01c + 0x4000 * (n) + \
-					       0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_CNTXT_7_RMSK 0xffffffff
-#define HWIO_EE_n_GSI_CH_k_CNTXT_7_MAXn 2
-#define HWIO_EE_n_GSI_CH_k_CNTXT_7_MAXk 22
-#define HWIO_EE_n_GSI_CH_k_CNTXT_7_ATTR 0x1
-#define HWIO_EE_n_GSI_CH_k_CNTXT_7_INI2(n, k) in_dword_masked( \
-		HWIO_EE_n_GSI_CH_k_CNTXT_7_ADDR(n, k), \
-		HWIO_EE_n_GSI_CH_k_CNTXT_7_RMSK)
-#define HWIO_EE_n_GSI_CH_k_CNTXT_7_INMI2(n, k, mask) in_dword_masked( \
-		HWIO_EE_n_GSI_CH_k_CNTXT_7_ADDR(n, k), \
-		mask)
-#define HWIO_EE_n_GSI_CH_k_CNTXT_7_WRITE_PTR_MSB_BMSK 0xffffffff
-#define HWIO_EE_n_GSI_CH_k_CNTXT_7_WRITE_PTR_MSB_SHFT 0x0
-#define HWIO_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_ADDR(n, k) (GSI_REG_BASE +	\
-							 0x0000f054 + \
-							 0x4000 * (n) +	\
-							 0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_PHYS(n, \
-						  k) (GSI_REG_BASE_PHYS + \
-						      0x0000f054 + \
-						      0x4000 * (n) + \
-						      0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_OFFS(n, \
-						  k) (GSI_REG_BASE_OFFS + \
-						      0x0000f054 + \
-						      0x4000 * (n) + \
-						      0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_RMSK 0xffff
-#define HWIO_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_MAXn 2
-#define HWIO_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_MAXk 22
-#define HWIO_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_ATTR 0x3
-#define HWIO_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_INI2(n, k) in_dword_masked( \
-		HWIO_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_ADDR(n, k), \
-		HWIO_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_RMSK)
-#define HWIO_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_INMI2(n, k, \
-						   mask) in_dword_masked( \
-		HWIO_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_ADDR(n, \
-							  k), \
-		mask)
-#define HWIO_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_OUTI2(n, k, val) out_dword( \
-		HWIO_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_ADDR(n, k), \
-		val)
-#define HWIO_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_OUTMI2(n, k, mask,	\
-						    val) \
-	out_dword_masked_ns(HWIO_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_ADDR( \
-				    n, \
-				    k), mask, val, \
-			    HWIO_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_INI2(n, k))
-#define HWIO_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_READ_PTR_BMSK 0xffff
-#define HWIO_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_READ_PTR_SHFT 0x0
-#define HWIO_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_ADDR(n, k) (GSI_REG_BASE + \
-							  0x0000f058 + \
-							  0x4000 * (n) + \
-							  0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_PHYS(n, \
-						   k) (GSI_REG_BASE_PHYS + \
-						       0x0000f058 + \
-						       0x4000 * (n) + \
-						       0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_OFFS(n, \
-						   k) (GSI_REG_BASE_OFFS + \
-						       0x0000f058 + \
-						       0x4000 * (n) + \
-						       0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_RMSK 0xffff
-#define HWIO_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_MAXn 2
-#define HWIO_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_MAXk 22
-#define HWIO_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_ATTR 0x3
-#define HWIO_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_INI2(n, k) in_dword_masked( \
-		HWIO_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_ADDR(n, k), \
-		HWIO_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_RMSK)
-#define HWIO_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_INMI2(n, k, \
-						    mask) in_dword_masked( \
-		HWIO_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_ADDR(n, \
-							   k), \
-		mask)
-#define HWIO_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_OUTI2(n, k, val) out_dword( \
-		HWIO_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_ADDR(n, k), \
-		val)
-#define HWIO_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_OUTMI2(n, k, mask, \
-						     val) \
-	out_dword_masked_ns(HWIO_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_ADDR(	\
-				    n, \
-				    k), mask, val, \
-			    HWIO_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_INI2(n, \
-								       k))
-#define HWIO_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_RE_INTR_DB_BMSK 0xffff
-#define HWIO_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_RE_INTR_DB_SHFT 0x0
-#define HWIO_EE_n_GSI_CH_k_QOS_ADDR(n, k) (GSI_REG_BASE + 0x0000f05c + \
-					   0x4000 * (n) + 0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_QOS_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-					   0x0000f05c + 0x4000 * (n) + \
-					   0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_QOS_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-					   0x0000f05c + 0x4000 * (n) + \
-					   0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_QOS_RMSK 0xff3f0f
-#define HWIO_EE_n_GSI_CH_k_QOS_MAXn 2
-#define HWIO_EE_n_GSI_CH_k_QOS_MAXk 22
-#define HWIO_EE_n_GSI_CH_k_QOS_ATTR 0x3
-#define HWIO_EE_n_GSI_CH_k_QOS_INI2(n, k) in_dword_masked( \
-		HWIO_EE_n_GSI_CH_k_QOS_ADDR(n, k), \
-		HWIO_EE_n_GSI_CH_k_QOS_RMSK)
-#define HWIO_EE_n_GSI_CH_k_QOS_INMI2(n, k, mask) in_dword_masked( \
-		HWIO_EE_n_GSI_CH_k_QOS_ADDR(n, k), \
-		mask)
-#define HWIO_EE_n_GSI_CH_k_QOS_OUTI2(n, k, val) out_dword( \
-		HWIO_EE_n_GSI_CH_k_QOS_ADDR(n, k), \
-		val)
-#define HWIO_EE_n_GSI_CH_k_QOS_OUTMI2(n, k, mask, val) out_dword_masked_ns( \
-		HWIO_EE_n_GSI_CH_k_QOS_ADDR(n, k), \
-		mask, \
-		val, \
-		HWIO_EE_n_GSI_CH_k_QOS_INI2(n, k))
-#define HWIO_EE_n_GSI_CH_k_QOS_EMPTY_LVL_THRSHOLD_BMSK 0xff0000
-#define HWIO_EE_n_GSI_CH_k_QOS_EMPTY_LVL_THRSHOLD_SHFT 0x10
-#define HWIO_EE_n_GSI_CH_k_QOS_PREFETCH_MODE_BMSK 0x3c00
-#define HWIO_EE_n_GSI_CH_k_QOS_PREFETCH_MODE_SHFT 0xa
-#define HWIO_EE_n_GSI_CH_k_QOS_PREFETCH_MODE_USE_PREFETCH_BUFS_FVAL 0x0
-#define HWIO_EE_n_GSI_CH_k_QOS_PREFETCH_MODE_ESCAPE_BUF_ONLY_FVAL 0x1
-#define HWIO_EE_n_GSI_CH_k_QOS_PREFETCH_MODE_SMART_PRE_FETCH_FVAL 0x2
-#define HWIO_EE_n_GSI_CH_k_QOS_PREFETCH_MODE_FREE_PRE_FETCH_FVAL 0x3
-#define HWIO_EE_n_GSI_CH_k_QOS_USE_DB_ENG_BMSK 0x200
-#define HWIO_EE_n_GSI_CH_k_QOS_USE_DB_ENG_SHFT 0x9
-#define HWIO_EE_n_GSI_CH_k_QOS_MAX_PREFETCH_BMSK 0x100
-#define HWIO_EE_n_GSI_CH_k_QOS_MAX_PREFETCH_SHFT 0x8
-#define HWIO_EE_n_GSI_CH_k_QOS_MAX_PREFETCH_ONE_PREFETCH_SEG_FVAL 0x0
-#define HWIO_EE_n_GSI_CH_k_QOS_MAX_PREFETCH_TWO_PREFETCH_SEG_FVAL 0x1
-#define HWIO_EE_n_GSI_CH_k_QOS_WRR_WEIGHT_BMSK 0xf
-#define HWIO_EE_n_GSI_CH_k_QOS_WRR_WEIGHT_SHFT 0x0
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_0_ADDR(n, k) (GSI_REG_BASE +	\
-						 0x0000f060 + 0x4000 * \
-						 (n) + 0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_0_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-						 0x0000f060 + 0x4000 * \
-						 (n) + 0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_0_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-						 0x0000f060 + 0x4000 * \
-						 (n) + 0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_0_RMSK 0xffffffff
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_0_MAXn 2
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_0_MAXk 22
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_0_ATTR 0x3
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_0_INI2(n, k) in_dword_masked( \
-		HWIO_EE_n_GSI_CH_k_SCRATCH_0_ADDR(n, k), \
-		HWIO_EE_n_GSI_CH_k_SCRATCH_0_RMSK)
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_0_INMI2(n, k, mask) in_dword_masked(	\
-		HWIO_EE_n_GSI_CH_k_SCRATCH_0_ADDR(n, k), \
-		mask)
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_0_OUTI2(n, k, val) out_dword( \
-		HWIO_EE_n_GSI_CH_k_SCRATCH_0_ADDR(n, k), \
-		val)
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_0_OUTMI2(n, k, mask,	\
-					    val) out_dword_masked_ns( \
-		HWIO_EE_n_GSI_CH_k_SCRATCH_0_ADDR(n, \
-						  k), \
-		mask, \
-		val, \
-		HWIO_EE_n_GSI_CH_k_SCRATCH_0_INI2(n, k))
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_0_SCRATCH_BMSK 0xffffffff
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_0_SCRATCH_SHFT 0x0
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_1_ADDR(n, k) (GSI_REG_BASE +	\
-						 0x0000f064 + 0x4000 * \
-						 (n) + 0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_1_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-						 0x0000f064 + 0x4000 * \
-						 (n) + 0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_1_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-						 0x0000f064 + 0x4000 * \
-						 (n) + 0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_1_RMSK 0xffffffff
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_1_MAXn 2
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_1_MAXk 22
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_1_ATTR 0x3
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_1_INI2(n, k) in_dword_masked( \
-		HWIO_EE_n_GSI_CH_k_SCRATCH_1_ADDR(n, k), \
-		HWIO_EE_n_GSI_CH_k_SCRATCH_1_RMSK)
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_1_INMI2(n, k, mask) in_dword_masked(	\
-		HWIO_EE_n_GSI_CH_k_SCRATCH_1_ADDR(n, k), \
-		mask)
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_1_OUTI2(n, k, val) out_dword( \
-		HWIO_EE_n_GSI_CH_k_SCRATCH_1_ADDR(n, k), \
-		val)
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_1_OUTMI2(n, k, mask,	\
-					    val) out_dword_masked_ns( \
-		HWIO_EE_n_GSI_CH_k_SCRATCH_1_ADDR(n, \
-						  k), \
-		mask, \
-		val, \
-		HWIO_EE_n_GSI_CH_k_SCRATCH_1_INI2(n, k))
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_1_SCRATCH_BMSK 0xffffffff
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_1_SCRATCH_SHFT 0x0
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_2_ADDR(n, k) (GSI_REG_BASE +	\
-						 0x0000f068 + 0x4000 * \
-						 (n) + 0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_2_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-						 0x0000f068 + 0x4000 * \
-						 (n) + 0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_2_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-						 0x0000f068 + 0x4000 * \
-						 (n) + 0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_2_RMSK 0xffffffff
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_2_MAXn 2
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_2_MAXk 22
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_2_ATTR 0x3
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_2_INI2(n, k) in_dword_masked( \
-		HWIO_EE_n_GSI_CH_k_SCRATCH_2_ADDR(n, k), \
-		HWIO_EE_n_GSI_CH_k_SCRATCH_2_RMSK)
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_2_INMI2(n, k, mask) in_dword_masked(	\
-		HWIO_EE_n_GSI_CH_k_SCRATCH_2_ADDR(n, k), \
-		mask)
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_2_OUTI2(n, k, val) out_dword( \
-		HWIO_EE_n_GSI_CH_k_SCRATCH_2_ADDR(n, k), \
-		val)
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_2_OUTMI2(n, k, mask,	\
-					    val) out_dword_masked_ns( \
-		HWIO_EE_n_GSI_CH_k_SCRATCH_2_ADDR(n, \
-						  k), \
-		mask, \
-		val, \
-		HWIO_EE_n_GSI_CH_k_SCRATCH_2_INI2(n, k))
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_2_SCRATCH_BMSK 0xffffffff
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_2_SCRATCH_SHFT 0x0
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_3_ADDR(n, k) (GSI_REG_BASE +	\
-						 0x0000f06c + 0x4000 * \
-						 (n) + 0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_3_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-						 0x0000f06c + 0x4000 * \
-						 (n) + 0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_3_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-						 0x0000f06c + 0x4000 * \
-						 (n) + 0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_3_RMSK 0xffffffff
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_3_MAXn 2
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_3_MAXk 22
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_3_ATTR 0x3
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_3_INI2(n, k) in_dword_masked( \
-		HWIO_EE_n_GSI_CH_k_SCRATCH_3_ADDR(n, k), \
-		HWIO_EE_n_GSI_CH_k_SCRATCH_3_RMSK)
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_3_INMI2(n, k, mask) in_dword_masked(	\
-		HWIO_EE_n_GSI_CH_k_SCRATCH_3_ADDR(n, k), \
-		mask)
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_3_OUTI2(n, k, val) out_dword( \
-		HWIO_EE_n_GSI_CH_k_SCRATCH_3_ADDR(n, k), \
-		val)
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_3_OUTMI2(n, k, mask,	\
-					    val) out_dword_masked_ns( \
-		HWIO_EE_n_GSI_CH_k_SCRATCH_3_ADDR(n, \
-						  k), \
-		mask, \
-		val, \
-		HWIO_EE_n_GSI_CH_k_SCRATCH_3_INI2(n, k))
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_3_SCRATCH_BMSK 0xffffffff
-#define HWIO_EE_n_GSI_CH_k_SCRATCH_3_SCRATCH_SHFT 0x0
-#define HWIO_EE_n_GSI_CH_k_DB_ENG_WRITE_PTR_ADDR(n, k) (GSI_REG_BASE + \
-							0x0000f070 + \
-							0x4000 * (n) + \
-							0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_DB_ENG_WRITE_PTR_PHYS(n, \
-						 k) (GSI_REG_BASE_PHYS + \
-						     0x0000f070 + 0x4000 * \
-						     (n) + 0x80 * (k))
-#define HWIO_EE_n_GSI_CH_k_DB_ENG_WRITE_PTR_OFFS(n, \
-						 k) (GSI_REG_BASE_OFFS + \
-						     0x0000f070 + 0x4000 * \
-						     (n) + 0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_0_ADDR(n, k) (GSI_REG_BASE + 0x00010000 + \
-					      0x4000 * (n) + 0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_0_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-					      0x00010000 + 0x4000 * (n) + \
-					      0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_0_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-					      0x00010000 + 0x4000 * (n) + \
-					      0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_0_RMSK 0xfff1ffff
-#define HWIO_EE_n_EV_CH_k_CNTXT_0_MAXn 2
-#define HWIO_EE_n_EV_CH_k_CNTXT_0_MAXk 19
-#define HWIO_EE_n_EV_CH_k_CNTXT_0_ATTR 0x3
-#define HWIO_EE_n_EV_CH_k_CNTXT_0_INI2(n, k) in_dword_masked( \
-		HWIO_EE_n_EV_CH_k_CNTXT_0_ADDR(n, k), \
-		HWIO_EE_n_EV_CH_k_CNTXT_0_RMSK)
-#define HWIO_EE_n_EV_CH_k_CNTXT_0_INMI2(n, k, mask) in_dword_masked( \
-		HWIO_EE_n_EV_CH_k_CNTXT_0_ADDR(n, k), \
-		mask)
-#define HWIO_EE_n_EV_CH_k_CNTXT_0_OUTI2(n, k, val) out_dword( \
-		HWIO_EE_n_EV_CH_k_CNTXT_0_ADDR(n, k), \
-		val)
-#define HWIO_EE_n_EV_CH_k_CNTXT_0_OUTMI2(n, k, mask, \
-					 val) out_dword_masked_ns( \
-		HWIO_EE_n_EV_CH_k_CNTXT_0_ADDR(n, \
-					       k), \
-		mask, \
-		val, \
-		HWIO_EE_n_EV_CH_k_CNTXT_0_INI2(n, k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_0_ELEMENT_SIZE_BMSK 0xff000000
-#define HWIO_EE_n_EV_CH_k_CNTXT_0_ELEMENT_SIZE_SHFT 0x18
-#define HWIO_EE_n_EV_CH_k_CNTXT_0_CHSTATE_BMSK 0xf00000
-#define HWIO_EE_n_EV_CH_k_CNTXT_0_CHSTATE_SHFT 0x14
-#define HWIO_EE_n_EV_CH_k_CNTXT_0_CHSTATE_NOT_ALLOCATED_FVAL 0x0
-#define HWIO_EE_n_EV_CH_k_CNTXT_0_CHSTATE_ALLOCATED_FVAL 0x1
-#define HWIO_EE_n_EV_CH_k_CNTXT_0_INTYPE_BMSK 0x10000
-#define HWIO_EE_n_EV_CH_k_CNTXT_0_INTYPE_SHFT 0x10
-#define HWIO_EE_n_EV_CH_k_CNTXT_0_INTYPE_MSI_FVAL 0x0
-#define HWIO_EE_n_EV_CH_k_CNTXT_0_INTYPE_IRQ_FVAL 0x1
-#define HWIO_EE_n_EV_CH_k_CNTXT_0_EVCHID_BMSK 0xff00
-#define HWIO_EE_n_EV_CH_k_CNTXT_0_EVCHID_SHFT 0x8
-#define HWIO_EE_n_EV_CH_k_CNTXT_0_EE_BMSK 0xf0
-#define HWIO_EE_n_EV_CH_k_CNTXT_0_EE_SHFT 0x4
-#define HWIO_EE_n_EV_CH_k_CNTXT_0_CHTYPE_BMSK 0xf
-#define HWIO_EE_n_EV_CH_k_CNTXT_0_CHTYPE_SHFT 0x0
-#define HWIO_EE_n_EV_CH_k_CNTXT_0_CHTYPE_MHI_EV_FVAL 0x0
-#define HWIO_EE_n_EV_CH_k_CNTXT_0_CHTYPE_XHCI_EV_FVAL 0x1
-#define HWIO_EE_n_EV_CH_k_CNTXT_0_CHTYPE_GPI_EV_FVAL 0x2
-#define HWIO_EE_n_EV_CH_k_CNTXT_0_CHTYPE_XDCI_FVAL 0x3
-#define HWIO_EE_n_EV_CH_k_CNTXT_1_ADDR(n, k) (GSI_REG_BASE + 0x00010004 + \
-					      0x4000 * (n) + 0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_1_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-					      0x00010004 + 0x4000 * (n) + \
-					      0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_1_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-					      0x00010004 + 0x4000 * (n) + \
-					      0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_1_RMSK 0xffff
-#define HWIO_EE_n_EV_CH_k_CNTXT_1_MAXn 2
-#define HWIO_EE_n_EV_CH_k_CNTXT_1_MAXk 19
-#define HWIO_EE_n_EV_CH_k_CNTXT_1_ATTR 0x3
-#define HWIO_EE_n_EV_CH_k_CNTXT_1_INI2(n, k) in_dword_masked( \
-		HWIO_EE_n_EV_CH_k_CNTXT_1_ADDR(n, k), \
-		HWIO_EE_n_EV_CH_k_CNTXT_1_RMSK)
-#define HWIO_EE_n_EV_CH_k_CNTXT_1_INMI2(n, k, mask) in_dword_masked( \
-		HWIO_EE_n_EV_CH_k_CNTXT_1_ADDR(n, k), \
-		mask)
-#define HWIO_EE_n_EV_CH_k_CNTXT_1_OUTI2(n, k, val) out_dword( \
-		HWIO_EE_n_EV_CH_k_CNTXT_1_ADDR(n, k), \
-		val)
-#define HWIO_EE_n_EV_CH_k_CNTXT_1_OUTMI2(n, k, mask, \
-					 val) out_dword_masked_ns( \
-		HWIO_EE_n_EV_CH_k_CNTXT_1_ADDR(n, \
-					       k), \
-		mask, \
-		val, \
-		HWIO_EE_n_EV_CH_k_CNTXT_1_INI2(n, k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_1_R_LENGTH_BMSK 0xffff
-#define HWIO_EE_n_EV_CH_k_CNTXT_1_R_LENGTH_SHFT 0x0
-#define HWIO_EE_n_EV_CH_k_CNTXT_2_ADDR(n, k) (GSI_REG_BASE + 0x00010008 + \
-					      0x4000 * (n) + 0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_2_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-					      0x00010008 + 0x4000 * (n) + \
-					      0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_2_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-					      0x00010008 + 0x4000 * (n) + \
-					      0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_2_RMSK 0xffffffff
-#define HWIO_EE_n_EV_CH_k_CNTXT_2_MAXn 2
-#define HWIO_EE_n_EV_CH_k_CNTXT_2_MAXk 19
-#define HWIO_EE_n_EV_CH_k_CNTXT_2_ATTR 0x3
-#define HWIO_EE_n_EV_CH_k_CNTXT_2_INI2(n, k) in_dword_masked( \
-		HWIO_EE_n_EV_CH_k_CNTXT_2_ADDR(n, k), \
-		HWIO_EE_n_EV_CH_k_CNTXT_2_RMSK)
-#define HWIO_EE_n_EV_CH_k_CNTXT_2_INMI2(n, k, mask) in_dword_masked( \
-		HWIO_EE_n_EV_CH_k_CNTXT_2_ADDR(n, k), \
-		mask)
-#define HWIO_EE_n_EV_CH_k_CNTXT_2_OUTI2(n, k, val) out_dword( \
-		HWIO_EE_n_EV_CH_k_CNTXT_2_ADDR(n, k), \
-		val)
-#define HWIO_EE_n_EV_CH_k_CNTXT_2_OUTMI2(n, k, mask, \
-					 val) out_dword_masked_ns( \
-		HWIO_EE_n_EV_CH_k_CNTXT_2_ADDR(n, \
-					       k), \
-		mask, \
-		val, \
-		HWIO_EE_n_EV_CH_k_CNTXT_2_INI2(n, k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_2_R_BASE_ADDR_LSBS_BMSK 0xffffffff
-#define HWIO_EE_n_EV_CH_k_CNTXT_2_R_BASE_ADDR_LSBS_SHFT 0x0
-#define HWIO_EE_n_EV_CH_k_CNTXT_3_ADDR(n, k) (GSI_REG_BASE + 0x0001000c + \
-					      0x4000 * (n) + 0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_3_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-					      0x0001000c + 0x4000 * (n) + \
-					      0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_3_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-					      0x0001000c + 0x4000 * (n) + \
-					      0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_3_RMSK 0xffffffff
-#define HWIO_EE_n_EV_CH_k_CNTXT_3_MAXn 2
-#define HWIO_EE_n_EV_CH_k_CNTXT_3_MAXk 19
-#define HWIO_EE_n_EV_CH_k_CNTXT_3_ATTR 0x3
-#define HWIO_EE_n_EV_CH_k_CNTXT_3_INI2(n, k) in_dword_masked( \
-		HWIO_EE_n_EV_CH_k_CNTXT_3_ADDR(n, k), \
-		HWIO_EE_n_EV_CH_k_CNTXT_3_RMSK)
-#define HWIO_EE_n_EV_CH_k_CNTXT_3_INMI2(n, k, mask) in_dword_masked( \
-		HWIO_EE_n_EV_CH_k_CNTXT_3_ADDR(n, k), \
-		mask)
-#define HWIO_EE_n_EV_CH_k_CNTXT_3_OUTI2(n, k, val) out_dword( \
-		HWIO_EE_n_EV_CH_k_CNTXT_3_ADDR(n, k), \
-		val)
-#define HWIO_EE_n_EV_CH_k_CNTXT_3_OUTMI2(n, k, mask, \
-					 val) out_dword_masked_ns( \
-		HWIO_EE_n_EV_CH_k_CNTXT_3_ADDR(n, \
-					       k), \
-		mask, \
-		val, \
-		HWIO_EE_n_EV_CH_k_CNTXT_3_INI2(n, k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_3_R_BASE_ADDR_MSBS_BMSK 0xffffffff
-#define HWIO_EE_n_EV_CH_k_CNTXT_3_R_BASE_ADDR_MSBS_SHFT 0x0
-#define HWIO_EE_n_EV_CH_k_CNTXT_4_ADDR(n, k) (GSI_REG_BASE + 0x00010010 + \
-					      0x4000 * (n) + 0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_4_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-					      0x00010010 + 0x4000 * (n) + \
-					      0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_4_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-					      0x00010010 + 0x4000 * (n) + \
-					      0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_4_RMSK 0xffffffff
-#define HWIO_EE_n_EV_CH_k_CNTXT_4_MAXn 2
-#define HWIO_EE_n_EV_CH_k_CNTXT_4_MAXk 19
-#define HWIO_EE_n_EV_CH_k_CNTXT_4_ATTR 0x3
-#define HWIO_EE_n_EV_CH_k_CNTXT_4_INI2(n, k) in_dword_masked( \
-		HWIO_EE_n_EV_CH_k_CNTXT_4_ADDR(n, k), \
-		HWIO_EE_n_EV_CH_k_CNTXT_4_RMSK)
-#define HWIO_EE_n_EV_CH_k_CNTXT_4_INMI2(n, k, mask) in_dword_masked( \
-		HWIO_EE_n_EV_CH_k_CNTXT_4_ADDR(n, k), \
-		mask)
-#define HWIO_EE_n_EV_CH_k_CNTXT_4_OUTI2(n, k, val) out_dword( \
-		HWIO_EE_n_EV_CH_k_CNTXT_4_ADDR(n, k), \
-		val)
-#define HWIO_EE_n_EV_CH_k_CNTXT_4_OUTMI2(n, k, mask, \
-					 val) out_dword_masked_ns( \
-		HWIO_EE_n_EV_CH_k_CNTXT_4_ADDR(n, \
-					       k), \
-		mask, \
-		val, \
-		HWIO_EE_n_EV_CH_k_CNTXT_4_INI2(n, k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_4_READ_PTR_LSB_BMSK 0xffffffff
-#define HWIO_EE_n_EV_CH_k_CNTXT_4_READ_PTR_LSB_SHFT 0x0
-#define HWIO_EE_n_EV_CH_k_CNTXT_5_ADDR(n, k) (GSI_REG_BASE + 0x00010014 + \
-					      0x4000 * (n) + 0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_5_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-					      0x00010014 + 0x4000 * (n) + \
-					      0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_5_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-					      0x00010014 + 0x4000 * (n) + \
-					      0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_5_RMSK 0xffffffff
-#define HWIO_EE_n_EV_CH_k_CNTXT_5_MAXn 2
-#define HWIO_EE_n_EV_CH_k_CNTXT_5_MAXk 19
-#define HWIO_EE_n_EV_CH_k_CNTXT_5_ATTR 0x1
-#define HWIO_EE_n_EV_CH_k_CNTXT_5_INI2(n, k) in_dword_masked( \
-		HWIO_EE_n_EV_CH_k_CNTXT_5_ADDR(n, k), \
-		HWIO_EE_n_EV_CH_k_CNTXT_5_RMSK)
-#define HWIO_EE_n_EV_CH_k_CNTXT_5_INMI2(n, k, mask) in_dword_masked( \
-		HWIO_EE_n_EV_CH_k_CNTXT_5_ADDR(n, k), \
-		mask)
-#define HWIO_EE_n_EV_CH_k_CNTXT_5_READ_PTR_MSB_BMSK 0xffffffff
-#define HWIO_EE_n_EV_CH_k_CNTXT_5_READ_PTR_MSB_SHFT 0x0
-#define HWIO_EE_n_EV_CH_k_CNTXT_6_ADDR(n, k) (GSI_REG_BASE + 0x00010018 + \
-					      0x4000 * (n) + 0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_6_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-					      0x00010018 + 0x4000 * (n) + \
-					      0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_6_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-					      0x00010018 + 0x4000 * (n) + \
-					      0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_6_RMSK 0xffffffff
-#define HWIO_EE_n_EV_CH_k_CNTXT_6_MAXn 2
-#define HWIO_EE_n_EV_CH_k_CNTXT_6_MAXk 19
-#define HWIO_EE_n_EV_CH_k_CNTXT_6_ATTR 0x1
-#define HWIO_EE_n_EV_CH_k_CNTXT_6_INI2(n, k) in_dword_masked( \
-		HWIO_EE_n_EV_CH_k_CNTXT_6_ADDR(n, k), \
-		HWIO_EE_n_EV_CH_k_CNTXT_6_RMSK)
-#define HWIO_EE_n_EV_CH_k_CNTXT_6_INMI2(n, k, mask) in_dword_masked( \
-		HWIO_EE_n_EV_CH_k_CNTXT_6_ADDR(n, k), \
-		mask)
-#define HWIO_EE_n_EV_CH_k_CNTXT_6_WRITE_PTR_LSB_BMSK 0xffffffff
-#define HWIO_EE_n_EV_CH_k_CNTXT_6_WRITE_PTR_LSB_SHFT 0x0
-#define HWIO_EE_n_EV_CH_k_CNTXT_7_ADDR(n, k) (GSI_REG_BASE + 0x0001001c + \
-					      0x4000 * (n) + 0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_7_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-					      0x0001001c + 0x4000 * (n) + \
-					      0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_7_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-					      0x0001001c + 0x4000 * (n) + \
-					      0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_7_RMSK 0xffffffff
-#define HWIO_EE_n_EV_CH_k_CNTXT_7_MAXn 2
-#define HWIO_EE_n_EV_CH_k_CNTXT_7_MAXk 19
-#define HWIO_EE_n_EV_CH_k_CNTXT_7_ATTR 0x1
-#define HWIO_EE_n_EV_CH_k_CNTXT_7_INI2(n, k) in_dword_masked( \
-		HWIO_EE_n_EV_CH_k_CNTXT_7_ADDR(n, k), \
-		HWIO_EE_n_EV_CH_k_CNTXT_7_RMSK)
-#define HWIO_EE_n_EV_CH_k_CNTXT_7_INMI2(n, k, mask) in_dword_masked( \
-		HWIO_EE_n_EV_CH_k_CNTXT_7_ADDR(n, k), \
-		mask)
-#define HWIO_EE_n_EV_CH_k_CNTXT_7_WRITE_PTR_MSB_BMSK 0xffffffff
-#define HWIO_EE_n_EV_CH_k_CNTXT_7_WRITE_PTR_MSB_SHFT 0x0
-#define HWIO_EE_n_EV_CH_k_CNTXT_8_ADDR(n, k) (GSI_REG_BASE + 0x00010020 + \
-					      0x4000 * (n) + 0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_8_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-					      0x00010020 + 0x4000 * (n) + \
-					      0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_8_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-					      0x00010020 + 0x4000 * (n) + \
-					      0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_8_RMSK 0xffffffff
-#define HWIO_EE_n_EV_CH_k_CNTXT_8_MAXn 2
-#define HWIO_EE_n_EV_CH_k_CNTXT_8_MAXk 19
-#define HWIO_EE_n_EV_CH_k_CNTXT_8_ATTR 0x3
-#define HWIO_EE_n_EV_CH_k_CNTXT_8_INI2(n, k) in_dword_masked( \
-		HWIO_EE_n_EV_CH_k_CNTXT_8_ADDR(n, k), \
-		HWIO_EE_n_EV_CH_k_CNTXT_8_RMSK)
-#define HWIO_EE_n_EV_CH_k_CNTXT_8_INMI2(n, k, mask) in_dword_masked( \
-		HWIO_EE_n_EV_CH_k_CNTXT_8_ADDR(n, k), \
-		mask)
-#define HWIO_EE_n_EV_CH_k_CNTXT_8_OUTI2(n, k, val) out_dword( \
-		HWIO_EE_n_EV_CH_k_CNTXT_8_ADDR(n, k), \
-		val)
-#define HWIO_EE_n_EV_CH_k_CNTXT_8_OUTMI2(n, k, mask, \
-					 val) out_dword_masked_ns( \
-		HWIO_EE_n_EV_CH_k_CNTXT_8_ADDR(n, \
-					       k), \
-		mask, \
-		val, \
-		HWIO_EE_n_EV_CH_k_CNTXT_8_INI2(n, k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_8_INT_MOD_CNT_BMSK 0xff000000
-#define HWIO_EE_n_EV_CH_k_CNTXT_8_INT_MOD_CNT_SHFT 0x18
-#define HWIO_EE_n_EV_CH_k_CNTXT_8_INT_MODC_BMSK 0xff0000
-#define HWIO_EE_n_EV_CH_k_CNTXT_8_INT_MODC_SHFT 0x10
-#define HWIO_EE_n_EV_CH_k_CNTXT_8_INT_MODT_BMSK 0xffff
-#define HWIO_EE_n_EV_CH_k_CNTXT_8_INT_MODT_SHFT 0x0
-#define HWIO_EE_n_EV_CH_k_CNTXT_9_ADDR(n, k) (GSI_REG_BASE + 0x00010024 + \
-					      0x4000 * (n) + 0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_9_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-					      0x00010024 + 0x4000 * (n) + \
-					      0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_9_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-					      0x00010024 + 0x4000 * (n) + \
-					      0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_9_RMSK 0xffffffff
-#define HWIO_EE_n_EV_CH_k_CNTXT_9_MAXn 2
-#define HWIO_EE_n_EV_CH_k_CNTXT_9_MAXk 19
-#define HWIO_EE_n_EV_CH_k_CNTXT_9_ATTR 0x3
-#define HWIO_EE_n_EV_CH_k_CNTXT_9_INI2(n, k) in_dword_masked( \
-		HWIO_EE_n_EV_CH_k_CNTXT_9_ADDR(n, k), \
-		HWIO_EE_n_EV_CH_k_CNTXT_9_RMSK)
-#define HWIO_EE_n_EV_CH_k_CNTXT_9_INMI2(n, k, mask) in_dword_masked( \
-		HWIO_EE_n_EV_CH_k_CNTXT_9_ADDR(n, k), \
-		mask)
-#define HWIO_EE_n_EV_CH_k_CNTXT_9_OUTI2(n, k, val) out_dword( \
-		HWIO_EE_n_EV_CH_k_CNTXT_9_ADDR(n, k), \
-		val)
-#define HWIO_EE_n_EV_CH_k_CNTXT_9_OUTMI2(n, k, mask, \
-					 val) out_dword_masked_ns( \
-		HWIO_EE_n_EV_CH_k_CNTXT_9_ADDR(n, \
-					       k), \
-		mask, \
-		val, \
-		HWIO_EE_n_EV_CH_k_CNTXT_9_INI2(n, k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_9_INTVEC_BMSK 0xffffffff
-#define HWIO_EE_n_EV_CH_k_CNTXT_9_INTVEC_SHFT 0x0
-#define HWIO_EE_n_EV_CH_k_CNTXT_10_ADDR(n, k) (GSI_REG_BASE + 0x00010028 + \
-					       0x4000 * (n) + 0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_10_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-					       0x00010028 + 0x4000 * (n) + \
-					       0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_10_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-					       0x00010028 + 0x4000 * (n) + \
-					       0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_10_RMSK 0xffffffff
-#define HWIO_EE_n_EV_CH_k_CNTXT_10_MAXn 2
-#define HWIO_EE_n_EV_CH_k_CNTXT_10_MAXk 19
-#define HWIO_EE_n_EV_CH_k_CNTXT_10_ATTR 0x3
-#define HWIO_EE_n_EV_CH_k_CNTXT_10_INI2(n, k) in_dword_masked( \
-		HWIO_EE_n_EV_CH_k_CNTXT_10_ADDR(n, k), \
-		HWIO_EE_n_EV_CH_k_CNTXT_10_RMSK)
-#define HWIO_EE_n_EV_CH_k_CNTXT_10_INMI2(n, k, mask) in_dword_masked( \
-		HWIO_EE_n_EV_CH_k_CNTXT_10_ADDR(n, k), \
-		mask)
-#define HWIO_EE_n_EV_CH_k_CNTXT_10_OUTI2(n, k, val) out_dword( \
-		HWIO_EE_n_EV_CH_k_CNTXT_10_ADDR(n, k), \
-		val)
-#define HWIO_EE_n_EV_CH_k_CNTXT_10_OUTMI2(n, k, mask, \
-					  val) out_dword_masked_ns( \
-		HWIO_EE_n_EV_CH_k_CNTXT_10_ADDR(n, \
-						k), \
-		mask, \
-		val, \
-		HWIO_EE_n_EV_CH_k_CNTXT_10_INI2(n, k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_10_MSI_ADDR_LSB_BMSK 0xffffffff
-#define HWIO_EE_n_EV_CH_k_CNTXT_10_MSI_ADDR_LSB_SHFT 0x0
-#define HWIO_EE_n_EV_CH_k_CNTXT_11_ADDR(n, k) (GSI_REG_BASE + 0x0001002c + \
-					       0x4000 * (n) + 0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_11_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-					       0x0001002c + 0x4000 * (n) + \
-					       0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_11_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-					       0x0001002c + 0x4000 * (n) + \
-					       0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_11_RMSK 0xffffffff
-#define HWIO_EE_n_EV_CH_k_CNTXT_11_MAXn 2
-#define HWIO_EE_n_EV_CH_k_CNTXT_11_MAXk 19
-#define HWIO_EE_n_EV_CH_k_CNTXT_11_ATTR 0x3
-#define HWIO_EE_n_EV_CH_k_CNTXT_11_INI2(n, k) in_dword_masked( \
-		HWIO_EE_n_EV_CH_k_CNTXT_11_ADDR(n, k), \
-		HWIO_EE_n_EV_CH_k_CNTXT_11_RMSK)
-#define HWIO_EE_n_EV_CH_k_CNTXT_11_INMI2(n, k, mask) in_dword_masked( \
-		HWIO_EE_n_EV_CH_k_CNTXT_11_ADDR(n, k), \
-		mask)
-#define HWIO_EE_n_EV_CH_k_CNTXT_11_OUTI2(n, k, val) out_dword( \
-		HWIO_EE_n_EV_CH_k_CNTXT_11_ADDR(n, k), \
-		val)
-#define HWIO_EE_n_EV_CH_k_CNTXT_11_OUTMI2(n, k, mask, \
-					  val) out_dword_masked_ns( \
-		HWIO_EE_n_EV_CH_k_CNTXT_11_ADDR(n, \
-						k), \
-		mask, \
-		val, \
-		HWIO_EE_n_EV_CH_k_CNTXT_11_INI2(n, k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_11_MSI_ADDR_MSB_BMSK 0xffffffff
-#define HWIO_EE_n_EV_CH_k_CNTXT_11_MSI_ADDR_MSB_SHFT 0x0
-#define HWIO_EE_n_EV_CH_k_CNTXT_12_ADDR(n, k) (GSI_REG_BASE + 0x00010030 + \
-					       0x4000 * (n) + 0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_12_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-					       0x00010030 + 0x4000 * (n) + \
-					       0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_12_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-					       0x00010030 + 0x4000 * (n) + \
-					       0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_12_RMSK 0xffffffff
-#define HWIO_EE_n_EV_CH_k_CNTXT_12_MAXn 2
-#define HWIO_EE_n_EV_CH_k_CNTXT_12_MAXk 19
-#define HWIO_EE_n_EV_CH_k_CNTXT_12_ATTR 0x3
-#define HWIO_EE_n_EV_CH_k_CNTXT_12_INI2(n, k) in_dword_masked( \
-		HWIO_EE_n_EV_CH_k_CNTXT_12_ADDR(n, k), \
-		HWIO_EE_n_EV_CH_k_CNTXT_12_RMSK)
-#define HWIO_EE_n_EV_CH_k_CNTXT_12_INMI2(n, k, mask) in_dword_masked( \
-		HWIO_EE_n_EV_CH_k_CNTXT_12_ADDR(n, k), \
-		mask)
-#define HWIO_EE_n_EV_CH_k_CNTXT_12_OUTI2(n, k, val) out_dword( \
-		HWIO_EE_n_EV_CH_k_CNTXT_12_ADDR(n, k), \
-		val)
-#define HWIO_EE_n_EV_CH_k_CNTXT_12_OUTMI2(n, k, mask, \
-					  val) out_dword_masked_ns( \
-		HWIO_EE_n_EV_CH_k_CNTXT_12_ADDR(n, \
-						k), \
-		mask, \
-		val, \
-		HWIO_EE_n_EV_CH_k_CNTXT_12_INI2(n, k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_12_RP_UPDATE_ADDR_LSB_BMSK 0xffffffff
-#define HWIO_EE_n_EV_CH_k_CNTXT_12_RP_UPDATE_ADDR_LSB_SHFT 0x0
-#define HWIO_EE_n_EV_CH_k_CNTXT_13_ADDR(n, k) (GSI_REG_BASE + 0x00010034 + \
-					       0x4000 * (n) + 0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_13_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-					       0x00010034 + 0x4000 * (n) + \
-					       0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_13_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-					       0x00010034 + 0x4000 * (n) + \
-					       0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_13_RMSK 0xffffffff
-#define HWIO_EE_n_EV_CH_k_CNTXT_13_MAXn 2
-#define HWIO_EE_n_EV_CH_k_CNTXT_13_MAXk 19
-#define HWIO_EE_n_EV_CH_k_CNTXT_13_ATTR 0x3
-#define HWIO_EE_n_EV_CH_k_CNTXT_13_INI2(n, k) in_dword_masked( \
-		HWIO_EE_n_EV_CH_k_CNTXT_13_ADDR(n, k), \
-		HWIO_EE_n_EV_CH_k_CNTXT_13_RMSK)
-#define HWIO_EE_n_EV_CH_k_CNTXT_13_INMI2(n, k, mask) in_dword_masked( \
-		HWIO_EE_n_EV_CH_k_CNTXT_13_ADDR(n, k), \
-		mask)
-#define HWIO_EE_n_EV_CH_k_CNTXT_13_OUTI2(n, k, val) out_dword( \
-		HWIO_EE_n_EV_CH_k_CNTXT_13_ADDR(n, k), \
-		val)
-#define HWIO_EE_n_EV_CH_k_CNTXT_13_OUTMI2(n, k, mask, \
-					  val) out_dword_masked_ns( \
-		HWIO_EE_n_EV_CH_k_CNTXT_13_ADDR(n, \
-						k), \
-		mask, \
-		val, \
-		HWIO_EE_n_EV_CH_k_CNTXT_13_INI2(n, k))
-#define HWIO_EE_n_EV_CH_k_CNTXT_13_RP_UPDATE_ADDR_MSB_BMSK 0xffffffff
-#define HWIO_EE_n_EV_CH_k_CNTXT_13_RP_UPDATE_ADDR_MSB_SHFT 0x0
-#define HWIO_EE_n_EV_CH_k_SCRATCH_0_ADDR(n, k) (GSI_REG_BASE + \
-						0x00010048 + 0x4000 * \
-						(n) + 0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_SCRATCH_0_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-						0x00010048 + 0x4000 * \
-						(n) + 0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_SCRATCH_0_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-						0x00010048 + 0x4000 * \
-						(n) + 0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_SCRATCH_0_RMSK 0xffffffff
-#define HWIO_EE_n_EV_CH_k_SCRATCH_0_MAXn 2
-#define HWIO_EE_n_EV_CH_k_SCRATCH_0_MAXk 19
-#define HWIO_EE_n_EV_CH_k_SCRATCH_0_ATTR 0x3
-#define HWIO_EE_n_EV_CH_k_SCRATCH_0_INI2(n, k) in_dword_masked(	\
-		HWIO_EE_n_EV_CH_k_SCRATCH_0_ADDR(n, k),	\
-		HWIO_EE_n_EV_CH_k_SCRATCH_0_RMSK)
-#define HWIO_EE_n_EV_CH_k_SCRATCH_0_INMI2(n, k, mask) in_dword_masked( \
-		HWIO_EE_n_EV_CH_k_SCRATCH_0_ADDR(n, k),	\
-		mask)
-#define HWIO_EE_n_EV_CH_k_SCRATCH_0_OUTI2(n, k, val) out_dword(	\
-		HWIO_EE_n_EV_CH_k_SCRATCH_0_ADDR(n, k),	\
-		val)
-#define HWIO_EE_n_EV_CH_k_SCRATCH_0_OUTMI2(n, k, mask, \
-					   val) out_dword_masked_ns( \
-		HWIO_EE_n_EV_CH_k_SCRATCH_0_ADDR(n, \
-						 k), \
-		mask, \
-		val, \
-		HWIO_EE_n_EV_CH_k_SCRATCH_0_INI2(n, k))
-#define HWIO_EE_n_EV_CH_k_SCRATCH_0_SCRATCH_BMSK 0xffffffff
-#define HWIO_EE_n_EV_CH_k_SCRATCH_0_SCRATCH_SHFT 0x0
-#define HWIO_EE_n_EV_CH_k_SCRATCH_1_ADDR(n, k) (GSI_REG_BASE + \
-						0x0001004c + 0x4000 * \
-						(n) + 0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_SCRATCH_1_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-						0x0001004c + 0x4000 * \
-						(n) + 0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_SCRATCH_1_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-						0x0001004c + 0x4000 * \
-						(n) + 0x80 * (k))
-#define HWIO_EE_n_EV_CH_k_SCRATCH_1_RMSK 0xffffffff
-#define HWIO_EE_n_EV_CH_k_SCRATCH_1_MAXn 2
-#define HWIO_EE_n_EV_CH_k_SCRATCH_1_MAXk 19
-#define HWIO_EE_n_EV_CH_k_SCRATCH_1_ATTR 0x3
-#define HWIO_EE_n_EV_CH_k_SCRATCH_1_INI2(n, k) in_dword_masked(	\
-		HWIO_EE_n_EV_CH_k_SCRATCH_1_ADDR(n, k),	\
-		HWIO_EE_n_EV_CH_k_SCRATCH_1_RMSK)
-#define HWIO_EE_n_EV_CH_k_SCRATCH_1_INMI2(n, k, mask) in_dword_masked( \
-		HWIO_EE_n_EV_CH_k_SCRATCH_1_ADDR(n, k),	\
-		mask)
-#define HWIO_EE_n_EV_CH_k_SCRATCH_1_OUTI2(n, k, val) out_dword(	\
-		HWIO_EE_n_EV_CH_k_SCRATCH_1_ADDR(n, k),	\
-		val)
-#define HWIO_EE_n_EV_CH_k_SCRATCH_1_OUTMI2(n, k, mask, \
-					   val) out_dword_masked_ns( \
-		HWIO_EE_n_EV_CH_k_SCRATCH_1_ADDR(n, \
-						 k), \
-		mask, \
-		val, \
-		HWIO_EE_n_EV_CH_k_SCRATCH_1_INI2(n, k))
-#define HWIO_EE_n_EV_CH_k_SCRATCH_1_SCRATCH_BMSK 0xffffffff
-#define HWIO_EE_n_EV_CH_k_SCRATCH_1_SCRATCH_SHFT 0x0
-#define HWIO_EE_n_GSI_CH_k_DOORBELL_0_ADDR(n, k) (GSI_REG_BASE + \
-						  0x00011000 + 0x4000 *	\
-						  (n) + 0x8 * (k))
-#define HWIO_EE_n_GSI_CH_k_DOORBELL_0_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-						  0x00011000 + 0x4000 *	\
-						  (n) + 0x8 * (k))
-#define HWIO_EE_n_GSI_CH_k_DOORBELL_0_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-						  0x00011000 + 0x4000 *	\
-						  (n) + 0x8 * (k))
-#define HWIO_EE_n_GSI_CH_k_DOORBELL_1_ADDR(n, k) (GSI_REG_BASE + \
-						  0x00011004 + 0x4000 *	\
-						  (n) + 0x8 * (k))
-#define HWIO_EE_n_GSI_CH_k_DOORBELL_1_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-						  0x00011004 + 0x4000 *	\
-						  (n) + 0x8 * (k))
-#define HWIO_EE_n_GSI_CH_k_DOORBELL_1_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-						  0x00011004 + 0x4000 *	\
-						  (n) + 0x8 * (k))
-#define HWIO_EE_n_EV_CH_k_DOORBELL_0_ADDR(n, k) (GSI_REG_BASE +	\
-						 0x00011100 + 0x4000 * \
-						 (n) + 0x8 * (k))
-#define HWIO_EE_n_EV_CH_k_DOORBELL_0_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-						 0x00011100 + 0x4000 * \
-						 (n) + 0x8 * (k))
-#define HWIO_EE_n_EV_CH_k_DOORBELL_0_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-						 0x00011100 + 0x4000 * \
-						 (n) + 0x8 * (k))
-#define HWIO_EE_n_EV_CH_k_DOORBELL_1_ADDR(n, k) (GSI_REG_BASE +	\
-						 0x00011104 + 0x4000 * \
-						 (n) + 0x8 * (k))
-#define HWIO_EE_n_EV_CH_k_DOORBELL_1_PHYS(n, k) (GSI_REG_BASE_PHYS + \
-						 0x00011104 + 0x4000 * \
-						 (n) + 0x8 * (k))
-#define HWIO_EE_n_EV_CH_k_DOORBELL_1_OFFS(n, k) (GSI_REG_BASE_OFFS + \
-						 0x00011104 + 0x4000 * \
-						 (n) + 0x8 * (k))
-#define HWIO_EE_n_GSI_STATUS_ADDR(n) (GSI_REG_BASE + 0x00012000 + 0x4000 * \
-				      (n))
-#define HWIO_EE_n_GSI_STATUS_PHYS(n) (GSI_REG_BASE_PHYS + 0x00012000 + \
-				      0x4000 * (n))
-#define HWIO_EE_n_GSI_STATUS_OFFS(n) (GSI_REG_BASE_OFFS + 0x00012000 + \
-				      0x4000 * (n))
-#define HWIO_EE_n_GSI_STATUS_RMSK 0x1
-#define HWIO_EE_n_GSI_STATUS_MAXn 2
-#define HWIO_EE_n_GSI_STATUS_ATTR 0x1
-#define HWIO_EE_n_GSI_STATUS_INI(n) in_dword_masked( \
-		HWIO_EE_n_GSI_STATUS_ADDR(n), \
-		HWIO_EE_n_GSI_STATUS_RMSK)
-#define HWIO_EE_n_GSI_STATUS_INMI(n, mask) in_dword_masked( \
-		HWIO_EE_n_GSI_STATUS_ADDR(n), \
-		mask)
-#define HWIO_EE_n_GSI_STATUS_ENABLED_BMSK 0x1
-#define HWIO_EE_n_GSI_STATUS_ENABLED_SHFT 0x0
-#define HWIO_EE_n_GSI_CH_CMD_ADDR(n) (GSI_REG_BASE + 0x00012008 + 0x4000 * \
-				      (n))
-#define HWIO_EE_n_GSI_CH_CMD_PHYS(n) (GSI_REG_BASE_PHYS + 0x00012008 + \
-				      0x4000 * (n))
-#define HWIO_EE_n_GSI_CH_CMD_OFFS(n) (GSI_REG_BASE_OFFS + 0x00012008 + \
-				      0x4000 * (n))
-#define HWIO_EE_n_EV_CH_CMD_ADDR(n) (GSI_REG_BASE + 0x00012010 + 0x4000 * \
-				     (n))
-#define HWIO_EE_n_EV_CH_CMD_PHYS(n) (GSI_REG_BASE_PHYS + 0x00012010 + \
-				     0x4000 * (n))
-#define HWIO_EE_n_EV_CH_CMD_OFFS(n) (GSI_REG_BASE_OFFS + 0x00012010 + \
-				     0x4000 * (n))
-#define HWIO_EE_n_GSI_EE_GENERIC_CMD_ADDR(n) (GSI_REG_BASE + 0x00012018 + \
-					      0x4000 * (n))
-#define HWIO_EE_n_GSI_EE_GENERIC_CMD_PHYS(n) (GSI_REG_BASE_PHYS + \
-					      0x00012018 + 0x4000 * (n))
-#define HWIO_EE_n_GSI_EE_GENERIC_CMD_OFFS(n) (GSI_REG_BASE_OFFS + \
-					      0x00012018 + 0x4000 * (n))
-#define HWIO_EE_n_GSI_HW_PARAM_0_ADDR(n) (GSI_REG_BASE + 0x00012038 + \
-					  0x4000 * (n))
-#define HWIO_EE_n_GSI_HW_PARAM_0_PHYS(n) (GSI_REG_BASE_PHYS + 0x00012038 + \
-					  0x4000 * (n))
-#define HWIO_EE_n_GSI_HW_PARAM_0_OFFS(n) (GSI_REG_BASE_OFFS + 0x00012038 + \
-					  0x4000 * (n))
-#define HWIO_EE_n_GSI_HW_PARAM_1_ADDR(n) (GSI_REG_BASE + 0x0001203c + \
-					  0x4000 * (n))
-#define HWIO_EE_n_GSI_HW_PARAM_1_PHYS(n) (GSI_REG_BASE_PHYS + 0x0001203c + \
-					  0x4000 * (n))
-#define HWIO_EE_n_GSI_HW_PARAM_1_OFFS(n) (GSI_REG_BASE_OFFS + 0x0001203c + \
-					  0x4000 * (n))
-#define HWIO_EE_n_GSI_HW_PARAM_2_ADDR(n) (GSI_REG_BASE + 0x00012040 + \
-					  0x4000 * (n))
-#define HWIO_EE_n_GSI_HW_PARAM_2_PHYS(n) (GSI_REG_BASE_PHYS + 0x00012040 + \
-					  0x4000 * (n))
-#define HWIO_EE_n_GSI_HW_PARAM_2_OFFS(n) (GSI_REG_BASE_OFFS + 0x00012040 + \
-					  0x4000 * (n))
-#define HWIO_EE_n_GSI_SW_VERSION_ADDR(n) (GSI_REG_BASE + 0x00012044 + \
-					  0x4000 * (n))
-#define HWIO_EE_n_GSI_SW_VERSION_PHYS(n) (GSI_REG_BASE_PHYS + 0x00012044 + \
-					  0x4000 * (n))
-#define HWIO_EE_n_GSI_SW_VERSION_OFFS(n) (GSI_REG_BASE_OFFS + 0x00012044 + \
-					  0x4000 * (n))
-#define HWIO_EE_n_GSI_MCS_CODE_VER_ADDR(n) (GSI_REG_BASE + 0x00012048 +	\
-					    0x4000 * (n))
-#define HWIO_EE_n_GSI_MCS_CODE_VER_PHYS(n) (GSI_REG_BASE_PHYS +	\
-					    0x00012048 + 0x4000 * (n))
-#define HWIO_EE_n_GSI_MCS_CODE_VER_OFFS(n) (GSI_REG_BASE_OFFS +	\
-					    0x00012048 + 0x4000 * (n))
-#define HWIO_EE_n_GSI_HW_PARAM_3_ADDR(n) (GSI_REG_BASE + 0x0001204c + \
-					  0x4000 * (n))
-#define HWIO_EE_n_GSI_HW_PARAM_3_PHYS(n) (GSI_REG_BASE_PHYS + 0x0001204c + \
-					  0x4000 * (n))
-#define HWIO_EE_n_GSI_HW_PARAM_3_OFFS(n) (GSI_REG_BASE_OFFS + 0x0001204c + \
-					  0x4000 * (n))
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_ADDR(n) (GSI_REG_BASE + 0x00012080 + \
-					  0x4000 * (n))
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_PHYS(n) (GSI_REG_BASE_PHYS + 0x00012080 + \
-					  0x4000 * (n))
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_OFFS(n) (GSI_REG_BASE_OFFS + 0x00012080 + \
-					  0x4000 * (n))
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_RMSK 0x7f
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_MAXn 2
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_ATTR 0x1
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_INI(n) in_dword_masked( \
-		HWIO_EE_n_CNTXT_TYPE_IRQ_ADDR(n), \
-		HWIO_EE_n_CNTXT_TYPE_IRQ_RMSK)
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_INMI(n, mask) in_dword_masked(	\
-		HWIO_EE_n_CNTXT_TYPE_IRQ_ADDR(n), \
-		mask)
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_GENERAL_BMSK 0x40
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_GENERAL_SHFT 0x6
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_INTER_EE_EV_CTRL_BMSK 0x20
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_INTER_EE_EV_CTRL_SHFT 0x5
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_INTER_EE_CH_CTRL_BMSK 0x10
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_INTER_EE_CH_CTRL_SHFT 0x4
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_IEOB_BMSK 0x8
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_IEOB_SHFT 0x3
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_GLOB_EE_BMSK 0x4
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_GLOB_EE_SHFT 0x2
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_EV_CTRL_BMSK 0x2
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_EV_CTRL_SHFT 0x1
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_CH_CTRL_BMSK 0x1
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_CH_CTRL_SHFT 0x0
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_MSK_ADDR(n) (GSI_REG_BASE + 0x00012088 + \
-					      0x4000 * (n))
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_MSK_PHYS(n) (GSI_REG_BASE_PHYS + \
-					      0x00012088 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_MSK_OFFS(n) (GSI_REG_BASE_OFFS + \
-					      0x00012088 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_MSK_RMSK 0x7f
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_MSK_MAXn 2
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_MSK_ATTR 0x3
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_MSK_INI(n) in_dword_masked( \
-		HWIO_EE_n_CNTXT_TYPE_IRQ_MSK_ADDR(n), \
-		HWIO_EE_n_CNTXT_TYPE_IRQ_MSK_RMSK)
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_MSK_INMI(n, mask) in_dword_masked( \
-		HWIO_EE_n_CNTXT_TYPE_IRQ_MSK_ADDR(n), \
-		mask)
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_MSK_OUTI(n, val) out_dword( \
-		HWIO_EE_n_CNTXT_TYPE_IRQ_MSK_ADDR(n), \
-		val)
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_MSK_OUTMI(n, mask, \
-					   val) out_dword_masked_ns( \
-		HWIO_EE_n_CNTXT_TYPE_IRQ_MSK_ADDR( \
-			n), \
-		mask, \
-		val, \
-		HWIO_EE_n_CNTXT_TYPE_IRQ_MSK_INI(n))
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_MSK_GENERAL_BMSK 0x40
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_MSK_GENERAL_SHFT 0x6
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_MSK_INTER_EE_EV_CTRL_BMSK 0x20
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_MSK_INTER_EE_EV_CTRL_SHFT 0x5
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_MSK_INTER_EE_CH_CTRL_BMSK 0x10
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_MSK_INTER_EE_CH_CTRL_SHFT 0x4
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_MSK_IEOB_BMSK 0x8
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_MSK_IEOB_SHFT 0x3
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_MSK_GLOB_EE_BMSK 0x4
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_MSK_GLOB_EE_SHFT 0x2
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_MSK_EV_CTRL_BMSK 0x2
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_MSK_EV_CTRL_SHFT 0x1
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_MSK_CH_CTRL_BMSK 0x1
-#define HWIO_EE_n_CNTXT_TYPE_IRQ_MSK_CH_CTRL_SHFT 0x0
-#define HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_ADDR(n) (GSI_REG_BASE + \
-						0x00012090 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_PHYS(n) (GSI_REG_BASE_PHYS + \
-						0x00012090 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_OFFS(n) (GSI_REG_BASE_OFFS + \
-						0x00012090 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_RMSK 0xffffffff
-#define HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_MAXn 2
-#define HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_ATTR 0x1
-#define HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_INI(n) in_dword_masked( \
-		HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_ADDR(n),	\
-		HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_RMSK)
-#define HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_INMI(n, mask) in_dword_masked( \
-		HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_ADDR(n),	\
-		mask)
-#define HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_GSI_CH_BIT_MAP_BMSK 0xffffffff
-#define HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_GSI_CH_BIT_MAP_SHFT 0x0
-#define HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_ADDR(n) (GSI_REG_BASE + 0x00012094 + \
-					       0x4000 * (n))
-#define HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_PHYS(n) (GSI_REG_BASE_PHYS + \
-					       0x00012094 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_OFFS(n) (GSI_REG_BASE_OFFS + \
-					       0x00012094 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_RMSK 0xffffffff
-#define HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_MAXn 2
-#define HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_ATTR 0x1
-#define HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_INI(n) in_dword_masked( \
-		HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_ADDR(n), \
-		HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_RMSK)
-#define HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_INMI(n, mask) in_dword_masked( \
-		HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_ADDR(n), \
-		mask)
-#define HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_EV_CH_BIT_MAP_BMSK 0xffffffff
-#define HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_EV_CH_BIT_MAP_SHFT 0x0
-#define HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_ADDR(n) (GSI_REG_BASE + \
-						    0x00012098 + 0x4000 * \
-						    (n))
-#define HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_PHYS(n) (GSI_REG_BASE_PHYS +	\
-						    0x00012098 + 0x4000 * \
-						    (n))
-#define HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_OFFS(n) (GSI_REG_BASE_OFFS +	\
-						    0x00012098 + 0x4000 * \
-						    (n))
-#define HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_RMSK 0x7fffff
-#define HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_MAXn 2
-#define HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_ATTR 0x3
-#define HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_INI(n) in_dword_masked( \
-		HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_ADDR(n), \
-		HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_RMSK)
-#define HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_INMI(n, mask) in_dword_masked( \
-		HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_ADDR(n), \
-		mask)
-#define HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_OUTI(n, val) out_dword( \
-		HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_ADDR(n), \
-		val)
-#define HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_OUTMI(n, mask, \
-						 val) out_dword_masked_ns( \
-		HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_ADDR( \
-			n), \
-		mask, \
-		val, \
-		HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_INI(n))
-#define HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_GSI_CH_BIT_MAP_MSK_BMSK \
-	0x7fffff
-#define HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_GSI_CH_BIT_MAP_MSK_SHFT 0x0
-#define HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_ADDR(n) (GSI_REG_BASE + \
-						   0x0001209c + 0x4000 * \
-						   (n))
-#define HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_PHYS(n) (GSI_REG_BASE_PHYS + \
-						   0x0001209c + 0x4000 * \
-						   (n))
-#define HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_OFFS(n) (GSI_REG_BASE_OFFS + \
-						   0x0001209c + 0x4000 * \
-						   (n))
-#define HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_RMSK 0xfffff
-#define HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_MAXn 2
-#define HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_ATTR 0x3
-#define HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_INI(n) in_dword_masked( \
-		HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_ADDR(n), \
-		HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_RMSK)
-#define HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_INMI(n, mask) in_dword_masked( \
-		HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_ADDR(n), \
-		mask)
-#define HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_OUTI(n, val) out_dword( \
-		HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_ADDR(n), \
-		val)
-#define HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_OUTMI(n, mask, \
-						val) out_dword_masked_ns( \
-		HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_ADDR(	\
-			n), \
-		mask, \
-		val, \
-		HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_INI(n))
-#define HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_EV_CH_BIT_MAP_MSK_BMSK 0xfffff
-#define HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_EV_CH_BIT_MAP_MSK_SHFT 0x0
-#define HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_CLR_ADDR(n) (GSI_REG_BASE + \
-						    0x000120a0 + 0x4000 * \
-						    (n))
-#define HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_CLR_PHYS(n) (GSI_REG_BASE_PHYS +	\
-						    0x000120a0 + 0x4000 * \
-						    (n))
-#define HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_CLR_OFFS(n) (GSI_REG_BASE_OFFS +	\
-						    0x000120a0 + 0x4000 * \
-						    (n))
-#define HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_CLR_RMSK 0xffffffff
-#define HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_CLR_MAXn 2
-#define HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_CLR_ATTR 0x2
-#define HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_CLR_OUTI(n, val) out_dword( \
-		HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_CLR_ADDR(n), \
-		val)
-#define HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_CLR_GSI_CH_BIT_MAP_BMSK 0xffffffff
-#define HWIO_EE_n_CNTXT_SRC_GSI_CH_IRQ_CLR_GSI_CH_BIT_MAP_SHFT 0x0
-#define HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_CLR_ADDR(n) (GSI_REG_BASE + \
-						   0x000120a4 + 0x4000 * \
-						   (n))
-#define HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_CLR_PHYS(n) (GSI_REG_BASE_PHYS + \
-						   0x000120a4 + 0x4000 * \
-						   (n))
-#define HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_CLR_OFFS(n) (GSI_REG_BASE_OFFS + \
-						   0x000120a4 + 0x4000 * \
-						   (n))
-#define HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_CLR_RMSK 0xffffffff
-#define HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_CLR_MAXn 2
-#define HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_CLR_ATTR 0x2
-#define HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_CLR_OUTI(n, val) out_dword( \
-		HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_CLR_ADDR(n), \
-		val)
-#define HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_CLR_EV_CH_BIT_MAP_BMSK 0xffffffff
-#define HWIO_EE_n_CNTXT_SRC_EV_CH_IRQ_CLR_EV_CH_BIT_MAP_SHFT 0x0
-#define HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_ADDR(n) (GSI_REG_BASE + 0x000120b0 + \
-					      0x4000 * (n))
-#define HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_PHYS(n) (GSI_REG_BASE_PHYS + \
-					      0x000120b0 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_OFFS(n) (GSI_REG_BASE_OFFS + \
-					      0x000120b0 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_RMSK 0xffffffff
-#define HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_MAXn 2
-#define HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_ATTR 0x1
-#define HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_INI(n) in_dword_masked( \
-		HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_ADDR(n), \
-		HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_RMSK)
-#define HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_INMI(n, mask) in_dword_masked( \
-		HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_ADDR(n), \
-		mask)
-#define HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_EV_CH_BIT_MAP_BMSK 0xffffffff
-#define HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_EV_CH_BIT_MAP_SHFT 0x0
-#define HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_ADDR(n) (GSI_REG_BASE + \
-						  0x000120b8 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_PHYS(n) (GSI_REG_BASE_PHYS + \
-						  0x000120b8 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_OFFS(n) (GSI_REG_BASE_OFFS + \
-						  0x000120b8 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_RMSK 0xfffff
-#define HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_MAXn 2
-#define HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_ATTR 0x3
-#define HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_INI(n) in_dword_masked( \
-		HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_ADDR(n), \
-		HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_RMSK)
-#define HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_INMI(n, mask) in_dword_masked(	\
-		HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_ADDR(n), \
-		mask)
-#define HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_OUTI(n, val) out_dword( \
-		HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_ADDR(n), \
-		val)
-#define HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_OUTMI(n, mask,	\
-					       val) out_dword_masked_ns( \
-		HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_ADDR( \
-			n), \
-		mask, \
-		val, \
-		HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_INI(n))
-#define HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_EV_CH_BIT_MAP_MSK_BMSK 0xfffff
-#define HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_EV_CH_BIT_MAP_MSK_SHFT 0x0
-#define HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_CLR_ADDR(n) (GSI_REG_BASE + \
-						  0x000120c0 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_CLR_PHYS(n) (GSI_REG_BASE_PHYS + \
-						  0x000120c0 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_CLR_OFFS(n) (GSI_REG_BASE_OFFS + \
-						  0x000120c0 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_CLR_RMSK 0xffffffff
-#define HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_CLR_MAXn 2
-#define HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_CLR_ATTR 0x2
-#define HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_CLR_OUTI(n, val) out_dword( \
-		HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_CLR_ADDR(n), \
-		val)
-#define HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_CLR_EV_CH_BIT_MAP_BMSK 0xffffffff
-#define HWIO_EE_n_CNTXT_SRC_IEOB_IRQ_CLR_EV_CH_BIT_MAP_SHFT 0x0
-#define HWIO_EE_n_CNTXT_GLOB_IRQ_STTS_ADDR(n) (GSI_REG_BASE + 0x00012100 + \
-					       0x4000 * (n))
-#define HWIO_EE_n_CNTXT_GLOB_IRQ_STTS_PHYS(n) (GSI_REG_BASE_PHYS + \
-					       0x00012100 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_GLOB_IRQ_STTS_OFFS(n) (GSI_REG_BASE_OFFS + \
-					       0x00012100 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_GLOB_IRQ_STTS_RMSK 0xf
-#define HWIO_EE_n_CNTXT_GLOB_IRQ_STTS_MAXn 2
-#define HWIO_EE_n_CNTXT_GLOB_IRQ_STTS_ATTR 0x1
-#define HWIO_EE_n_CNTXT_GLOB_IRQ_STTS_INI(n) in_dword_masked( \
-		HWIO_EE_n_CNTXT_GLOB_IRQ_STTS_ADDR(n), \
-		HWIO_EE_n_CNTXT_GLOB_IRQ_STTS_RMSK)
-#define HWIO_EE_n_CNTXT_GLOB_IRQ_STTS_INMI(n, mask) in_dword_masked( \
-		HWIO_EE_n_CNTXT_GLOB_IRQ_STTS_ADDR(n), \
-		mask)
-#define HWIO_EE_n_CNTXT_GLOB_IRQ_STTS_GP_INT3_BMSK 0x8
-#define HWIO_EE_n_CNTXT_GLOB_IRQ_STTS_GP_INT3_SHFT 0x3
-#define HWIO_EE_n_CNTXT_GLOB_IRQ_STTS_GP_INT2_BMSK 0x4
-#define HWIO_EE_n_CNTXT_GLOB_IRQ_STTS_GP_INT2_SHFT 0x2
-#define HWIO_EE_n_CNTXT_GLOB_IRQ_STTS_GP_INT1_BMSK 0x2
-#define HWIO_EE_n_CNTXT_GLOB_IRQ_STTS_GP_INT1_SHFT 0x1
-#define HWIO_EE_n_CNTXT_GLOB_IRQ_STTS_ERROR_INT_BMSK 0x1
-#define HWIO_EE_n_CNTXT_GLOB_IRQ_STTS_ERROR_INT_SHFT 0x0
-#define HWIO_EE_n_CNTXT_GLOB_IRQ_EN_ADDR(n) (GSI_REG_BASE + 0x00012108 + \
-					     0x4000 * (n))
-#define HWIO_EE_n_CNTXT_GLOB_IRQ_EN_PHYS(n) (GSI_REG_BASE_PHYS + \
-					     0x00012108 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_GLOB_IRQ_EN_OFFS(n) (GSI_REG_BASE_OFFS + \
-					     0x00012108 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_GLOB_IRQ_CLR_ADDR(n) (GSI_REG_BASE + 0x00012110 + \
-					      0x4000 * (n))
-#define HWIO_EE_n_CNTXT_GLOB_IRQ_CLR_PHYS(n) (GSI_REG_BASE_PHYS + \
-					      0x00012110 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_GLOB_IRQ_CLR_OFFS(n) (GSI_REG_BASE_OFFS + \
-					      0x00012110 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_GSI_IRQ_STTS_ADDR(n) (GSI_REG_BASE + 0x00012118 + \
-					      0x4000 * (n))
-#define HWIO_EE_n_CNTXT_GSI_IRQ_STTS_PHYS(n) (GSI_REG_BASE_PHYS + \
-					      0x00012118 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_GSI_IRQ_STTS_OFFS(n) (GSI_REG_BASE_OFFS + \
-					      0x00012118 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_GSI_IRQ_STTS_RMSK 0xf
-#define HWIO_EE_n_CNTXT_GSI_IRQ_STTS_MAXn 2
-#define HWIO_EE_n_CNTXT_GSI_IRQ_STTS_ATTR 0x1
-#define HWIO_EE_n_CNTXT_GSI_IRQ_STTS_INI(n) in_dword_masked( \
-		HWIO_EE_n_CNTXT_GSI_IRQ_STTS_ADDR(n), \
-		HWIO_EE_n_CNTXT_GSI_IRQ_STTS_RMSK)
-#define HWIO_EE_n_CNTXT_GSI_IRQ_STTS_INMI(n, mask) in_dword_masked( \
-		HWIO_EE_n_CNTXT_GSI_IRQ_STTS_ADDR(n), \
-		mask)
-#define HWIO_EE_n_CNTXT_GSI_IRQ_STTS_GSI_MCS_STACK_OVRFLOW_BMSK 0x8
-#define HWIO_EE_n_CNTXT_GSI_IRQ_STTS_GSI_MCS_STACK_OVRFLOW_SHFT 0x3
-#define HWIO_EE_n_CNTXT_GSI_IRQ_STTS_GSI_CMD_FIFO_OVRFLOW_BMSK 0x4
-#define HWIO_EE_n_CNTXT_GSI_IRQ_STTS_GSI_CMD_FIFO_OVRFLOW_SHFT 0x2
-#define HWIO_EE_n_CNTXT_GSI_IRQ_STTS_GSI_BUS_ERROR_BMSK 0x2
-#define HWIO_EE_n_CNTXT_GSI_IRQ_STTS_GSI_BUS_ERROR_SHFT 0x1
-#define HWIO_EE_n_CNTXT_GSI_IRQ_STTS_GSI_BREAK_POINT_BMSK 0x1
-#define HWIO_EE_n_CNTXT_GSI_IRQ_STTS_GSI_BREAK_POINT_SHFT 0x0
-#define HWIO_EE_n_CNTXT_GSI_IRQ_EN_ADDR(n) (GSI_REG_BASE + 0x00012120 +	\
-					    0x4000 * (n))
-#define HWIO_EE_n_CNTXT_GSI_IRQ_EN_PHYS(n) (GSI_REG_BASE_PHYS +	\
-					    0x00012120 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_GSI_IRQ_EN_OFFS(n) (GSI_REG_BASE_OFFS +	\
-					    0x00012120 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_GSI_IRQ_CLR_ADDR(n) (GSI_REG_BASE + 0x00012128 + \
-					     0x4000 * (n))
-#define HWIO_EE_n_CNTXT_GSI_IRQ_CLR_PHYS(n) (GSI_REG_BASE_PHYS + \
-					     0x00012128 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_GSI_IRQ_CLR_OFFS(n) (GSI_REG_BASE_OFFS + \
-					     0x00012128 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_INTSET_ADDR(n) (GSI_REG_BASE + 0x00012180 + \
-					0x4000 * (n))
-#define HWIO_EE_n_CNTXT_INTSET_PHYS(n) (GSI_REG_BASE_PHYS + 0x00012180 + \
-					0x4000 * (n))
-#define HWIO_EE_n_CNTXT_INTSET_OFFS(n) (GSI_REG_BASE_OFFS + 0x00012180 + \
-					0x4000 * (n))
-#define HWIO_EE_n_CNTXT_INTSET_RMSK 0x1
-#define HWIO_EE_n_CNTXT_INTSET_MAXn 2
-#define HWIO_EE_n_CNTXT_INTSET_ATTR 0x3
-#define HWIO_EE_n_CNTXT_INTSET_INI(n) in_dword_masked( \
-		HWIO_EE_n_CNTXT_INTSET_ADDR(n),	\
-		HWIO_EE_n_CNTXT_INTSET_RMSK)
-#define HWIO_EE_n_CNTXT_INTSET_INMI(n, mask) in_dword_masked( \
-		HWIO_EE_n_CNTXT_INTSET_ADDR(n),	\
-		mask)
-#define HWIO_EE_n_CNTXT_INTSET_OUTI(n, val) out_dword( \
-		HWIO_EE_n_CNTXT_INTSET_ADDR(n),	\
-		val)
-#define HWIO_EE_n_CNTXT_INTSET_OUTMI(n, mask, val) out_dword_masked_ns(	\
-		HWIO_EE_n_CNTXT_INTSET_ADDR(n),	\
-		mask, \
-		val, \
-		HWIO_EE_n_CNTXT_INTSET_INI(n))
-#define HWIO_EE_n_CNTXT_INTSET_INTYPE_BMSK 0x1
-#define HWIO_EE_n_CNTXT_INTSET_INTYPE_SHFT 0x0
-#define HWIO_EE_n_CNTXT_INTSET_INTYPE_MSI_FVAL 0x0
-#define HWIO_EE_n_CNTXT_INTSET_INTYPE_IRQ_FVAL 0x1
-#define HWIO_EE_n_CNTXT_MSI_BASE_LSB_ADDR(n) (GSI_REG_BASE + 0x00012188 + \
-					      0x4000 * (n))
-#define HWIO_EE_n_CNTXT_MSI_BASE_LSB_PHYS(n) (GSI_REG_BASE_PHYS + \
-					      0x00012188 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_MSI_BASE_LSB_OFFS(n) (GSI_REG_BASE_OFFS + \
-					      0x00012188 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_MSI_BASE_LSB_RMSK 0xffffffff
-#define HWIO_EE_n_CNTXT_MSI_BASE_LSB_MAXn 2
-#define HWIO_EE_n_CNTXT_MSI_BASE_LSB_ATTR 0x3
-#define HWIO_EE_n_CNTXT_MSI_BASE_LSB_INI(n) in_dword_masked( \
-		HWIO_EE_n_CNTXT_MSI_BASE_LSB_ADDR(n), \
-		HWIO_EE_n_CNTXT_MSI_BASE_LSB_RMSK)
-#define HWIO_EE_n_CNTXT_MSI_BASE_LSB_INMI(n, mask) in_dword_masked( \
-		HWIO_EE_n_CNTXT_MSI_BASE_LSB_ADDR(n), \
-		mask)
-#define HWIO_EE_n_CNTXT_MSI_BASE_LSB_OUTI(n, val) out_dword( \
-		HWIO_EE_n_CNTXT_MSI_BASE_LSB_ADDR(n), \
-		val)
-#define HWIO_EE_n_CNTXT_MSI_BASE_LSB_OUTMI(n, mask, \
-					   val) out_dword_masked_ns( \
-		HWIO_EE_n_CNTXT_MSI_BASE_LSB_ADDR( \
-			n), \
-		mask, \
-		val, \
-		HWIO_EE_n_CNTXT_MSI_BASE_LSB_INI(n))
-#define HWIO_EE_n_CNTXT_MSI_BASE_LSB_MSI_ADDR_LSB_BMSK 0xffffffff
-#define HWIO_EE_n_CNTXT_MSI_BASE_LSB_MSI_ADDR_LSB_SHFT 0x0
-#define HWIO_EE_n_CNTXT_MSI_BASE_MSB_ADDR(n) (GSI_REG_BASE + 0x0001218c + \
-					      0x4000 * (n))
-#define HWIO_EE_n_CNTXT_MSI_BASE_MSB_PHYS(n) (GSI_REG_BASE_PHYS + \
-					      0x0001218c + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_MSI_BASE_MSB_OFFS(n) (GSI_REG_BASE_OFFS + \
-					      0x0001218c + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_MSI_BASE_MSB_RMSK 0xffffffff
-#define HWIO_EE_n_CNTXT_MSI_BASE_MSB_MAXn 2
-#define HWIO_EE_n_CNTXT_MSI_BASE_MSB_ATTR 0x3
-#define HWIO_EE_n_CNTXT_MSI_BASE_MSB_INI(n) in_dword_masked( \
-		HWIO_EE_n_CNTXT_MSI_BASE_MSB_ADDR(n), \
-		HWIO_EE_n_CNTXT_MSI_BASE_MSB_RMSK)
-#define HWIO_EE_n_CNTXT_MSI_BASE_MSB_INMI(n, mask) in_dword_masked( \
-		HWIO_EE_n_CNTXT_MSI_BASE_MSB_ADDR(n), \
-		mask)
-#define HWIO_EE_n_CNTXT_MSI_BASE_MSB_OUTI(n, val) out_dword( \
-		HWIO_EE_n_CNTXT_MSI_BASE_MSB_ADDR(n), \
-		val)
-#define HWIO_EE_n_CNTXT_MSI_BASE_MSB_OUTMI(n, mask, \
-					   val) out_dword_masked_ns( \
-		HWIO_EE_n_CNTXT_MSI_BASE_MSB_ADDR( \
-			n), \
-		mask, \
-		val, \
-		HWIO_EE_n_CNTXT_MSI_BASE_MSB_INI(n))
-#define HWIO_EE_n_CNTXT_MSI_BASE_MSB_MSI_ADDR_MSB_BMSK 0xffffffff
-#define HWIO_EE_n_CNTXT_MSI_BASE_MSB_MSI_ADDR_MSB_SHFT 0x0
-#define HWIO_EE_n_CNTXT_INT_VEC_ADDR(n) (GSI_REG_BASE + 0x00012190 + \
-					 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_INT_VEC_PHYS(n) (GSI_REG_BASE_PHYS + 0x00012190 + \
-					 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_INT_VEC_OFFS(n) (GSI_REG_BASE_OFFS + 0x00012190 + \
-					 0x4000 * (n))
-#define HWIO_EE_n_ERROR_LOG_ADDR(n) (GSI_REG_BASE + 0x00012200 + 0x4000 * \
-				     (n))
-#define HWIO_EE_n_ERROR_LOG_PHYS(n) (GSI_REG_BASE_PHYS + 0x00012200 + \
-				     0x4000 * (n))
-#define HWIO_EE_n_ERROR_LOG_OFFS(n) (GSI_REG_BASE_OFFS + 0x00012200 + \
-				     0x4000 * (n))
-#define HWIO_EE_n_ERROR_LOG_RMSK 0xffffffff
-#define HWIO_EE_n_ERROR_LOG_MAXn 2
-#define HWIO_EE_n_ERROR_LOG_ATTR 0x3
-#define HWIO_EE_n_ERROR_LOG_INI(n) in_dword_masked( \
-		HWIO_EE_n_ERROR_LOG_ADDR(n), \
-		HWIO_EE_n_ERROR_LOG_RMSK)
-#define HWIO_EE_n_ERROR_LOG_INMI(n, mask) in_dword_masked( \
-		HWIO_EE_n_ERROR_LOG_ADDR(n), \
-		mask)
-#define HWIO_EE_n_ERROR_LOG_OUTI(n, val) out_dword( \
-		HWIO_EE_n_ERROR_LOG_ADDR(n), \
-		val)
-#define HWIO_EE_n_ERROR_LOG_OUTMI(n, mask, val) out_dword_masked_ns( \
-		HWIO_EE_n_ERROR_LOG_ADDR(n), \
-		mask, \
-		val, \
-		HWIO_EE_n_ERROR_LOG_INI(n))
-#define HWIO_EE_n_ERROR_LOG_ERROR_LOG_BMSK 0xffffffff
-#define HWIO_EE_n_ERROR_LOG_ERROR_LOG_SHFT 0x0
-#define HWIO_EE_n_ERROR_LOG_CLR_ADDR(n) (GSI_REG_BASE + 0x00012210 + \
-					 0x4000 * (n))
-#define HWIO_EE_n_ERROR_LOG_CLR_PHYS(n) (GSI_REG_BASE_PHYS + 0x00012210 + \
-					 0x4000 * (n))
-#define HWIO_EE_n_ERROR_LOG_CLR_OFFS(n) (GSI_REG_BASE_OFFS + 0x00012210 + \
-					 0x4000 * (n))
-#define HWIO_EE_n_ERROR_LOG_CLR_RMSK 0xffffffff
-#define HWIO_EE_n_ERROR_LOG_CLR_MAXn 2
-#define HWIO_EE_n_ERROR_LOG_CLR_ATTR 0x2
-#define HWIO_EE_n_ERROR_LOG_CLR_OUTI(n, val) out_dword(	\
-		HWIO_EE_n_ERROR_LOG_CLR_ADDR(n), \
-		val)
-#define HWIO_EE_n_ERROR_LOG_CLR_ERROR_LOG_CLR_BMSK 0xffffffff
-#define HWIO_EE_n_ERROR_LOG_CLR_ERROR_LOG_CLR_SHFT 0x0
-#define HWIO_EE_n_CNTXT_SCRATCH_0_ADDR(n) (GSI_REG_BASE + 0x00012400 + \
-					   0x4000 * (n))
-#define HWIO_EE_n_CNTXT_SCRATCH_0_PHYS(n) (GSI_REG_BASE_PHYS + \
-					   0x00012400 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_SCRATCH_0_OFFS(n) (GSI_REG_BASE_OFFS + \
-					   0x00012400 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_SCRATCH_0_RMSK 0xffffffff
-#define HWIO_EE_n_CNTXT_SCRATCH_0_MAXn 2
-#define HWIO_EE_n_CNTXT_SCRATCH_0_ATTR 0x3
-#define HWIO_EE_n_CNTXT_SCRATCH_0_INI(n) in_dword_masked( \
-		HWIO_EE_n_CNTXT_SCRATCH_0_ADDR(n), \
-		HWIO_EE_n_CNTXT_SCRATCH_0_RMSK)
-#define HWIO_EE_n_CNTXT_SCRATCH_0_INMI(n, mask) in_dword_masked( \
-		HWIO_EE_n_CNTXT_SCRATCH_0_ADDR(n), \
-		mask)
-#define HWIO_EE_n_CNTXT_SCRATCH_0_OUTI(n, val) out_dword( \
-		HWIO_EE_n_CNTXT_SCRATCH_0_ADDR(n), \
-		val)
-#define HWIO_EE_n_CNTXT_SCRATCH_0_OUTMI(n, mask, val) out_dword_masked_ns( \
-		HWIO_EE_n_CNTXT_SCRATCH_0_ADDR(n), \
-		mask, \
-		val, \
-		HWIO_EE_n_CNTXT_SCRATCH_0_INI(n))
-#define HWIO_EE_n_CNTXT_SCRATCH_0_SCRATCH_BMSK 0xffffffff
-#define HWIO_EE_n_CNTXT_SCRATCH_0_SCRATCH_SHFT 0x0
-#define HWIO_EE_n_CNTXT_SCRATCH_1_ADDR(n) (GSI_REG_BASE + 0x00012404 + \
-					   0x4000 * (n))
-#define HWIO_EE_n_CNTXT_SCRATCH_1_PHYS(n) (GSI_REG_BASE_PHYS + \
-					   0x00012404 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_SCRATCH_1_OFFS(n) (GSI_REG_BASE_OFFS + \
-					   0x00012404 + 0x4000 * (n))
-#define HWIO_EE_n_CNTXT_SCRATCH_1_RMSK 0xffffffff
-#define HWIO_EE_n_CNTXT_SCRATCH_1_MAXn 2
-#define HWIO_EE_n_CNTXT_SCRATCH_1_ATTR 0x3
-#define HWIO_EE_n_CNTXT_SCRATCH_1_INI(n) in_dword_masked( \
-		HWIO_EE_n_CNTXT_SCRATCH_1_ADDR(n), \
-		HWIO_EE_n_CNTXT_SCRATCH_1_RMSK)
-#define HWIO_EE_n_CNTXT_SCRATCH_1_INMI(n, mask) in_dword_masked( \
-		HWIO_EE_n_CNTXT_SCRATCH_1_ADDR(n), \
-		mask)
-#define HWIO_EE_n_CNTXT_SCRATCH_1_OUTI(n, val) out_dword( \
-		HWIO_EE_n_CNTXT_SCRATCH_1_ADDR(n), \
-		val)
-#define HWIO_EE_n_CNTXT_SCRATCH_1_OUTMI(n, mask, val) out_dword_masked_ns( \
-		HWIO_EE_n_CNTXT_SCRATCH_1_ADDR(n), \
-		mask, \
-		val, \
-		HWIO_EE_n_CNTXT_SCRATCH_1_INI(n))
-#define HWIO_EE_n_CNTXT_SCRATCH_1_SCRATCH_BMSK 0xffffffff
-#define HWIO_EE_n_CNTXT_SCRATCH_1_SCRATCH_SHFT 0x0
-#define HWIO_GSI_MCS_CFG_ADDR (GSI_REG_BASE + 0x0000b000)
-#define HWIO_GSI_MCS_CFG_PHYS (GSI_REG_BASE_PHYS + 0x0000b000)
-#define HWIO_GSI_MCS_CFG_OFFS (GSI_REG_BASE_OFFS + 0x0000b000)
-#define HWIO_GSI_TZ_FW_AUTH_LOCK_ADDR (GSI_REG_BASE + 0x0000b008)
-#define HWIO_GSI_TZ_FW_AUTH_LOCK_PHYS (GSI_REG_BASE_PHYS + 0x0000b008)
-#define HWIO_GSI_TZ_FW_AUTH_LOCK_OFFS (GSI_REG_BASE_OFFS + 0x0000b008)
-#define HWIO_GSI_MSA_FW_AUTH_LOCK_ADDR (GSI_REG_BASE + 0x0000b010)
-#define HWIO_GSI_MSA_FW_AUTH_LOCK_PHYS (GSI_REG_BASE_PHYS + 0x0000b010)
-#define HWIO_GSI_MSA_FW_AUTH_LOCK_OFFS (GSI_REG_BASE_OFFS + 0x0000b010)
-#define HWIO_GSI_SP_FW_AUTH_LOCK_ADDR (GSI_REG_BASE + 0x0000b018)
-#define HWIO_GSI_SP_FW_AUTH_LOCK_PHYS (GSI_REG_BASE_PHYS + 0x0000b018)
-#define HWIO_GSI_SP_FW_AUTH_LOCK_OFFS (GSI_REG_BASE_OFFS + 0x0000b018)
-#define HWIO_INTER_EE_n_ORIGINATOR_EE_ADDR(n) (GSI_REG_BASE + 0x0000c000 + \
-					       0x1000 * (n))
-#define HWIO_INTER_EE_n_ORIGINATOR_EE_PHYS(n) (GSI_REG_BASE_PHYS + \
-					       0x0000c000 + 0x1000 * (n))
-#define HWIO_INTER_EE_n_ORIGINATOR_EE_OFFS(n) (GSI_REG_BASE_OFFS + \
-					       0x0000c000 + 0x1000 * (n))
-#define HWIO_INTER_EE_n_GSI_CH_CMD_ADDR(n) (GSI_REG_BASE + 0x0000c008 +	\
-					    0x1000 * (n))
-#define HWIO_INTER_EE_n_GSI_CH_CMD_PHYS(n) (GSI_REG_BASE_PHYS +	\
-					    0x0000c008 + 0x1000 * (n))
-#define HWIO_INTER_EE_n_GSI_CH_CMD_OFFS(n) (GSI_REG_BASE_OFFS +	\
-					    0x0000c008 + 0x1000 * (n))
-#define HWIO_INTER_EE_n_EV_CH_CMD_ADDR(n) (GSI_REG_BASE + 0x0000c010 + \
-					   0x1000 * (n))
-#define HWIO_INTER_EE_n_EV_CH_CMD_PHYS(n) (GSI_REG_BASE_PHYS + \
-					   0x0000c010 + 0x1000 * (n))
-#define HWIO_INTER_EE_n_EV_CH_CMD_OFFS(n) (GSI_REG_BASE_OFFS + \
-					   0x0000c010 + 0x1000 * (n))
-#define HWIO_INTER_EE_n_SRC_GSI_CH_IRQ_ADDR(n) (GSI_REG_BASE + \
-						0x0000c018 + 0x1000 * (n))
-#define HWIO_INTER_EE_n_SRC_GSI_CH_IRQ_PHYS(n) (GSI_REG_BASE_PHYS + \
-						0x0000c018 + 0x1000 * (n))
-#define HWIO_INTER_EE_n_SRC_GSI_CH_IRQ_OFFS(n) (GSI_REG_BASE_OFFS + \
-						0x0000c018 + 0x1000 * (n))
-#define HWIO_INTER_EE_n_SRC_EV_CH_IRQ_ADDR(n) (GSI_REG_BASE + 0x0000c01c + \
-					       0x1000 * (n))
-#define HWIO_INTER_EE_n_SRC_EV_CH_IRQ_PHYS(n) (GSI_REG_BASE_PHYS + \
-					       0x0000c01c + 0x1000 * (n))
-#define HWIO_INTER_EE_n_SRC_EV_CH_IRQ_OFFS(n) (GSI_REG_BASE_OFFS + \
-					       0x0000c01c + 0x1000 * (n))
-#define HWIO_INTER_EE_n_SRC_GSI_CH_IRQ_MSK_ADDR(n) (GSI_REG_BASE + \
-						    0x0000c020 + 0x1000 * \
-						    (n))
-#define HWIO_INTER_EE_n_SRC_GSI_CH_IRQ_MSK_PHYS(n) (GSI_REG_BASE_PHYS +	\
-						    0x0000c020 + 0x1000 * \
-						    (n))
-#define HWIO_INTER_EE_n_SRC_GSI_CH_IRQ_MSK_OFFS(n) (GSI_REG_BASE_OFFS +	\
-						    0x0000c020 + 0x1000 * \
-						    (n))
-#define HWIO_INTER_EE_n_SRC_EV_CH_IRQ_MSK_ADDR(n) (GSI_REG_BASE + \
-						   0x0000c024 + 0x1000 * \
-						   (n))
-#define HWIO_INTER_EE_n_SRC_EV_CH_IRQ_MSK_PHYS(n) (GSI_REG_BASE_PHYS + \
-						   0x0000c024 + 0x1000 * \
-						   (n))
-#define HWIO_INTER_EE_n_SRC_EV_CH_IRQ_MSK_OFFS(n) (GSI_REG_BASE_OFFS + \
-						   0x0000c024 + 0x1000 * \
-						   (n))
-#define HWIO_INTER_EE_n_SRC_GSI_CH_IRQ_CLR_ADDR(n) (GSI_REG_BASE + \
-						    0x0000c028 + 0x1000 * \
-						    (n))
-#define HWIO_INTER_EE_n_SRC_GSI_CH_IRQ_CLR_PHYS(n) (GSI_REG_BASE_PHYS +	\
-						    0x0000c028 + 0x1000 * \
-						    (n))
-#define HWIO_INTER_EE_n_SRC_GSI_CH_IRQ_CLR_OFFS(n) (GSI_REG_BASE_OFFS +	\
-						    0x0000c028 + 0x1000 * \
-						    (n))
-#define HWIO_INTER_EE_n_SRC_EV_CH_IRQ_CLR_ADDR(n) (GSI_REG_BASE + \
-						   0x0000c02c + 0x1000 * \
-						   (n))
-#define HWIO_INTER_EE_n_SRC_EV_CH_IRQ_CLR_PHYS(n) (GSI_REG_BASE_PHYS + \
-						   0x0000c02c + 0x1000 * \
-						   (n))
-#define HWIO_INTER_EE_n_SRC_EV_CH_IRQ_CLR_OFFS(n) (GSI_REG_BASE_OFFS + \
-						   0x0000c02c + 0x1000 * \
-						   (n))
-#endif

ipa/ipa_v3/dump/ipa4.5/gsi_hwio_def.h

@@ -1,530 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2019, The Linux Foundation. All rights reserved.
- */
-#if !defined(_GSI_HWIO_DEF_H_)
-#define _GSI_HWIO_DEF_H_
-struct gsi_hwio_def_gsi_cfg_s {
-	u32	gsi_enable : 1;
-	u32	mcs_enable : 1;
-	u32	double_mcs_clk_freq : 1;
-	u32	uc_is_mcs : 1;
-	u32	gsi_pwr_clps : 1;
-	u32	bp_mtrix_disable : 1;
-	u32	reserved0 : 2;
-	u32	sleep_clk_div : 4;
-	u32	reserved1 : 20;
-};
-union gsi_hwio_def_gsi_cfg_u {
-	struct gsi_hwio_def_gsi_cfg_s	def;
-	u32				value;
-};
-struct gsi_hwio_def_gsi_ree_cfg_s {
-	u32	move_to_esc_clr_mode_trsh : 1;
-	u32	channel_empty_int_enable : 1;
-	u32	reserved0 : 6;
-	u32	max_burst_size : 8;
-	u32	reserved1 : 16;
-};
-union gsi_hwio_def_gsi_ree_cfg_u {
-	struct gsi_hwio_def_gsi_ree_cfg_s	def;
-	u32					value;
-};
-struct gsi_hwio_def_gsi_manager_ee_qos_n_s {
-	u32	ee_prio : 2;
-	u32	reserved0 : 6;
-	u32	max_ch_alloc : 5;
-	u32	reserved1 : 3;
-	u32	max_ev_alloc : 5;
-	u32	reserved2 : 11;
-};
-union gsi_hwio_def_gsi_manager_ee_qos_n_u {
-	struct gsi_hwio_def_gsi_manager_ee_qos_n_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_gsi_shram_n_s {
-	u32 shram : 32;
-};
-union gsi_hwio_def_gsi_shram_n_u {
-	struct gsi_hwio_def_gsi_shram_n_s	def;
-	u32					value;
-};
-struct gsi_hwio_def_gsi_map_ee_n_ch_k_vp_table_s {
-	u32	phy_ch : 5;
-	u32	valid : 1;
-	u32	reserved0 : 26;
-};
-union gsi_hwio_def_gsi_map_ee_n_ch_k_vp_table_u {
-	struct gsi_hwio_def_gsi_map_ee_n_ch_k_vp_table_s	def;
-	u32							value;
-};
-struct gsi_hwio_def_gsi_test_bus_sel_s {
-	u32	gsi_testbus_sel : 8;
-	u32	reserved0 : 8;
-	u32	gsi_hw_events_sel : 4;
-	u32	reserved1 : 12;
-};
-union gsi_hwio_def_gsi_test_bus_sel_u {
-	struct gsi_hwio_def_gsi_test_bus_sel_s	def;
-	u32					value;
-};
-struct gsi_hwio_def_gsi_test_bus_reg_s {
-	u32 gsi_testbus_reg : 32;
-};
-union gsi_hwio_def_gsi_test_bus_reg_u {
-	struct gsi_hwio_def_gsi_test_bus_reg_s	def;
-	u32					value;
-};
-struct gsi_hwio_def_gsi_debug_countern_s {
-	u32	counter_value : 16;
-	u32	reserved0 : 16;
-};
-union gsi_hwio_def_gsi_debug_countern_u {
-	struct gsi_hwio_def_gsi_debug_countern_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_gsi_debug_qsb_log_last_misc_idn_s {
-	u32 addr_20_0 : 21;
-	u32 write : 1;
-	u32 tid : 5;
-	u32 mid : 5;
-};
-union gsi_hwio_def_gsi_debug_qsb_log_last_misc_idn_u {
-	struct gsi_hwio_def_gsi_debug_qsb_log_last_misc_idn_s   def;
-	u32						value;
-};
-struct gsi_hwio_def_gsi_debug_sw_rf_n_read_s {
-	u32 rf_reg : 32;
-};
-union gsi_hwio_def_gsi_debug_sw_rf_n_read_u {
-	struct gsi_hwio_def_gsi_debug_sw_rf_n_read_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_gsi_debug_ee_n_ev_k_vp_table_s {
-	u32	phy_ev_ch : 5;
-	u32	valid : 1;
-	u32	reserved0 : 26;
-};
-union gsi_hwio_def_gsi_debug_ee_n_ev_k_vp_table_u {
-	struct gsi_hwio_def_gsi_debug_ee_n_ev_k_vp_table_s	def;
-	u32							value;
-};
-struct gsi_hwio_def_ee_n_gsi_ch_k_cntxt_0_s {
-	u32	chtype_protocol : 3;
-	u32	chtype_dir : 1;
-	u32	ee : 4;
-	u32	chid : 5;
-	u32	chtype_protocol_msb : 1;
-	u32	erindex : 5;
-	u32	reserved0 : 1;
-	u32	chstate : 4;
-	u32	element_size : 8;
-};
-union gsi_hwio_def_ee_n_gsi_ch_k_cntxt_0_u {
-	struct gsi_hwio_def_ee_n_gsi_ch_k_cntxt_0_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_gsi_ch_k_cntxt_1_s {
-	u32	r_length : 16;
-	u32	reserved0 : 16;
-};
-union gsi_hwio_def_ee_n_gsi_ch_k_cntxt_1_u {
-	struct gsi_hwio_def_ee_n_gsi_ch_k_cntxt_1_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_gsi_ch_k_cntxt_2_s {
-	u32 r_base_addr_lsbs : 32;
-};
-union gsi_hwio_def_ee_n_gsi_ch_k_cntxt_2_u {
-	struct gsi_hwio_def_ee_n_gsi_ch_k_cntxt_2_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_gsi_ch_k_cntxt_3_s {
-	u32 r_base_addr_msbs : 32;
-};
-union gsi_hwio_def_ee_n_gsi_ch_k_cntxt_3_u {
-	struct gsi_hwio_def_ee_n_gsi_ch_k_cntxt_3_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_gsi_ch_k_cntxt_4_s {
-	u32 read_ptr_lsb : 32;
-};
-union gsi_hwio_def_ee_n_gsi_ch_k_cntxt_4_u {
-	struct gsi_hwio_def_ee_n_gsi_ch_k_cntxt_4_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_gsi_ch_k_cntxt_5_s {
-	u32 read_ptr_msb : 32;
-};
-union gsi_hwio_def_ee_n_gsi_ch_k_cntxt_5_u {
-	struct gsi_hwio_def_ee_n_gsi_ch_k_cntxt_5_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_gsi_ch_k_cntxt_6_s {
-	u32 write_ptr_lsb : 32;
-};
-union gsi_hwio_def_ee_n_gsi_ch_k_cntxt_6_u {
-	struct gsi_hwio_def_ee_n_gsi_ch_k_cntxt_6_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_gsi_ch_k_cntxt_7_s {
-	u32 write_ptr_msb : 32;
-};
-union gsi_hwio_def_ee_n_gsi_ch_k_cntxt_7_u {
-	struct gsi_hwio_def_ee_n_gsi_ch_k_cntxt_7_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_gsi_ch_k_re_fetch_read_ptr_s {
-	u32	read_ptr : 16;
-	u32	reserved0 : 16;
-};
-union gsi_hwio_def_ee_n_gsi_ch_k_re_fetch_read_ptr_u {
-	struct gsi_hwio_def_ee_n_gsi_ch_k_re_fetch_read_ptr_s	def;
-	u32							value;
-};
-struct gsi_hwio_def_ee_n_gsi_ch_k_re_fetch_write_ptr_s {
-	u32	re_intr_db : 16;
-	u32	reserved0 : 16;
-};
-union gsi_hwio_def_ee_n_gsi_ch_k_re_fetch_write_ptr_u {
-	struct gsi_hwio_def_ee_n_gsi_ch_k_re_fetch_write_ptr_s	def;
-	u32							value;
-};
-struct gsi_hwio_def_ee_n_gsi_ch_k_qos_s {
-	u32	wrr_weight : 4;
-	u32	reserved0 : 4;
-	u32	max_prefetch : 1;
-	u32	use_db_eng : 1;
-	u32	prefetch_mode : 4;
-	u32	reserved1 : 2;
-	u32	empty_lvl_thrshold : 8;
-	u32	reserved2 : 8;
-};
-union gsi_hwio_def_ee_n_gsi_ch_k_qos_u {
-	struct gsi_hwio_def_ee_n_gsi_ch_k_qos_s def;
-	u32					value;
-};
-struct gsi_hwio_def_ee_n_gsi_ch_k_scratch_0_s {
-	u32 scratch : 32;
-};
-union gsi_hwio_def_ee_n_gsi_ch_k_scratch_0_u {
-	struct gsi_hwio_def_ee_n_gsi_ch_k_scratch_0_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_gsi_ch_k_scratch_1_s {
-	u32 scratch : 32;
-};
-union gsi_hwio_def_ee_n_gsi_ch_k_scratch_1_u {
-	struct gsi_hwio_def_ee_n_gsi_ch_k_scratch_1_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_gsi_ch_k_scratch_2_s {
-	u32 scratch : 32;
-};
-union gsi_hwio_def_ee_n_gsi_ch_k_scratch_2_u {
-	struct gsi_hwio_def_ee_n_gsi_ch_k_scratch_2_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_gsi_ch_k_scratch_3_s {
-	u32 scratch : 32;
-};
-union gsi_hwio_def_ee_n_gsi_ch_k_scratch_3_u {
-	struct gsi_hwio_def_ee_n_gsi_ch_k_scratch_3_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_0_s {
-	u32	chtype : 4;
-	u32	ee : 4;
-	u32	evchid : 8;
-	u32	intype : 1;
-	u32	reserved0 : 3;
-	u32	chstate : 4;
-	u32	element_size : 8;
-};
-union gsi_hwio_def_ee_n_ev_ch_k_cntxt_0_u {
-	struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_0_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_1_s {
-	u32	r_length : 16;
-	u32	reserved0 : 16;
-};
-union gsi_hwio_def_ee_n_ev_ch_k_cntxt_1_u {
-	struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_1_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_2_s {
-	u32 r_base_addr_lsbs : 32;
-};
-union gsi_hwio_def_ee_n_ev_ch_k_cntxt_2_u {
-	struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_2_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_3_s {
-	u32 r_base_addr_msbs : 32;
-};
-union gsi_hwio_def_ee_n_ev_ch_k_cntxt_3_u {
-	struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_3_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_4_s {
-	u32 read_ptr_lsb : 32;
-};
-union gsi_hwio_def_ee_n_ev_ch_k_cntxt_4_u {
-	struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_4_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_5_s {
-	u32 read_ptr_msb : 32;
-};
-union gsi_hwio_def_ee_n_ev_ch_k_cntxt_5_u {
-	struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_5_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_6_s {
-	u32 write_ptr_lsb : 32;
-};
-union gsi_hwio_def_ee_n_ev_ch_k_cntxt_6_u {
-	struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_6_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_7_s {
-	u32 write_ptr_msb : 32;
-};
-union gsi_hwio_def_ee_n_ev_ch_k_cntxt_7_u {
-	struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_7_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_8_s {
-	u32	int_modt : 16;
-	u32	int_modc : 8;
-	u32	int_mod_cnt : 8;
-};
-union gsi_hwio_def_ee_n_ev_ch_k_cntxt_8_u {
-	struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_8_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_9_s {
-	u32 intvec : 32;
-};
-union gsi_hwio_def_ee_n_ev_ch_k_cntxt_9_u {
-	struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_9_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_10_s {
-	u32 msi_addr_lsb : 32;
-};
-union gsi_hwio_def_ee_n_ev_ch_k_cntxt_10_u {
-	struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_10_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_11_s {
-	u32 msi_addr_msb : 32;
-};
-union gsi_hwio_def_ee_n_ev_ch_k_cntxt_11_u {
-	struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_11_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_12_s {
-	u32 rp_update_addr_lsb : 32;
-};
-union gsi_hwio_def_ee_n_ev_ch_k_cntxt_12_u {
-	struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_12_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_13_s {
-	u32 rp_update_addr_msb : 32;
-};
-union gsi_hwio_def_ee_n_ev_ch_k_cntxt_13_u {
-	struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_13_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_ev_ch_k_scratch_0_s {
-	u32 scratch : 32;
-};
-union gsi_hwio_def_ee_n_ev_ch_k_scratch_0_u {
-	struct gsi_hwio_def_ee_n_ev_ch_k_scratch_0_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_ev_ch_k_scratch_1_s {
-	u32 scratch : 32;
-};
-union gsi_hwio_def_ee_n_ev_ch_k_scratch_1_u {
-	struct gsi_hwio_def_ee_n_ev_ch_k_scratch_1_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_gsi_status_s {
-	u32	enabled : 1;
-	u32	reserved0 : 31;
-};
-union gsi_hwio_def_ee_n_gsi_status_u {
-	struct gsi_hwio_def_ee_n_gsi_status_s	def;
-	u32					value;
-};
-struct gsi_hwio_def_ee_n_cntxt_type_irq_s {
-	u32	ch_ctrl : 1;
-	u32	ev_ctrl : 1;
-	u32	glob_ee : 1;
-	u32	ieob : 1;
-	u32	inter_ee_ch_ctrl : 1;
-	u32	inter_ee_ev_ctrl : 1;
-	u32	general : 1;
-	u32	reserved0 : 25;
-};
-union gsi_hwio_def_ee_n_cntxt_type_irq_u {
-	struct gsi_hwio_def_ee_n_cntxt_type_irq_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_cntxt_type_irq_msk_s {
-	u32	ch_ctrl : 1;
-	u32	ev_ctrl : 1;
-	u32	glob_ee : 1;
-	u32	ieob : 1;
-	u32	inter_ee_ch_ctrl : 1;
-	u32	inter_ee_ev_ctrl : 1;
-	u32	general : 1;
-	u32	reserved0 : 25;
-};
-union gsi_hwio_def_ee_n_cntxt_type_irq_msk_u {
-	struct gsi_hwio_def_ee_n_cntxt_type_irq_msk_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_cntxt_src_gsi_ch_irq_s {
-	u32 gsi_ch_bit_map : 32;
-};
-union gsi_hwio_def_ee_n_cntxt_src_gsi_ch_irq_u {
-	struct gsi_hwio_def_ee_n_cntxt_src_gsi_ch_irq_s def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_cntxt_src_ev_ch_irq_s {
-	u32 ev_ch_bit_map : 32;
-};
-union gsi_hwio_def_ee_n_cntxt_src_ev_ch_irq_u {
-	struct gsi_hwio_def_ee_n_cntxt_src_ev_ch_irq_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_cntxt_src_gsi_ch_irq_msk_s {
-	u32	gsi_ch_bit_map_msk : 23;
-	u32	reserved0 : 9;
-};
-union gsi_hwio_def_ee_n_cntxt_src_gsi_ch_irq_msk_u {
-	struct gsi_hwio_def_ee_n_cntxt_src_gsi_ch_irq_msk_s	def;
-	u32							value;
-};
-struct gsi_hwio_def_ee_n_cntxt_src_ev_ch_irq_msk_s {
-	u32	ev_ch_bit_map_msk : 20;
-	u32	reserved0 : 12;
-};
-union gsi_hwio_def_ee_n_cntxt_src_ev_ch_irq_msk_u {
-	struct gsi_hwio_def_ee_n_cntxt_src_ev_ch_irq_msk_s	def;
-	u32							value;
-};
-struct gsi_hwio_def_ee_n_cntxt_src_gsi_ch_irq_clr_s {
-	u32 gsi_ch_bit_map : 32;
-};
-union gsi_hwio_def_ee_n_cntxt_src_gsi_ch_irq_clr_u {
-	struct gsi_hwio_def_ee_n_cntxt_src_gsi_ch_irq_clr_s	def;
-	u32							value;
-};
-struct gsi_hwio_def_ee_n_cntxt_src_ev_ch_irq_clr_s {
-	u32 ev_ch_bit_map : 32;
-};
-union gsi_hwio_def_ee_n_cntxt_src_ev_ch_irq_clr_u {
-	struct gsi_hwio_def_ee_n_cntxt_src_ev_ch_irq_clr_s	def;
-	u32							value;
-};
-struct gsi_hwio_def_ee_n_cntxt_src_ieob_irq_s {
-	u32 ev_ch_bit_map : 32;
-};
-union gsi_hwio_def_ee_n_cntxt_src_ieob_irq_u {
-	struct gsi_hwio_def_ee_n_cntxt_src_ieob_irq_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_cntxt_src_ieob_irq_msk_s {
-	u32	ev_ch_bit_map_msk : 20;
-	u32	reserved0 : 12;
-};
-union gsi_hwio_def_ee_n_cntxt_src_ieob_irq_msk_u {
-	struct gsi_hwio_def_ee_n_cntxt_src_ieob_irq_msk_s	def;
-	u32							value;
-};
-struct gsi_hwio_def_ee_n_cntxt_src_ieob_irq_clr_s {
-	u32 ev_ch_bit_map : 32;
-};
-union gsi_hwio_def_ee_n_cntxt_src_ieob_irq_clr_u {
-	struct gsi_hwio_def_ee_n_cntxt_src_ieob_irq_clr_s	def;
-	u32							value;
-};
-struct gsi_hwio_def_ee_n_cntxt_glob_irq_stts_s {
-	u32	error_int : 1;
-	u32	gp_int1 : 1;
-	u32	gp_int2 : 1;
-	u32	gp_int3 : 1;
-	u32	reserved0 : 28;
-};
-union gsi_hwio_def_ee_n_cntxt_glob_irq_stts_u {
-	struct gsi_hwio_def_ee_n_cntxt_glob_irq_stts_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_cntxt_gsi_irq_stts_s {
-	u32	gsi_break_point : 1;
-	u32	gsi_bus_error : 1;
-	u32	gsi_cmd_fifo_ovrflow : 1;
-	u32	gsi_mcs_stack_ovrflow : 1;
-	u32	reserved0 : 28;
-};
-union gsi_hwio_def_ee_n_cntxt_gsi_irq_stts_u {
-	struct gsi_hwio_def_ee_n_cntxt_gsi_irq_stts_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_cntxt_intset_s {
-	u32	intype : 1;
-	u32	reserved0 : 31;
-};
-union gsi_hwio_def_ee_n_cntxt_intset_u {
-	struct gsi_hwio_def_ee_n_cntxt_intset_s def;
-	u32					value;
-};
-struct gsi_hwio_def_ee_n_cntxt_msi_base_lsb_s {
-	u32 msi_addr_lsb : 32;
-};
-union gsi_hwio_def_ee_n_cntxt_msi_base_lsb_u {
-	struct gsi_hwio_def_ee_n_cntxt_msi_base_lsb_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_cntxt_msi_base_msb_s {
-	u32 msi_addr_msb : 32;
-};
-union gsi_hwio_def_ee_n_cntxt_msi_base_msb_u {
-	struct gsi_hwio_def_ee_n_cntxt_msi_base_msb_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_error_log_s {
-	u32 error_log : 32;
-};
-union gsi_hwio_def_ee_n_error_log_u {
-	struct gsi_hwio_def_ee_n_error_log_s	def;
-	u32					value;
-};
-struct gsi_hwio_def_ee_n_error_log_clr_s {
-	u32 error_log_clr : 32;
-};
-union gsi_hwio_def_ee_n_error_log_clr_u {
-	struct gsi_hwio_def_ee_n_error_log_clr_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_cntxt_scratch_0_s {
-	u32 scratch : 32;
-};
-union gsi_hwio_def_ee_n_cntxt_scratch_0_u {
-	struct gsi_hwio_def_ee_n_cntxt_scratch_0_s	def;
-	u32						value;
-};
-struct gsi_hwio_def_ee_n_cntxt_scratch_1_s {
-	u32 scratch : 32;
-};
-union gsi_hwio_def_ee_n_cntxt_scratch_1_u {
-	struct gsi_hwio_def_ee_n_cntxt_scratch_1_s	def;
-	u32						value;
-};
-#endif

ipa/ipa_v3/dump/ipa4.5/ipa_access_control.h

@@ -1,42 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2019, The Linux Foundation. All rights reserved.
- */
-#if !defined(_IPA_ACCESS_CONTROL_H_)
-#define _IPA_ACCESS_CONTROL_H_
-
-#include "ipa_reg_dump.h"
-
-/*
- * The following is target specific.
- */
-static struct reg_mem_access_map_t mem_access_map[] = {
-	/*------------------------------------------------------------*/
-	/*      Range               Use when              Use when    */
-	/*  Begin    End           SD_ENABLED           SD_DISABLED   */
-	/*------------------------------------------------------------*/
-	{ 0x04000, 0x05000, { &io_matrix[AN_COMBO], &io_matrix[AN_COMBO] } },
-	{ 0x1F000, 0x27000, { &io_matrix[AN_COMBO], &io_matrix[AN_COMBO] } },
-	{ 0x05000, 0x0f000, { &io_matrix[AA_COMBO], &io_matrix[AN_COMBO] } },
-	{ 0x0f000, 0x10000, { &io_matrix[NN_COMBO], &io_matrix[NN_COMBO] } },
-	{ 0x13000, 0x17000, { &io_matrix[AA_COMBO], &io_matrix[AA_COMBO] } },
-	{ 0x17000, 0x1b000, { &io_matrix[AN_COMBO], &io_matrix[AN_COMBO] } },
-	{ 0x1b000, 0x1f000, { &io_matrix[AN_COMBO], &io_matrix[AN_COMBO] } },
-	{ 0x10000, 0x11000, { &io_matrix[AA_COMBO], &io_matrix[AA_COMBO] } },
-	{ 0x11000, 0x12000, { &io_matrix[AN_COMBO], &io_matrix[AN_COMBO] } },
-	{ 0x12000, 0x13000, { &io_matrix[AA_COMBO], &io_matrix[AA_COMBO] } },
-	{ 0x43000, 0x44000, { &io_matrix[AA_COMBO], &io_matrix[AA_COMBO] } },
-	{ 0x44000, 0x45000, { &io_matrix[AA_COMBO], &io_matrix[AA_COMBO] } },
-	{ 0x45000, 0x47000, { &io_matrix[AN_COMBO], &io_matrix[AN_COMBO] } },
-	{ 0x40000, 0x42000, { &io_matrix[AA_COMBO], &io_matrix[AA_COMBO] } },
-	{ 0x42000, 0x43000, { &io_matrix[AA_COMBO], &io_matrix[AN_COMBO] } },
-	{ 0x50000, 0x60000, { &io_matrix[AA_COMBO], &io_matrix[AA_COMBO] } },
-	{ 0x60000, 0x80000, { &io_matrix[AN_COMBO], &io_matrix[NN_COMBO] } },
-	{ 0x80000, 0x81000, { &io_matrix[NN_COMBO], &io_matrix[NN_COMBO] } },
-	{ 0x81000, 0x83000, { &io_matrix[AN_COMBO], &io_matrix[AN_COMBO] } },
-	{ 0xa0000, 0xc0000, { &io_matrix[AN_COMBO], &io_matrix[AN_COMBO] } },
-	{ 0xc0000, 0xc2000, { &io_matrix[AA_COMBO], &io_matrix[AA_COMBO] } },
-	{ 0xc2000, 0xd0000, { &io_matrix[AA_COMBO], &io_matrix[AA_COMBO] } },
-};
-
-#endif /* #if !defined(_IPA_ACCESS_CONTROL_H_) */

ipa/ipa_v3/dump/ipa4.5/ipa_gcc_hwio.h

@@ -1,12 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2019, The Linux Foundation. All rights reserved.
- */
-#if !defined(_IPA_GCC_HWIO_H_)
-#define _IPA_GCC_HWIO_H_
-/*
- *
- * HWIO register definitions to follow:
- *
- */
-#endif

ipa/ipa_v3/dump/ipa4.5/ipa_gcc_hwio_def.h

@@ -1,7 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2019, The Linux Foundation. All rights reserved.
- */
-#if !defined(_IPA_GCC_HWIO_DEF_H_)
-#define _IPA_GCC_HWIO_DEF_H_
-#endif

ipa/ipa_v3/dump/ipa4.5/ipa_hw_common_ex.h

@@ -1,593 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2019, The Linux Foundation. All rights reserved.
- */
-#if !defined(_IPA_HW_COMMON_EX_H_)
-#define _IPA_HW_COMMON_EX_H_
-
-/* VLVL defs are available for 854 */
-#define FEATURE_VLVL_DEFS                            true
-
-#define FEATURE_IPA_HW_VERSION_4_5                   true
-
-/* Important Platform Specific Values : IRQ_NUM, IRQ_CNT, BCR */
-#define IPA_HW_BAM_IRQ_NUM                           639
-
-/* Q6 IRQ number for IPA. */
-#define IPA_HW_IRQ_NUM                               640
-
-/* Total number of different interrupts that can be enabled */
-#define IPA_HW_IRQ_CNT_TOTAL                         23
-
-/* IPAv4 spare reg value */
-#define IPA_HW_SPARE_1_REG_VAL                       0xC0000005
-
-/* Whether to allow setting step mode on IPA when we crash or not */
-#define IPA_CFG_HW_IS_STEP_MODE_ALLOWED              (false)
-
-/* GSI MHI related definitions */
-#define IPA_HW_GSI_MHI_CONSUMER_CHANNEL_NUM          0x0
-#define IPA_HW_GSI_MHI_PRODUCER_CHANNEL_NUM          0x1
-
-#define IPA_HW_GSI_MHI_CONSUMER_EP_NUM               0x1
-#define IPA_HW_GSI_MHI_PRODUCER_EP_NUM               0x11
-
-/* IPA ZIP WA related Macros */
-#define IPA_HW_DCMP_SRC_PIPE                         0x8
-#define IPA_HW_DCMP_DEST_PIPE                        0x4
-#define IPA_HW_ACK_MNGR_MASK                         0x1D
-#define IPA_HW_DCMP_SRC_GRP                          0x5
-
-/* IPA Clock resource name */
-#define IPA_CLK_RESOURCE_NAME                        "/clk/pcnoc"
-
-/* IPA Clock Bus Client name */
-#define IPA_CLK_BUS_CLIENT_NAME                      "IPA_PCNOC_BUS_CLIENT"
-
-/* HPS Sequences */
-#define IPA_HW_PKT_PROCESS_HPS_DMA                      0x0
-#define IPA_HW_PKT_PROCESS_HPS_DMA_DECIPH_CIPHE         0x1
-#define IPA_HW_PKT_PROCESS_HPS_PKT_PRS_NO_DECIPH_UCP    0x2
-#define IPA_HW_PKT_PROCESS_HPS_PKT_PRS_DECIPH_UCP       0x3
-#define IPA_HW_PKT_PROCESS_HPS_2_PKT_PRS_NO_DECIPH      0x4
-#define IPA_HW_PKT_PROCESS_HPS_2_PKT_PRS_DECIPH         0x5
-#define IPA_HW_PKT_PROCESS_HPS_PKT_PRS_NO_DECIPH_NO_UCP 0x6
-#define IPA_HW_PKT_PROCESS_HPS_PKT_PRS_DECIPH_NO_UCP    0x7
-#define IPA_HW_PKT_PROCESS_HPS_DMA_PARSER               0x8
-#define IPA_HW_PKT_PROCESS_HPS_DMA_DECIPH_PARSER        0x9
-#define IPA_HW_PKT_PROCESS_HPS_2_PKT_PRS_UCP_TWICE_NO_DECIPH  0xA
-#define IPA_HW_PKT_PROCESS_HPS_2_PKT_PRS_UCP_TWICE_DECIPH     0xB
-#define IPA_HW_PKT_PROCESS_HPS_3_PKT_PRS_UCP_TWICE_NO_DECIPH  0xC
-#define IPA_HW_PKT_PROCESS_HPS_3_PKT_PRS_UCP_TWICE_DECIPH     0xD
-
-/* DPS Sequences */
-#define IPA_HW_PKT_PROCESS_DPS_DMA                      0x0
-#define IPA_HW_PKT_PROCESS_DPS_DMA_WITH_DECIPH          0x1
-#define IPA_HW_PKT_PROCESS_DPS_DMA_WITH_DECOMP          0x2
-#define IPA_HW_PKT_PROCESS_DPS_DMA_WITH_CIPH            0x3
-
-/* Src RSRC GRP config */
-#define IPA_HW_SRC_RSRC_GRP_01_RSRC_TYPE_0           0x0B040803
-#define IPA_HW_SRC_RSRC_GRP_01_RSRC_TYPE_1           0x0C0C0909
-#define IPA_HW_SRC_RSRC_GRP_01_RSRC_TYPE_2           0x0E0E0909
-#define IPA_HW_SRC_RSRC_GRP_01_RSRC_TYPE_3           0x3F003F00
-#define IPA_HW_SRC_RSRC_GRP_01_RSRC_TYPE_4           0x10101616
-
-#define IPA_HW_SRC_RSRC_GRP_23_RSRC_TYPE_0           0x01010101
-#define IPA_HW_SRC_RSRC_GRP_23_RSRC_TYPE_1           0x02020202
-#define IPA_HW_SRC_RSRC_GRP_23_RSRC_TYPE_2           0x04040404
-#define IPA_HW_SRC_RSRC_GRP_23_RSRC_TYPE_3           0x3F003F00
-#define IPA_HW_SRC_RSRC_GRP_23_RSRC_TYPE_4           0x02020606
-
-#define IPA_HW_SRC_RSRC_GRP_45_RSRC_TYPE_0           0x00000000
-#define IPA_HW_SRC_RSRC_GRP_45_RSRC_TYPE_1           0x00000000
-#define IPA_HW_SRC_RSRC_GRP_45_RSRC_TYPE_2           0x00000000
-#define IPA_HW_SRC_RSRC_GRP_45_RSRC_TYPE_3           0x00003F00
-#define IPA_HW_SRC_RSRC_GRP_45_RSRC_TYPE_4           0x00000000
-
-/* Dest RSRC GRP config */
-#define IPA_HW_DST_RSRC_GRP_01_RSRC_TYPE_0           0x05051010
-#define IPA_HW_DST_RSRC_GRP_01_RSRC_TYPE_1           0x3F013F02
-
-#define IPA_HW_DST_RSRC_GRP_23_RSRC_TYPE_0           0x02020202
-#define IPA_HW_DST_RSRC_GRP_23_RSRC_TYPE_1           0x02010201
-
-#define IPA_HW_DST_RSRC_GRP_45_RSRC_TYPE_0           0x00000000
-#define IPA_HW_DST_RSRC_GRP_45_RSRC_TYPE_1           0x00000200
-
-#define IPA_HW_RX_HPS_CLIENTS_MIN_DEPTH_0            0x03030303
-#define IPA_HW_RX_HPS_CLIENTS_MAX_DEPTH_0            0x03030303
-
-#define IPA_HW_RSRP_GRP_0                            0x0
-#define IPA_HW_RSRP_GRP_1                            0x1
-#define IPA_HW_RSRP_GRP_2                            0x2
-#define IPA_HW_RSRP_GRP_3                            0x3
-
-#define IPA_HW_PCIE_SRC_RSRP_GRP                     IPA_HW_RSRP_GRP_0
-#define IPA_HW_PCIE_DEST_RSRP_GRP                    IPA_HW_RSRP_GRP_0
-
-#define IPA_HW_DDR_SRC_RSRP_GRP                      IPA_HW_RSRP_GRP_1
-#define IPA_HW_DDR_DEST_RSRP_GRP                     IPA_HW_RSRP_GRP_1
-
-#define IPA_HW_DMA_SRC_RSRP_GRP                      IPA_HW_RSRP_GRP_2
-#define IPA_HW_DMA_DEST_RSRP_GRP                     IPA_HW_RSRP_GRP_2
-
-#define IPA_HW_SRC_RSRP_TYPE_MAX 0x05
-#define IPA_HW_DST_RSRP_TYPE_MAX 0x02
-
-#define GSI_HW_QSB_LOG_MISC_MAX 0x4
-
-/* IPA Clock Bus Client name */
-#define IPA_CLK_BUS_CLIENT_NAME                      "IPA_PCNOC_BUS_CLIENT"
-
-/* Is IPA decompression feature enabled */
-#define IPA_HW_IS_DECOMPRESSION_ENABLED              (1)
-
-/* Whether to allow setting step mode on IPA when we crash or not */
-#define IPA_HW_IS_STEP_MODE_ALLOWED                  (true)
-
-/* Max number of virtual pipes for UL QBAP provided by HW */
-#define IPA_HW_MAX_VP_NUM                             (32)
-
-/*
- * HW specific clock vote freq values in KHz
- * (BIMC/SNOC/PCNOC/IPA/Q6 CPU)
- */
-enum ipa_hw_clk_freq_e {
-	/* BIMC */
-	IPA_HW_CLK_FREQ_BIMC_PEAK       = 518400,
-	IPA_HW_CLK_FREQ_BIMC_NOM_PLUS   = 404200,
-	IPA_HW_CLK_FREQ_BIMC_NOM        = 404200,
-	IPA_HW_CLK_FREQ_BIMC_SVS        = 100000,
-
-	/* PCNOC */
-	IPA_HW_CLK_FREQ_PCNOC_PEAK      = 133330,
-	IPA_HW_CLK_FREQ_PCNOC_NOM_PLUS  = 100000,
-	IPA_HW_CLK_FREQ_PCNOC_NOM       = 100000,
-	IPA_HW_CLK_FREQ_PCNOC_SVS       = 50000,
-
-	/*IPA_HW_CLK_SNOC*/
-	IPA_HW_CLK_FREQ_SNOC_PEAK       = 200000,
-	IPA_HW_CLK_FREQ_SNOC_NOM_PLUS   = 150000,
-	IPA_HW_CLK_FREQ_SNOC_NOM        = 150000,
-	IPA_HW_CLK_FREQ_SNOC_SVS        = 85000,
-	IPA_HW_CLK_FREQ_SNOC_SVS_2      = 50000,
-
-	/* IPA */
-	IPA_HW_CLK_FREQ_IPA_PEAK        = 600000,
-	IPA_HW_CLK_FREQ_IPA_NOM_PLUS    = 500000,
-	IPA_HW_CLK_FREQ_IPA_NOM         = 500000,
-	IPA_HW_CLK_FREQ_IPA_SVS         = 250000,
-	IPA_HW_CLK_FREQ_IPA_SVS_2       = 150000,
-
-	/* Q6 CPU */
-	IPA_HW_CLK_FREQ_Q6_PEAK         = 729600,
-	IPA_HW_CLK_FREQ_Q6_NOM_PLUS     = 729600,
-	IPA_HW_CLK_FREQ_Q6_NOM          = 729600,
-	IPA_HW_CLK_FREQ_Q6_SVS          = 729600,
-};
-
-enum ipa_hw_qtimer_gran_e {
-	IPA_HW_QTIMER_GRAN_0 = 0, /* granularity 0 is 10us */
-	IPA_HW_QTIMER_GRAN_1 = 1, /* granularity 1 is 100us */
-	IPA_HW_QTIMER_GRAN_MAX,
-};
-
-/* Pipe ID of all the IPA pipes */
-enum ipa_hw_pipe_id_e {
-	IPA_HW_PIPE_ID_0,
-	IPA_HW_PIPE_ID_1,
-	IPA_HW_PIPE_ID_2,
-	IPA_HW_PIPE_ID_3,
-	IPA_HW_PIPE_ID_4,
-	IPA_HW_PIPE_ID_5,
-	IPA_HW_PIPE_ID_6,
-	IPA_HW_PIPE_ID_7,
-	IPA_HW_PIPE_ID_8,
-	IPA_HW_PIPE_ID_9,
-	IPA_HW_PIPE_ID_10,
-	IPA_HW_PIPE_ID_11,
-	IPA_HW_PIPE_ID_12,
-	IPA_HW_PIPE_ID_13,
-	IPA_HW_PIPE_ID_14,
-	IPA_HW_PIPE_ID_15,
-	IPA_HW_PIPE_ID_16,
-	IPA_HW_PIPE_ID_17,
-	IPA_HW_PIPE_ID_18,
-	IPA_HW_PIPE_ID_19,
-	IPA_HW_PIPE_ID_20,
-	IPA_HW_PIPE_ID_21,
-	IPA_HW_PIPE_ID_22,
-	IPA_HW_PIPE_ID_23,
-	IPA_HW_PIPE_ID_24,
-	IPA_HW_PIPE_ID_25,
-	IPA_HW_PIPE_ID_26,
-	IPA_HW_PIPE_ID_27,
-	IPA_HW_PIPE_ID_28,
-	IPA_HW_PIPE_ID_29,
-	IPA_HW_PIPE_ID_30,
-	IPA_HW_PIPE_ID_MAX
-};
-
-/* Pipe ID's of System Bam Endpoints between Q6 & IPA */
-enum ipa_hw_q6_pipe_id_e {
-	/* Pipes used by IPA Q6 driver */
-	IPA_HW_Q6_DL_CONSUMER_PIPE_ID           = IPA_HW_PIPE_ID_5,
-	IPA_HW_Q6_CTL_CONSUMER_PIPE_ID          = IPA_HW_PIPE_ID_6,
-	IPA_HW_Q6_DL_NLO_CONSUMER_PIPE_ID       = IPA_HW_PIPE_ID_8,
-
-	IPA_HW_Q6_UL_ACC_ACK_PRODUCER_PIPE_ID   = IPA_HW_PIPE_ID_20,
-	IPA_HW_Q6_UL_PRODUCER_PIPE_ID           = IPA_HW_PIPE_ID_21,
-	IPA_HW_Q6_DL_PRODUCER_PIPE_ID           = IPA_HW_PIPE_ID_17,
-	IPA_HW_Q6_QBAP_STATUS_PRODUCER_PIPE_ID  = IPA_HW_PIPE_ID_18,
-	IPA_HW_Q6_UL_ACC_DATA_PRODUCER_PIPE_ID  = IPA_HW_PIPE_ID_19,
-
-	IPA_HW_Q6_UL_ACK_PRODUCER_PIPE_ID  =
-	  IPA_HW_Q6_UL_ACC_ACK_PRODUCER_PIPE_ID,
-	IPA_HW_Q6_UL_DATA_PRODUCER_PIPE_ID =
-	  IPA_HW_Q6_UL_ACC_DATA_PRODUCER_PIPE_ID,
-
-	IPA_HW_Q6_DMA_ASYNC_CONSUMER_PIPE_ID    = IPA_HW_PIPE_ID_4,
-	IPA_HW_Q6_DMA_ASYNC_PRODUCER_PIPE_ID    = IPA_HW_PIPE_ID_29,
-
-	/* Test Simulator Pipes */
-	IPA_HW_Q6_SIM_UL_CONSUMER_PIPE_0_ID     = IPA_HW_PIPE_ID_0,
-	IPA_HW_Q6_SIM_UL_CONSUMER_PIPE_1_ID     = IPA_HW_PIPE_ID_1,
-
-	/* GSI UT channel SW->IPA */
-	IPA_HW_Q6_GSI_UT_CONSUMER_PIPE_1_ID     = IPA_HW_PIPE_ID_3,
-	/* GSI UT channel SW->IPA */
-	IPA_HW_Q6_GSI_UT_CONSUMER_PIPE_2_ID     = IPA_HW_PIPE_ID_10,
-
-	IPA_HW_Q6_SIM_UL_CONSUMER_PIPE_2_ID     = IPA_HW_PIPE_ID_7,
-
-	/* GSI UT channel IPA->SW */
-	IPA_HW_Q6_DIAG_CONSUMER_PIPE_ID         = IPA_HW_PIPE_ID_9,
-
-	IPA_HW_Q6_SIM_DL_PRODUCER_PIPE_0_ID     = IPA_HW_PIPE_ID_23,
-	IPA_HW_Q6_SIM_DL_PRODUCER_PIPE_1_ID     = IPA_HW_PIPE_ID_24,
-
-	IPA_HW_Q6_SIM_DL_PRODUCER_PIPE_2_ID     = IPA_HW_PIPE_ID_25,
-
-	/* GSI UT channel IPA->SW */
-	IPA_HW_Q6_GSI_UT_PRODUCER_PIPE_1_ID     = IPA_HW_PIPE_ID_26,
-
-	/* GSI UT channel IPA->SW */
-	IPA_HW_Q6_GSI_UT_PRODUCER_PIPE_2_ID     = IPA_HW_PIPE_ID_27,
-	IPA_HW_Q6_PIPE_ID_MAX                   = IPA_HW_PIPE_ID_MAX,
-};
-
-enum ipa_hw_q6_pipe_ch_id_e {
-	/* Channels used by IPA Q6 driver */
-	IPA_HW_Q6_DL_CONSUMER_PIPE_CH_ID                = 0,
-	IPA_HW_Q6_CTL_CONSUMER_PIPE_CH_ID               = 1,
-	IPA_HW_Q6_DL_NLO_CONSUMER_PIPE_CH_ID            = 2,
-	IPA_HW_Q6_UL_ACC_PATH_ACK_PRODUCER_PIPE_CH_ID   = 6,
-	IPA_HW_Q6_UL_PRODUCER_PIPE_CH_ID                = 7,
-	IPA_HW_Q6_DL_PRODUCER_PIPE_CH_ID                = 3,
-	IPA_HW_Q6_UL_ACC_PATH_DATA_PRODUCER_PIPE_CH_ID  = 5,
-	IPA_HW_Q6_QBAP_STATUS_PRODUCER_PIPE_CH_ID       = 4,
-
-	IPA_HW_Q6_DMA_ASYNC_CONSUMER_PIPE_CH_ID         = 8,
-	IPA_HW_Q6_DMA_ASYNC_PRODUCER_PIPE_CH_ID         = 9,
-	/* CH_ID 8 and 9 are Q6 SPARE CONSUMERs */
-
-	/* Test Simulator Channels */
-	IPA_HW_Q6_SIM_UL_CONSUMER_PIPE_0_CH_ID     = 10,
-	IPA_HW_Q6_SIM_DL_PRODUCER_PIPE_0_CH_ID     = 11,
-	IPA_HW_Q6_SIM_UL_CONSUMER_PIPE_1_CH_ID     = 12,
-	IPA_HW_Q6_SIM_DL_PRODUCER_PIPE_1_CH_ID     = 13,
-	IPA_HW_Q6_SIM_UL_CONSUMER_PIPE_2_CH_ID     = 14,
-	IPA_HW_Q6_SIM_DL_PRODUCER_PIPE_2_CH_ID     = 15,
-	/* GSI UT channel SW->IPA */
-	IPA_HW_Q6_GSI_UT_CONSUMER_PIPE_1_CH_ID     = 16,
-	/* GSI UT channel IPA->SW */
-	IPA_HW_Q6_GSI_UT_PRODUCER_PIPE_1_CH_ID     = 17,
-	/* GSI UT channel SW->IPA */
-	IPA_HW_Q6_GSI_UT_CONSUMER_PIPE_2_CH_ID     = 18,
-	/* GSI UT channel IPA->SW */
-	IPA_HW_Q6_GSI_UT_PRODUCER_PIPE_2_CH_ID     = 19,
-};
-
-/* System Bam Endpoints between Q6 & IPA */
-enum ipa_hw_q6_pipe_e {
-	/* DL Pipe IPA->Q6 */
-	IPA_HW_Q6_DL_PRODUCER_PIPE = 0,
-	/* UL Pipe IPA->Q6 */
-	IPA_HW_Q6_UL_PRODUCER_PIPE = 1,
-	/* DL Pipe Q6->IPA */
-	IPA_HW_Q6_DL_CONSUMER_PIPE = 2,
-	/* CTL Pipe Q6->IPA */
-	IPA_HW_Q6_CTL_CONSUMER_PIPE = 3,
-	/*  Q6 -> IPA,  DL NLO  */
-	IPA_HW_Q6_DL_NLO_CONSUMER_PIPE = 4,
-	/* DMA ASYNC CONSUMER */
-	IPA_HW_Q6_DMA_ASYNC_CONSUMER_PIPE = 5,
-	/* DMA ASYNC PRODUCER */
-	IPA_HW_Q6_DMA_ASYNC_PRODUCER_PIPE = 6,
-	/* UL Acc Path Data Pipe IPA->Q6 */
-	IPA_HW_Q6_UL_ACC_DATA_PRODUCER_PIPE = 7,
-	/* UL Acc Path ACK Pipe IPA->Q6 */
-	IPA_HW_Q6_UL_ACC_ACK_PRODUCER_PIPE = 8,
-	/* UL Acc Path QBAP status Pipe IPA->Q6 */
-	IPA_HW_Q6_QBAP_STATUS_PRODUCER_PIPE = 9,
-	/* Diag status pipe IPA->Q6 */
-	/* Used only when FEATURE_IPA_TEST_PER_SIM is ON */
-	/* SIM Pipe IPA->Sim */
-	IPA_HW_Q6_SIM_DL_PRODUCER_PIPE_0 = 10,
-	/* SIM Pipe Sim->IPA */
-	IPA_HW_Q6_SIM_DL_PRODUCER_PIPE_1 = 11,
-	/* SIM Pipe Sim->IPA */
-	IPA_HW_Q6_SIM_DL_PRODUCER_PIPE_2 = 12,
-	/* SIM Pipe Sim->IPA */
-	IPA_HW_Q6_SIM_UL_CONSUMER_PIPE_0 = 13,
-	/* SIM B2B PROD Pipe  */
-	IPA_HW_Q6_SIM_UL_CONSUMER_PIPE_1 = 14,
-	/* SIM Pipe IPA->Sim */
-	IPA_HW_Q6_SIM_UL_CONSUMER_PIPE_2 = 15,
-	/* End FEATURE_IPA_TEST_PER_SIM */
-	/* GSI UT channel SW->IPA */
-	IPA_HW_Q6_GSI_UT_CONSUMER_PIPE_1 = 16,
-	/* GSI UT channel IPA->SW */
-	IPA_HW_Q6_GSI_UT_PRODUCER_PIPE_1 = 17,
-	/* GSI UT channel SW->IPA */
-	IPA_HW_Q6_GSI_UT_CONSUMER_PIPE_2 = 18,
-	/* GSI UT channel IPA->SW */
-	IPA_HW_Q6_GSI_UT_PRODUCER_PIPE_2 = 19,
-
-	IPA_HW_Q6_PIPE_TOTAL
-};
-
-/* System Bam Endpoints between Q6 & IPA */
-enum ipa_hw_q6_gsi_ev_e { /* In Sdx24 0..11 */
-	/* DL Pipe IPA->Q6 */
-	IPA_HW_Q6_DL_PRODUCER_PIPE_GSI_EV = 0,
-	/* UL Pipe IPA->Q6 */
-	IPA_HW_Q6_UL_PRODUCER_PIPE_GSI_EV = 1,
-	/* DL Pipe Q6->IPA */
-	//IPA_HW_Q6_DL_CONSUMER_PIPE_GSI_EV = 2,
-	/* CTL Pipe Q6->IPA */
-	//IPA_HW_Q6_CTL_CONSUMER_PIPE_GSI_EV = 3,
-	/*  Q6 -> IPA,  LTE DL Optimized path */
-	//IPA_HW_Q6_LTE_DL_CONSUMER_PIPE_GSI_EV = 4,
-	/* LWA DL(Wifi to Q6) */
-	//IPA_HW_Q6_LWA_DL_PRODUCER_PIPE_GSI_EV = 5,
-	/* Diag status pipe IPA->Q6 */
-	//IPA_HW_Q6_DIAG_STATUS_PRODUCER_PIPE_GSI_EV = 6,
-	/* Used only when FEATURE_IPA_TEST_PER_SIM is ON */
-	/* SIM Pipe IPA->Sim */
-	IPA_HW_Q6_SIM_DL_PRODUCER_PIPE_0_GSI_EV = 2,
-	/* SIM Pipe Sim->IPA */
-	IPA_HW_Q6_SIM_DL_PRODUCER_PIPE_1_GSI_EV = 3,
-	/* SIM Pipe Sim->IPA */
-	IPA_HW_Q6_SIM_DL_PRODUCER_PIPE_2_GSI_EV = 4,
-	/* SIM Pipe Sim->IPA */
-	IPA_HW_Q6_SIM_1_GSI_EV = 5,
-	IPA_HW_Q6_SIM_2_GSI_EV = 6,
-	IPA_HW_Q6_SIM_3_GSI_EV = 7,
-	IPA_HW_Q6_SIM_4_GSI_EV = 8,
-
-	IPA_HW_Q6_PIPE_GSI_EV_TOTAL
-};
-
-/*
- * All the IRQ's supported by the IPA HW. Use this enum to set IRQ_EN
- * register and read IRQ_STTS register
- */
-enum ipa_hw_irq_e {
-	IPA_HW_IRQ_GSI_HWP                     = (1 << 25),
-	IPA_HW_IRQ_GSI_IPA_IF_TLV_RCVD         = (1 << 24),
-	IPA_HW_IRQ_GSI_EE_IRQ                  = (1 << 23),
-	IPA_HW_IRQ_DCMP_ERR                    = (1 << 22),
-	IPA_HW_IRQ_HWP_ERR                     = (1 << 21),
-	IPA_HW_IRQ_RED_MARKER_ABOVE            = (1 << 20),
-	IPA_HW_IRQ_YELLOW_MARKER_ABOVE         = (1 << 19),
-	IPA_HW_IRQ_RED_MARKER_BELOW            = (1 << 18),
-	IPA_HW_IRQ_YELLOW_MARKER_BELOW         = (1 << 17),
-	IPA_HW_IRQ_BAM_IDLE_IRQ                = (1 << 16),
-	IPA_HW_IRQ_TX_HOLB_DROP                = (1 << 15),
-	IPA_HW_IRQ_TX_SUSPEND                  = (1 << 14),
-	IPA_HW_IRQ_PROC_ERR                    = (1 << 13),
-	IPA_HW_IRQ_STEP_MODE                   = (1 << 12),
-	IPA_HW_IRQ_TX_ERR                      = (1 << 11),
-	IPA_HW_IRQ_DEAGGR_ERR                  = (1 << 10),
-	IPA_HW_IRQ_RX_ERR                      = (1 << 9),
-	IPA_HW_IRQ_PROC_TO_HW_ACK_Q_NOT_EMPTY  = (1 << 8),
-	IPA_HW_IRQ_HWP_RX_CMD_Q_NOT_FULL       = (1 << 7),
-	IPA_HW_IRQ_HWP_IN_Q_NOT_EMPTY          = (1 << 6),
-	IPA_HW_IRQ_HWP_IRQ_3                   = (1 << 5),
-	IPA_HW_IRQ_HWP_IRQ_2                   = (1 << 4),
-	IPA_HW_IRQ_HWP_IRQ_1                   = (1 << 3),
-	IPA_HW_IRQ_HWP_IRQ_0                   = (1 << 2),
-	IPA_HW_IRQ_EOT_COAL                    = (1 << 1),
-	IPA_HW_IRQ_BAD_SNOC_ACCESS             = (1 << 0),
-	IPA_HW_IRQ_NONE                        = 0,
-	IPA_HW_IRQ_ALL                         = 0xFFFFFFFF
-};
-
-/*
- * All the IRQ sources supported by the IPA HW. Use this enum to set
- * IRQ_SRCS register
- */
-enum ipa_hw_irq_srcs_e {
-	IPA_HW_IRQ_SRCS_PIPE_0  = (1 << IPA_HW_PIPE_ID_0),
-	IPA_HW_IRQ_SRCS_PIPE_1  = (1 << IPA_HW_PIPE_ID_1),
-	IPA_HW_IRQ_SRCS_PIPE_2  = (1 << IPA_HW_PIPE_ID_2),
-	IPA_HW_IRQ_SRCS_PIPE_3  = (1 << IPA_HW_PIPE_ID_3),
-	IPA_HW_IRQ_SRCS_PIPE_4  = (1 << IPA_HW_PIPE_ID_4),
-	IPA_HW_IRQ_SRCS_PIPE_5  = (1 << IPA_HW_PIPE_ID_5),
-	IPA_HW_IRQ_SRCS_PIPE_6  = (1 << IPA_HW_PIPE_ID_6),
-	IPA_HW_IRQ_SRCS_PIPE_7  = (1 << IPA_HW_PIPE_ID_7),
-	IPA_HW_IRQ_SRCS_PIPE_8  = (1 << IPA_HW_PIPE_ID_8),
-	IPA_HW_IRQ_SRCS_PIPE_9  = (1 << IPA_HW_PIPE_ID_9),
-	IPA_HW_IRQ_SRCS_PIPE_10 = (1 << IPA_HW_PIPE_ID_10),
-	IPA_HW_IRQ_SRCS_PIPE_11 = (1 << IPA_HW_PIPE_ID_11),
-	IPA_HW_IRQ_SRCS_PIPE_12 = (1 << IPA_HW_PIPE_ID_12),
-	IPA_HW_IRQ_SRCS_PIPE_13 = (1 << IPA_HW_PIPE_ID_13),
-	IPA_HW_IRQ_SRCS_PIPE_14 = (1 << IPA_HW_PIPE_ID_14),
-	IPA_HW_IRQ_SRCS_PIPE_15 = (1 << IPA_HW_PIPE_ID_15),
-	IPA_HW_IRQ_SRCS_PIPE_16 = (1 << IPA_HW_PIPE_ID_16),
-	IPA_HW_IRQ_SRCS_PIPE_17 = (1 << IPA_HW_PIPE_ID_17),
-	IPA_HW_IRQ_SRCS_PIPE_18 = (1 << IPA_HW_PIPE_ID_18),
-	IPA_HW_IRQ_SRCS_PIPE_19 = (1 << IPA_HW_PIPE_ID_19),
-	IPA_HW_IRQ_SRCS_PIPE_20 = (1 << IPA_HW_PIPE_ID_20),
-	IPA_HW_IRQ_SRCS_PIPE_21 = (1 << IPA_HW_PIPE_ID_21),
-	IPA_HW_IRQ_SRCS_PIPE_22 = (1 << IPA_HW_PIPE_ID_22),
-	IPA_HW_IRQ_SRCS_NONE    = 0,
-	IPA_HW_IRQ_SRCS_ALL     = 0xFFFFFFFF,
-};
-
-/*
- * Total number of channel contexts that need to be saved for APPS
- */
-#define IPA_HW_REG_SAVE_GSI_NUM_CH_CNTXT_A7          20
-
-/*
- * Total number of channel contexts that need to be saved for UC
- */
-#define IPA_HW_REG_SAVE_GSI_NUM_CH_CNTXT_UC          2
-
-/*
- * Total number of event ring contexts that need to be saved for APPS
- */
-#define IPA_HW_REG_SAVE_GSI_NUM_EVT_CNTXT_A7         19
-
-/*
- * Total number of event ring contexts that need to be saved for UC
- */
-#define IPA_HW_REG_SAVE_GSI_NUM_EVT_CNTXT_UC         1
-
-/*
- * Total number of endpoints for which ipa_reg_save.pipes[endp_number]
- * are not saved by default (only if ipa_cfg.gen.full_reg_trace =
- * true) There is no extra endpoints in Stingray
- */
-#define IPA_HW_REG_SAVE_NUM_ENDP_EXTRA               0
-
-/*
- * Total number of endpoints for which ipa_reg_save.pipes[endp_number]
- * are always saved
- */
-#define IPA_HW_REG_SAVE_NUM_ACTIVE_PIPES             IPA_HW_PIPE_ID_MAX
-
-/*
- * SHRAM Bytes per ch
- */
-#define IPA_REG_SAVE_BYTES_PER_CHNL_SHRAM         12
-
-/*
- * Total number of rx splt cmdq's see:
- * ipa_rx_splt_cmdq_n_cmd[IPA_RX_SPLT_CMDQ_MAX]
- */
-#define IPA_RX_SPLT_CMDQ_MAX 4
-
-/*
- * Although not necessary for the numbers below, the use of round_up
- * is so that future developers know that these particular constants
- * have to be a multiple of four bytes, because the IPA memory reads
- * that they drive are always 32 bits...
- */
-#define IPA_IU_ADDR   0x000A0000
-#define IPA_IU_SIZE   round_up(40704, sizeof(u32))
-
-#define IPA_SRAM_ADDR 0x00050000
-#define IPA_SRAM_SIZE round_up(19232, sizeof(u32))
-
-#define IPA_MBOX_ADDR 0x000C2000
-#define IPA_MBOX_SIZE round_up(256, sizeof(u32))
-
-#define IPA_HRAM_ADDR 0x00060000
-#define IPA_HRAM_SIZE round_up(47536, sizeof(u32))
-
-#define IPA_SEQ_ADDR  0x00081000
-#define IPA_SEQ_SIZE  round_up(768, sizeof(u32))
-
-#define IPA_GSI_ADDR  0x00006000
-#define IPA_GSI_SIZE  round_up(5376, sizeof(u32))
-
-/*
- * Macro to define a particular register cfg entry for all pipe
- * indexed register
- */
-#define IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP(reg_name, var_name)	\
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 0), \
-		(u32 *)&ipa_reg_save.ipa.pipes[0].endp.var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 1), \
-		(u32 *)&ipa_reg_save.ipa.pipes[1].endp.var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 2), \
-		(u32 *)&ipa_reg_save.ipa.pipes[2].endp.var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 3), \
-		(u32 *)&ipa_reg_save.ipa.pipes[3].endp.var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 4), \
-		(u32 *)&ipa_reg_save.ipa.pipes[4].endp.var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 5), \
-		(u32 *)&ipa_reg_save.ipa.pipes[5].endp.var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 6), \
-		(u32 *)&ipa_reg_save.ipa.pipes[6].endp.var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 7), \
-		(u32 *)&ipa_reg_save.ipa.pipes[7].endp.var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 8), \
-		(u32 *)&ipa_reg_save.ipa.pipes[8].endp.var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 9), \
-		(u32 *)&ipa_reg_save.ipa.pipes[9].endp.var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 10), \
-		(u32 *)&ipa_reg_save.ipa.pipes[10].endp.var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 11), \
-		(u32 *)&ipa_reg_save.ipa.pipes[11].endp.var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 12), \
-		(u32 *)&ipa_reg_save.ipa.pipes[12].endp.var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 13), \
-		(u32 *)&ipa_reg_save.ipa.pipes[13].endp.var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 14), \
-		(u32 *)&ipa_reg_save.ipa.pipes[14].endp.var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 15), \
-		(u32 *)&ipa_reg_save.ipa.pipes[15].endp.var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 16), \
-		(u32 *)&ipa_reg_save.ipa.pipes[16].endp.var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 17), \
-		(u32 *)&ipa_reg_save.ipa.pipes[17].endp.var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 18), \
-		(u32 *)&ipa_reg_save.ipa.pipes[18].endp.var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 19), \
-		(u32 *)&ipa_reg_save.ipa.pipes[19].endp.var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 20), \
-		(u32 *)&ipa_reg_save.ipa.pipes[20].endp.var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 21), \
-		(u32 *)&ipa_reg_save.ipa.pipes[21].endp.var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 22), \
-		(u32 *)&ipa_reg_save.ipa.pipes[22].endp.var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 23), \
-		(u32 *)&ipa_reg_save.ipa.pipes[23].endp.var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 24), \
-		(u32 *)&ipa_reg_save.ipa.pipes[24].endp.var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 25), \
-		(u32 *)&ipa_reg_save.ipa.pipes[25].endp.var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 26), \
-		(u32 *)&ipa_reg_save.ipa.pipes[26].endp.var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 27), \
-		(u32 *)&ipa_reg_save.ipa.pipes[27].endp.var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 28), \
-		(u32 *)&ipa_reg_save.ipa.pipes[28].endp.var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 29), \
-		(u32 *)&ipa_reg_save.ipa.pipes[29].endp.var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 30), \
-		(u32 *)&ipa_reg_save.ipa.pipes[30].endp.var_name }
-
-/*
- * Macro to define a particular register cfg entry for the remaining
- * pipe indexed register.  In Stingray case we don't have extra
- * endpoints so it is intentially empty
- */
-#define IPA_HW_REG_SAVE_CFG_ENTRY_PIPE_ENDP_EXTRA(REG_NAME, VAR_NAME)  \
-	{ 0, 0 }
-
-/*
- * Macro to set the active flag for all active pipe indexed register
- * In Stingray case we don't have extra endpoints so it is intentially
- * empty
- */
-#define IPA_HW_REG_SAVE_CFG_ENTRY_PIPE_ENDP_EXTRA_ACTIVE()  \
-	do { \
-	} while (0)
-
-#endif /* #if !defined(_IPA_HW_COMMON_EX_H_) */

ipa/ipa_v3/dump/ipa4.5/ipa_hwio.h

@@ -1,10895 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2019, The Linux Foundation. All rights reserved.
- */
-#if !defined(_IPA_HWIO_H_)
-#define _IPA_HWIO_H_
-/*
- *
- * HWIO register definitions to follow:
- *
- */
-#define IPA_GSI_TOP_GSI_REG_BASE (IPA_0_IPA_WRAPPER_BASE + 0x00004000)
-#define IPA_GSI_TOP_GSI_REG_BASE_PHYS (IPA_0_IPA_WRAPPER_BASE_PHYS + \
-				       0x00004000)
-#define IPA_GSI_TOP_GSI_REG_BASE_OFFS 0x00004000
-#define HWIO_IPA_GSI_TOP_GSI_CFG_ADDR (IPA_GSI_TOP_GSI_REG_BASE + \
-				       0x00000000)
-#define HWIO_IPA_GSI_TOP_GSI_CFG_PHYS (IPA_GSI_TOP_GSI_REG_BASE_PHYS + \
-				       0x00000000)
-#define HWIO_IPA_GSI_TOP_GSI_CFG_OFFS (IPA_GSI_TOP_GSI_REG_BASE_OFFS + \
-				       0x00000000)
-#define HWIO_IPA_GSI_TOP_GSI_CFG_RMSK 0xf3f
-#define HWIO_IPA_GSI_TOP_GSI_CFG_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_GSI_CFG_IN in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_CFG_ADDR, \
-		HWIO_IPA_GSI_TOP_GSI_CFG_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_CFG_INM(m) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_CFG_ADDR, \
-		m)
-#define HWIO_IPA_GSI_TOP_GSI_CFG_OUT(v) out_dword( \
-		HWIO_IPA_GSI_TOP_GSI_CFG_ADDR, \
-		v)
-#define HWIO_IPA_GSI_TOP_GSI_CFG_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_GSI_CFG_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_GSI_TOP_GSI_CFG_IN)
-#define HWIO_IPA_GSI_TOP_GSI_CFG_SLEEP_CLK_DIV_BMSK 0xf00
-#define HWIO_IPA_GSI_TOP_GSI_CFG_SLEEP_CLK_DIV_SHFT 0x8
-#define HWIO_IPA_GSI_TOP_GSI_CFG_BP_MTRIX_DISABLE_BMSK 0x20
-#define HWIO_IPA_GSI_TOP_GSI_CFG_BP_MTRIX_DISABLE_SHFT 0x5
-#define HWIO_IPA_GSI_TOP_GSI_CFG_GSI_PWR_CLPS_BMSK 0x10
-#define HWIO_IPA_GSI_TOP_GSI_CFG_GSI_PWR_CLPS_SHFT 0x4
-#define HWIO_IPA_GSI_TOP_GSI_CFG_UC_IS_MCS_BMSK 0x8
-#define HWIO_IPA_GSI_TOP_GSI_CFG_UC_IS_MCS_SHFT 0x3
-#define HWIO_IPA_GSI_TOP_GSI_CFG_DOUBLE_MCS_CLK_FREQ_BMSK 0x4
-#define HWIO_IPA_GSI_TOP_GSI_CFG_DOUBLE_MCS_CLK_FREQ_SHFT 0x2
-#define HWIO_IPA_GSI_TOP_GSI_CFG_MCS_ENABLE_BMSK 0x2
-#define HWIO_IPA_GSI_TOP_GSI_CFG_MCS_ENABLE_SHFT 0x1
-#define HWIO_IPA_GSI_TOP_GSI_CFG_GSI_ENABLE_BMSK 0x1
-#define HWIO_IPA_GSI_TOP_GSI_CFG_GSI_ENABLE_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_GSI_MANAGER_MCS_CODE_VER_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000008)
-#define HWIO_IPA_GSI_TOP_GSI_MANAGER_MCS_CODE_VER_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000008)
-#define HWIO_IPA_GSI_TOP_GSI_MANAGER_MCS_CODE_VER_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000008)
-#define HWIO_IPA_GSI_TOP_GSI_ZEROS_ADDR (IPA_GSI_TOP_GSI_REG_BASE + \
-					 0x00000010)
-#define HWIO_IPA_GSI_TOP_GSI_ZEROS_PHYS (IPA_GSI_TOP_GSI_REG_BASE_PHYS + \
-					 0x00000010)
-#define HWIO_IPA_GSI_TOP_GSI_ZEROS_OFFS (IPA_GSI_TOP_GSI_REG_BASE_OFFS + \
-					 0x00000010)
-#define HWIO_IPA_GSI_TOP_GSI_PERIPH_BASE_ADDR_LSB_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000018)
-#define HWIO_IPA_GSI_TOP_GSI_PERIPH_BASE_ADDR_LSB_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000018)
-#define HWIO_IPA_GSI_TOP_GSI_PERIPH_BASE_ADDR_LSB_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000018)
-#define HWIO_IPA_GSI_TOP_GSI_PERIPH_BASE_ADDR_MSB_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000001c)
-#define HWIO_IPA_GSI_TOP_GSI_PERIPH_BASE_ADDR_MSB_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000001c)
-#define HWIO_IPA_GSI_TOP_GSI_PERIPH_BASE_ADDR_MSB_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000001c)
-#define HWIO_IPA_GSI_TOP_GSI_PERIPH_PENDING_ADDR (IPA_GSI_TOP_GSI_REG_BASE \
-						  + 0x00000020)
-#define HWIO_IPA_GSI_TOP_GSI_PERIPH_PENDING_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000020)
-#define HWIO_IPA_GSI_TOP_GSI_PERIPH_PENDING_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000020)
-#define HWIO_IPA_GSI_TOP_GSI_MOQA_CFG_ADDR (IPA_GSI_TOP_GSI_REG_BASE + \
-					    0x00000030)
-#define HWIO_IPA_GSI_TOP_GSI_MOQA_CFG_PHYS (IPA_GSI_TOP_GSI_REG_BASE_PHYS \
-					    + 0x00000030)
-#define HWIO_IPA_GSI_TOP_GSI_MOQA_CFG_OFFS (IPA_GSI_TOP_GSI_REG_BASE_OFFS \
-					    + 0x00000030)
-#define HWIO_IPA_GSI_TOP_GSI_REE_CFG_ADDR (IPA_GSI_TOP_GSI_REG_BASE + \
-					   0x00000038)
-#define HWIO_IPA_GSI_TOP_GSI_REE_CFG_PHYS (IPA_GSI_TOP_GSI_REG_BASE_PHYS + \
-					   0x00000038)
-#define HWIO_IPA_GSI_TOP_GSI_REE_CFG_OFFS (IPA_GSI_TOP_GSI_REG_BASE_OFFS + \
-					   0x00000038)
-#define HWIO_IPA_GSI_TOP_GSI_REE_CFG_RMSK 0xff03
-#define HWIO_IPA_GSI_TOP_GSI_REE_CFG_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_GSI_REE_CFG_IN in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_REE_CFG_ADDR, \
-		HWIO_IPA_GSI_TOP_GSI_REE_CFG_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_REE_CFG_INM(m) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_REE_CFG_ADDR, \
-		m)
-#define HWIO_IPA_GSI_TOP_GSI_REE_CFG_OUT(v) out_dword( \
-		HWIO_IPA_GSI_TOP_GSI_REE_CFG_ADDR, \
-		v)
-#define HWIO_IPA_GSI_TOP_GSI_REE_CFG_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_GSI_REE_CFG_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_GSI_TOP_GSI_REE_CFG_IN)
-#define HWIO_IPA_GSI_TOP_GSI_REE_CFG_MAX_BURST_SIZE_BMSK 0xff00
-#define HWIO_IPA_GSI_TOP_GSI_REE_CFG_MAX_BURST_SIZE_SHFT 0x8
-#define HWIO_IPA_GSI_TOP_GSI_REE_CFG_CHANNEL_EMPTY_INT_ENABLE_BMSK 0x2
-#define HWIO_IPA_GSI_TOP_GSI_REE_CFG_CHANNEL_EMPTY_INT_ENABLE_SHFT 0x1
-#define HWIO_IPA_GSI_TOP_GSI_REE_CFG_MOVE_TO_ESC_CLR_MODE_TRSH_BMSK 0x1
-#define HWIO_IPA_GSI_TOP_GSI_REE_CFG_MOVE_TO_ESC_CLR_MODE_TRSH_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_GSI_CGC_CTRL_ADDR (IPA_GSI_TOP_GSI_REG_BASE + \
-					    0x00000060)
-#define HWIO_IPA_GSI_TOP_GSI_CGC_CTRL_PHYS (IPA_GSI_TOP_GSI_REG_BASE_PHYS \
-					    + 0x00000060)
-#define HWIO_IPA_GSI_TOP_GSI_CGC_CTRL_OFFS (IPA_GSI_TOP_GSI_REG_BASE_OFFS \
-					    + 0x00000060)
-#define HWIO_IPA_GSI_TOP_GSI_MSI_CACHEATTR_ADDR (IPA_GSI_TOP_GSI_REG_BASE \
-						 + 0x00000080)
-#define HWIO_IPA_GSI_TOP_GSI_MSI_CACHEATTR_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000080)
-#define HWIO_IPA_GSI_TOP_GSI_MSI_CACHEATTR_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000080)
-#define HWIO_IPA_GSI_TOP_GSI_EVENT_CACHEATTR_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000084)
-#define HWIO_IPA_GSI_TOP_GSI_EVENT_CACHEATTR_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000084)
-#define HWIO_IPA_GSI_TOP_GSI_EVENT_CACHEATTR_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000084)
-#define HWIO_IPA_GSI_TOP_GSI_DATA_CACHEATTR_ADDR (IPA_GSI_TOP_GSI_REG_BASE \
-						  + 0x00000088)
-#define HWIO_IPA_GSI_TOP_GSI_DATA_CACHEATTR_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000088)
-#define HWIO_IPA_GSI_TOP_GSI_DATA_CACHEATTR_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000088)
-#define HWIO_IPA_GSI_TOP_GSI_TRE_CACHEATTR_ADDR (IPA_GSI_TOP_GSI_REG_BASE \
-						 + 0x00000090)
-#define HWIO_IPA_GSI_TOP_GSI_TRE_CACHEATTR_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000090)
-#define HWIO_IPA_GSI_TOP_GSI_TRE_CACHEATTR_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000090)
-#define HWIO_IPA_GSI_TOP_IC_DISABLE_CHNL_BCK_PRS_LSB_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x000000a0)
-#define HWIO_IPA_GSI_TOP_IC_DISABLE_CHNL_BCK_PRS_LSB_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x000000a0)
-#define HWIO_IPA_GSI_TOP_IC_DISABLE_CHNL_BCK_PRS_LSB_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x000000a0)
-#define HWIO_IPA_GSI_TOP_IC_DISABLE_CHNL_BCK_PRS_MSB_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x000000a4)
-#define HWIO_IPA_GSI_TOP_IC_DISABLE_CHNL_BCK_PRS_MSB_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x000000a4)
-#define HWIO_IPA_GSI_TOP_IC_DISABLE_CHNL_BCK_PRS_MSB_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x000000a4)
-#define HWIO_IPA_GSI_TOP_IC_GEN_EVNT_BCK_PRS_LSB_ADDR (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x000000a8)
-#define HWIO_IPA_GSI_TOP_IC_GEN_EVNT_BCK_PRS_LSB_PHYS (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x000000a8)
-#define HWIO_IPA_GSI_TOP_IC_GEN_EVNT_BCK_PRS_LSB_OFFS (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x000000a8)
-#define HWIO_IPA_GSI_TOP_IC_GEN_EVNT_BCK_PRS_MSB_ADDR (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x000000ac)
-#define HWIO_IPA_GSI_TOP_IC_GEN_EVNT_BCK_PRS_MSB_PHYS (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x000000ac)
-#define HWIO_IPA_GSI_TOP_IC_GEN_EVNT_BCK_PRS_MSB_OFFS (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x000000ac)
-#define HWIO_IPA_GSI_TOP_IC_GEN_INT_BCK_PRS_LSB_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x000000b0)
-#define HWIO_IPA_GSI_TOP_IC_GEN_INT_BCK_PRS_LSB_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x000000b0)
-#define HWIO_IPA_GSI_TOP_IC_GEN_INT_BCK_PRS_LSB_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x000000b0)
-#define HWIO_IPA_GSI_TOP_IC_GEN_INT_BCK_PRS_MSB_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x000000b4)
-#define HWIO_IPA_GSI_TOP_IC_GEN_INT_BCK_PRS_MSB_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x000000b4)
-#define HWIO_IPA_GSI_TOP_IC_GEN_INT_BCK_PRS_MSB_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x000000b4)
-#define HWIO_IPA_GSI_TOP_IC_STOP_INT_MOD_BCK_PRS_LSB_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x000000b8)
-#define HWIO_IPA_GSI_TOP_IC_STOP_INT_MOD_BCK_PRS_LSB_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x000000b8)
-#define HWIO_IPA_GSI_TOP_IC_STOP_INT_MOD_BCK_PRS_LSB_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x000000b8)
-#define HWIO_IPA_GSI_TOP_IC_STOP_INT_MOD_BCK_PRS_MSB_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x000000bc)
-#define HWIO_IPA_GSI_TOP_IC_STOP_INT_MOD_BCK_PRS_MSB_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x000000bc)
-#define HWIO_IPA_GSI_TOP_IC_STOP_INT_MOD_BCK_PRS_MSB_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x000000bc)
-#define HWIO_IPA_GSI_TOP_IC_PROCESS_DESC_BCK_PRS_LSB_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x000000c0)
-#define HWIO_IPA_GSI_TOP_IC_PROCESS_DESC_BCK_PRS_LSB_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x000000c0)
-#define HWIO_IPA_GSI_TOP_IC_PROCESS_DESC_BCK_PRS_LSB_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x000000c0)
-#define HWIO_IPA_GSI_TOP_IC_PROCESS_DESC_BCK_PRS_MSB_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x000000c4)
-#define HWIO_IPA_GSI_TOP_IC_PROCESS_DESC_BCK_PRS_MSB_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x000000c4)
-#define HWIO_IPA_GSI_TOP_IC_PROCESS_DESC_BCK_PRS_MSB_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x000000c4)
-#define HWIO_IPA_GSI_TOP_IC_TLV_STOP_BCK_PRS_LSB_ADDR (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x000000c8)
-#define HWIO_IPA_GSI_TOP_IC_TLV_STOP_BCK_PRS_LSB_PHYS (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x000000c8)
-#define HWIO_IPA_GSI_TOP_IC_TLV_STOP_BCK_PRS_LSB_OFFS (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x000000c8)
-#define HWIO_IPA_GSI_TOP_IC_TLV_STOP_BCK_PRS_MSB_ADDR (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x000000cc)
-#define HWIO_IPA_GSI_TOP_IC_TLV_STOP_BCK_PRS_MSB_PHYS (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x000000cc)
-#define HWIO_IPA_GSI_TOP_IC_TLV_STOP_BCK_PRS_MSB_OFFS (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x000000cc)
-#define HWIO_IPA_GSI_TOP_IC_TLV_RESET_BCK_PRS_LSB_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x000000d0)
-#define HWIO_IPA_GSI_TOP_IC_TLV_RESET_BCK_PRS_LSB_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x000000d0)
-#define HWIO_IPA_GSI_TOP_IC_TLV_RESET_BCK_PRS_LSB_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x000000d0)
-#define HWIO_IPA_GSI_TOP_IC_TLV_RESET_BCK_PRS_MSB_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x000000d4)
-#define HWIO_IPA_GSI_TOP_IC_TLV_RESET_BCK_PRS_MSB_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x000000d4)
-#define HWIO_IPA_GSI_TOP_IC_TLV_RESET_BCK_PRS_MSB_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x000000d4)
-#define HWIO_IPA_GSI_TOP_IC_RGSTR_TIMER_BCK_PRS_LSB_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x000000d8)
-#define HWIO_IPA_GSI_TOP_IC_RGSTR_TIMER_BCK_PRS_LSB_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x000000d8)
-#define HWIO_IPA_GSI_TOP_IC_RGSTR_TIMER_BCK_PRS_LSB_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x000000d8)
-#define HWIO_IPA_GSI_TOP_IC_RGSTR_TIMER_BCK_PRS_MSB_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x000000dc)
-#define HWIO_IPA_GSI_TOP_IC_RGSTR_TIMER_BCK_PRS_MSB_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x000000dc)
-#define HWIO_IPA_GSI_TOP_IC_RGSTR_TIMER_BCK_PRS_MSB_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x000000dc)
-#define HWIO_IPA_GSI_TOP_IC_READ_BCK_PRS_LSB_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x000000e0)
-#define HWIO_IPA_GSI_TOP_IC_READ_BCK_PRS_LSB_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x000000e0)
-#define HWIO_IPA_GSI_TOP_IC_READ_BCK_PRS_LSB_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x000000e0)
-#define HWIO_IPA_GSI_TOP_IC_READ_BCK_PRS_MSB_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x000000e4)
-#define HWIO_IPA_GSI_TOP_IC_READ_BCK_PRS_MSB_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x000000e4)
-#define HWIO_IPA_GSI_TOP_IC_READ_BCK_PRS_MSB_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x000000e4)
-#define HWIO_IPA_GSI_TOP_IC_WRITE_BCK_PRS_LSB_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x000000e8)
-#define HWIO_IPA_GSI_TOP_IC_WRITE_BCK_PRS_LSB_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x000000e8)
-#define HWIO_IPA_GSI_TOP_IC_WRITE_BCK_PRS_LSB_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x000000e8)
-#define HWIO_IPA_GSI_TOP_IC_WRITE_BCK_PRS_MSB_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x000000ec)
-#define HWIO_IPA_GSI_TOP_IC_WRITE_BCK_PRS_MSB_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x000000ec)
-#define HWIO_IPA_GSI_TOP_IC_WRITE_BCK_PRS_MSB_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x000000ec)
-#define HWIO_IPA_GSI_TOP_IC_UCONTROLLER_GPR_BCK_PRS_LSB_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x000000f0)
-#define HWIO_IPA_GSI_TOP_IC_UCONTROLLER_GPR_BCK_PRS_LSB_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x000000f0)
-#define HWIO_IPA_GSI_TOP_IC_UCONTROLLER_GPR_BCK_PRS_LSB_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x000000f0)
-#define HWIO_IPA_GSI_TOP_IC_UCONTROLLER_GPR_BCK_PRS_MSB_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x000000f4)
-#define HWIO_IPA_GSI_TOP_IC_UCONTROLLER_GPR_BCK_PRS_MSB_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x000000f4)
-#define HWIO_IPA_GSI_TOP_IC_UCONTROLLER_GPR_BCK_PRS_MSB_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x000000f4)
-#define HWIO_IPA_GSI_TOP_IC_INT_WEIGHT_REE_ADDR (IPA_GSI_TOP_GSI_REG_BASE \
-						 + 0x00000100)
-#define HWIO_IPA_GSI_TOP_IC_INT_WEIGHT_REE_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000100)
-#define HWIO_IPA_GSI_TOP_IC_INT_WEIGHT_REE_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000100)
-#define HWIO_IPA_GSI_TOP_IC_INT_WEIGHT_EVT_ENG_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000104)
-#define HWIO_IPA_GSI_TOP_IC_INT_WEIGHT_EVT_ENG_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000104)
-#define HWIO_IPA_GSI_TOP_IC_INT_WEIGHT_EVT_ENG_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000104)
-#define HWIO_IPA_GSI_TOP_IC_INT_WEIGHT_INT_ENG_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000108)
-#define HWIO_IPA_GSI_TOP_IC_INT_WEIGHT_INT_ENG_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000108)
-#define HWIO_IPA_GSI_TOP_IC_INT_WEIGHT_INT_ENG_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000108)
-#define HWIO_IPA_GSI_TOP_IC_INT_WEIGHT_CSR_ADDR (IPA_GSI_TOP_GSI_REG_BASE \
-						 + 0x0000010c)
-#define HWIO_IPA_GSI_TOP_IC_INT_WEIGHT_CSR_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000010c)
-#define HWIO_IPA_GSI_TOP_IC_INT_WEIGHT_CSR_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000010c)
-#define HWIO_IPA_GSI_TOP_IC_INT_WEIGHT_TLV_ENG_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000110)
-#define HWIO_IPA_GSI_TOP_IC_INT_WEIGHT_TLV_ENG_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000110)
-#define HWIO_IPA_GSI_TOP_IC_INT_WEIGHT_TLV_ENG_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000110)
-#define HWIO_IPA_GSI_TOP_IC_INT_WEIGHT_TIMER_ENG_ADDR (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000114)
-#define HWIO_IPA_GSI_TOP_IC_INT_WEIGHT_TIMER_ENG_PHYS (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000114)
-#define HWIO_IPA_GSI_TOP_IC_INT_WEIGHT_TIMER_ENG_OFFS (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000114)
-#define HWIO_IPA_GSI_TOP_IC_INT_WEIGHT_DB_ENG_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000118)
-#define HWIO_IPA_GSI_TOP_IC_INT_WEIGHT_DB_ENG_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000118)
-#define HWIO_IPA_GSI_TOP_IC_INT_WEIGHT_DB_ENG_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000118)
-#define HWIO_IPA_GSI_TOP_IC_INT_WEIGHT_RD_WR_ENG_ADDR (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000011c)
-#define HWIO_IPA_GSI_TOP_IC_INT_WEIGHT_RD_WR_ENG_PHYS (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000011c)
-#define HWIO_IPA_GSI_TOP_IC_INT_WEIGHT_RD_WR_ENG_OFFS (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000011c)
-#define HWIO_IPA_GSI_TOP_IC_INT_WEIGHT_UCONTROLLER_ENG_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000120)
-#define HWIO_IPA_GSI_TOP_IC_INT_WEIGHT_UCONTROLLER_ENG_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000120)
-#define HWIO_IPA_GSI_TOP_IC_INT_WEIGHT_UCONTROLLER_ENG_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000120)
-#define HWIO_IPA_GSI_TOP_IC_INT_WEIGHT_SDMA_ADDR (IPA_GSI_TOP_GSI_REG_BASE \
-						  + 0x00000124)
-#define HWIO_IPA_GSI_TOP_IC_INT_WEIGHT_SDMA_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000124)
-#define HWIO_IPA_GSI_TOP_IC_INT_WEIGHT_SDMA_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000124)
-#define HWIO_IPA_GSI_TOP_GSI_SDMA_CFG_ADDR (IPA_GSI_TOP_GSI_REG_BASE + \
-					    0x0000003c)
-#define HWIO_IPA_GSI_TOP_GSI_SDMA_CFG_PHYS (IPA_GSI_TOP_GSI_REG_BASE_PHYS \
-					    + 0x0000003c)
-#define HWIO_IPA_GSI_TOP_GSI_SDMA_CFG_OFFS (IPA_GSI_TOP_GSI_REG_BASE_OFFS \
-					    + 0x0000003c)
-#define HWIO_IPA_GSI_TOP_GSI_SDMA_CACHEATTR_ADDR (IPA_GSI_TOP_GSI_REG_BASE \
-						  + 0x00000094)
-#define HWIO_IPA_GSI_TOP_GSI_SDMA_CACHEATTR_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000094)
-#define HWIO_IPA_GSI_TOP_GSI_SDMA_CACHEATTR_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000094)
-#define HWIO_IPA_GSI_TOP_GSI_SDMA_SG_IOVEC_LSB_n_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000140 + 0x8 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_SDMA_SG_IOVEC_LSB_n_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000140 + 0x8 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_SDMA_SG_IOVEC_LSB_n_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000140 + 0x8 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_SDMA_SG_IOVEC_MSB_n_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000144 + 0x8 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_SDMA_SG_IOVEC_MSB_n_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000144 + 0x8 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_SDMA_SG_IOVEC_MSB_n_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000144 + 0x8 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_MANAGER_EE_QOS_n_ADDR(n) (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000300 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_MANAGER_EE_QOS_n_PHYS(n) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000300 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_MANAGER_EE_QOS_n_OFFS(n) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000300 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_CH_CNTXT_BASE_ADDR_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000200)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_CH_CNTXT_BASE_ADDR_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000200)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_CH_CNTXT_BASE_ADDR_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000200)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_CH_CNTXT_BASE_ADDR_RMSK 0xffff
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_CH_CNTXT_BASE_ADDR_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_CH_CNTXT_BASE_ADDR_IN \
-	in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_CH_CNTXT_BASE_ADDR_ADDR,	\
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_CH_CNTXT_BASE_ADDR_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_CH_CNTXT_BASE_ADDR_INM(m) \
-	in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_CH_CNTXT_BASE_ADDR_ADDR,	\
-		m)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_CH_CNTXT_BASE_ADDR_OUT(v) out_dword( \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_CH_CNTXT_BASE_ADDR_ADDR,	\
-		v)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_CH_CNTXT_BASE_ADDR_OUTM(m, \
-							       v) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_CH_CNTXT_BASE_ADDR_ADDR,	\
-		m, \
-		v, \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_CH_CNTXT_BASE_ADDR_IN)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_CH_CNTXT_BASE_ADDR_SHRAM_PTR_BMSK \
-	0xffff
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_CH_CNTXT_BASE_ADDR_SHRAM_PTR_SHFT \
-	0x0
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EV_CNTXT_BASE_ADDR_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000204)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EV_CNTXT_BASE_ADDR_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000204)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EV_CNTXT_BASE_ADDR_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000204)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EV_CNTXT_BASE_ADDR_RMSK 0xffff
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EV_CNTXT_BASE_ADDR_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EV_CNTXT_BASE_ADDR_IN \
-	in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EV_CNTXT_BASE_ADDR_ADDR,	\
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EV_CNTXT_BASE_ADDR_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EV_CNTXT_BASE_ADDR_INM(m) \
-	in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EV_CNTXT_BASE_ADDR_ADDR,	\
-		m)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EV_CNTXT_BASE_ADDR_OUT(v) out_dword( \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EV_CNTXT_BASE_ADDR_ADDR,	\
-		v)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EV_CNTXT_BASE_ADDR_OUTM(m, \
-							       v) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EV_CNTXT_BASE_ADDR_ADDR,	\
-		m, \
-		v, \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EV_CNTXT_BASE_ADDR_IN)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EV_CNTXT_BASE_ADDR_SHRAM_PTR_BMSK \
-	0xffff
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EV_CNTXT_BASE_ADDR_SHRAM_PTR_SHFT \
-	0x0
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_STORAGE_BASE_ADDR_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000208)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_STORAGE_BASE_ADDR_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000208)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_STORAGE_BASE_ADDR_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000208)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_STORAGE_BASE_ADDR_RMSK 0xffff
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_STORAGE_BASE_ADDR_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_STORAGE_BASE_ADDR_IN \
-	in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_STORAGE_BASE_ADDR_ADDR, \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_STORAGE_BASE_ADDR_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_STORAGE_BASE_ADDR_INM(m) \
-	in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_STORAGE_BASE_ADDR_ADDR, \
-		m)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_STORAGE_BASE_ADDR_OUT(v) \
-	out_dword(HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_STORAGE_BASE_ADDR_ADDR, \
-		  v)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_STORAGE_BASE_ADDR_OUTM(m, \
-								 v) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_STORAGE_BASE_ADDR_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_STORAGE_BASE_ADDR_IN)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_STORAGE_BASE_ADDR_SHRAM_PTR_BMSK \
-	0xffff
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_STORAGE_BASE_ADDR_SHRAM_PTR_SHFT \
-	0x0
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_ESC_BUF_BASE_ADDR_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000020c)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_ESC_BUF_BASE_ADDR_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000020c)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_ESC_BUF_BASE_ADDR_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000020c)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_ESC_BUF_BASE_ADDR_RMSK 0xffff
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_ESC_BUF_BASE_ADDR_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_ESC_BUF_BASE_ADDR_IN \
-	in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_ESC_BUF_BASE_ADDR_ADDR, \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_ESC_BUF_BASE_ADDR_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_ESC_BUF_BASE_ADDR_INM(m) \
-	in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_ESC_BUF_BASE_ADDR_ADDR, \
-		m)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_ESC_BUF_BASE_ADDR_OUT(v) \
-	out_dword(HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_ESC_BUF_BASE_ADDR_ADDR, \
-		  v)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_ESC_BUF_BASE_ADDR_OUTM(m, \
-								 v) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_ESC_BUF_BASE_ADDR_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_ESC_BUF_BASE_ADDR_IN)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_ESC_BUF_BASE_ADDR_SHRAM_PTR_BMSK \
-	0xffff
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_RE_ESC_BUF_BASE_ADDR_SHRAM_PTR_SHFT \
-	0x0
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EE_SCRACH_BASE_ADDR_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000240)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EE_SCRACH_BASE_ADDR_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000240)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EE_SCRACH_BASE_ADDR_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000240)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EE_SCRACH_BASE_ADDR_RMSK 0xffff
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EE_SCRACH_BASE_ADDR_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EE_SCRACH_BASE_ADDR_IN \
-	in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EE_SCRACH_BASE_ADDR_ADDR, \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EE_SCRACH_BASE_ADDR_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EE_SCRACH_BASE_ADDR_INM(m) \
-	in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EE_SCRACH_BASE_ADDR_ADDR, \
-		m)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EE_SCRACH_BASE_ADDR_OUT(v) \
-	out_dword(HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EE_SCRACH_BASE_ADDR_ADDR, \
-		  v)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EE_SCRACH_BASE_ADDR_OUTM(m, \
-								v) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EE_SCRACH_BASE_ADDR_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EE_SCRACH_BASE_ADDR_IN)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EE_SCRACH_BASE_ADDR_SHRAM_PTR_BMSK \
-	0xffff
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EE_SCRACH_BASE_ADDR_SHRAM_PTR_SHFT \
-	0x0
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_FUNC_STACK_BASE_ADDR_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000244)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_FUNC_STACK_BASE_ADDR_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000244)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_FUNC_STACK_BASE_ADDR_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000244)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_FUNC_STACK_BASE_ADDR_RMSK 0xffff
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_FUNC_STACK_BASE_ADDR_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_FUNC_STACK_BASE_ADDR_IN \
-	in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_FUNC_STACK_BASE_ADDR_ADDR, \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_FUNC_STACK_BASE_ADDR_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_FUNC_STACK_BASE_ADDR_INM(m) \
-	in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_FUNC_STACK_BASE_ADDR_ADDR, \
-		m)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_FUNC_STACK_BASE_ADDR_OUT(v) \
-	out_dword(HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_FUNC_STACK_BASE_ADDR_ADDR, \
-		  v)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_FUNC_STACK_BASE_ADDR_OUTM(m, \
-								 v) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_FUNC_STACK_BASE_ADDR_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_FUNC_STACK_BASE_ADDR_IN)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_FUNC_STACK_BASE_ADDR_SHRAM_PTR_BMSK \
-	0xffff
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_FUNC_STACK_BASE_ADDR_SHRAM_PTR_SHFT \
-	0x0
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_MCS_SCRATCH_BASE_ADDR_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000248)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_MCS_SCRATCH_BASE_ADDR_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000248)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_MCS_SCRATCH_BASE_ADDR_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000248)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_MCS_SCRATCH1_BASE_ADDR_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000024c)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_MCS_SCRATCH1_BASE_ADDR_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000024c)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_MCS_SCRATCH1_BASE_ADDR_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000024c)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_MCS_SCRATCH2_BASE_ADDR_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000250)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_MCS_SCRATCH2_BASE_ADDR_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000250)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_MCS_SCRATCH2_BASE_ADDR_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000250)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_CH_VP_TRANS_TABLE_BASE_ADDR_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000254)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_CH_VP_TRANS_TABLE_BASE_ADDR_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000254)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_CH_VP_TRANS_TABLE_BASE_ADDR_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000254)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EV_VP_TRANS_TABLE_BASE_ADDR_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000258)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EV_VP_TRANS_TABLE_BASE_ADDR_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000258)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EV_VP_TRANS_TABLE_BASE_ADDR_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000258)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_USER_INFO_DATA_BASE_ADDR_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000025c)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_USER_INFO_DATA_BASE_ADDR_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000025c)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_USER_INFO_DATA_BASE_ADDR_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000025c)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EE_CMD_FIFO_BASE_ADDR_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000260)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EE_CMD_FIFO_BASE_ADDR_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000260)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_EE_CMD_FIFO_BASE_ADDR_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000260)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_CH_CMD_FIFO_BASE_ADDR_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000264)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_CH_CMD_FIFO_BASE_ADDR_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000264)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_PTR_CH_CMD_FIFO_BASE_ADDR_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000264)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_CMD_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000400)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_CMD_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000400)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_CMD_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000400)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_CMD_RMSK 0xfff
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_CMD_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_CMD_IN in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_CMD_ADDR, \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_CMD_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_CMD_INM(m) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_CMD_ADDR, \
-		m)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_CMD_OUT(v) out_dword( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_CMD_ADDR, \
-		v)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_CMD_OUTM(m, \
-						  v) out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_CMD_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_CMD_IN)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_CMD_IRAM_PTR_BMSK 0xfff
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_CMD_IRAM_PTR_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EE_GENERIC_CMD_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000404)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EE_GENERIC_CMD_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000404)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EE_GENERIC_CMD_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000404)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EE_GENERIC_CMD_RMSK 0xfff
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EE_GENERIC_CMD_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EE_GENERIC_CMD_IN in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EE_GENERIC_CMD_ADDR, \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EE_GENERIC_CMD_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EE_GENERIC_CMD_INM(m) \
-	in_dword_masked(HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EE_GENERIC_CMD_ADDR, \
-			m)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EE_GENERIC_CMD_OUT(v) out_dword( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EE_GENERIC_CMD_ADDR, \
-		v)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EE_GENERIC_CMD_OUTM(m, \
-							  v) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EE_GENERIC_CMD_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EE_GENERIC_CMD_IN)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EE_GENERIC_CMD_IRAM_PTR_BMSK 0xfff
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EE_GENERIC_CMD_IRAM_PTR_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_TLV_CH_NOT_FULL_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000408)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_TLV_CH_NOT_FULL_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000408)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_TLV_CH_NOT_FULL_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000408)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DB_ADDR (IPA_GSI_TOP_GSI_REG_BASE \
-						  + 0x00000418)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DB_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000418)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DB_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000418)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DB_RMSK 0xfff
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DB_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DB_IN in_dword_masked(	\
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DB_ADDR, \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DB_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DB_INM(m) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DB_ADDR, \
-		m)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DB_OUT(v) out_dword( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DB_ADDR, \
-		v)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DB_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DB_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DB_IN)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DB_IRAM_PTR_BMSK 0xfff
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DB_IRAM_PTR_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EV_DB_ADDR (IPA_GSI_TOP_GSI_REG_BASE \
-						  + 0x0000041c)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EV_DB_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000041c)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EV_DB_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000041c)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EV_DB_RMSK 0xfff
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EV_DB_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EV_DB_IN in_dword_masked(	\
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EV_DB_ADDR, \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EV_DB_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EV_DB_INM(m) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EV_DB_ADDR, \
-		m)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EV_DB_OUT(v) out_dword( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EV_DB_ADDR, \
-		v)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EV_DB_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EV_DB_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EV_DB_IN)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EV_DB_IRAM_PTR_BMSK 0xfff
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EV_DB_IRAM_PTR_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_NEW_RE_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000420)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_NEW_RE_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000420)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_NEW_RE_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000420)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_NEW_RE_RMSK 0xfff
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_NEW_RE_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_NEW_RE_IN in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_NEW_RE_ADDR, \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_NEW_RE_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_NEW_RE_INM(m) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_NEW_RE_ADDR, \
-		m)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_NEW_RE_OUT(v) out_dword( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_NEW_RE_ADDR, \
-		v)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_NEW_RE_OUTM(m, \
-						  v) out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_NEW_RE_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_NEW_RE_IN)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_NEW_RE_IRAM_PTR_BMSK 0xfff
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_NEW_RE_IRAM_PTR_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DIS_COMP_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000424)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DIS_COMP_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000424)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DIS_COMP_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000424)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DIS_COMP_RMSK 0xfff
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DIS_COMP_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DIS_COMP_IN in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DIS_COMP_ADDR,	\
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DIS_COMP_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DIS_COMP_INM(m) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DIS_COMP_ADDR,	\
-		m)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DIS_COMP_OUT(v) out_dword( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DIS_COMP_ADDR,	\
-		v)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DIS_COMP_OUTM(m, \
-						       v) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DIS_COMP_ADDR,	\
-		m, \
-		v, \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DIS_COMP_IN)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DIS_COMP_IRAM_PTR_BMSK 0xfff
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_DIS_COMP_IRAM_PTR_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_EMPTY_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000428)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_EMPTY_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000428)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_EMPTY_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000428)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_EMPTY_RMSK 0xfff
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_EMPTY_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_EMPTY_IN in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_EMPTY_ADDR, \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_EMPTY_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_EMPTY_INM(m) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_EMPTY_ADDR, \
-		m)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_EMPTY_OUT(v) out_dword( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_EMPTY_ADDR, \
-		v)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_EMPTY_OUTM(m, \
-						    v) out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_EMPTY_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_EMPTY_IN)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_EMPTY_IRAM_PTR_BMSK 0xfff
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_CH_EMPTY_IRAM_PTR_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EVENT_GEN_COMP_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000042c)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EVENT_GEN_COMP_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000042c)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EVENT_GEN_COMP_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000042c)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EVENT_GEN_COMP_RMSK 0xfff
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EVENT_GEN_COMP_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EVENT_GEN_COMP_IN in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EVENT_GEN_COMP_ADDR, \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EVENT_GEN_COMP_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EVENT_GEN_COMP_INM(m) \
-	in_dword_masked(HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EVENT_GEN_COMP_ADDR, \
-			m)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EVENT_GEN_COMP_OUT(v) out_dword( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EVENT_GEN_COMP_ADDR, \
-		v)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EVENT_GEN_COMP_OUTM(m, \
-							  v) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EVENT_GEN_COMP_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EVENT_GEN_COMP_IN)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EVENT_GEN_COMP_IRAM_PTR_BMSK 0xfff
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_EVENT_GEN_COMP_IRAM_PTR_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_0_ADDR (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000430)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_0_PHYS (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000430)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_0_OFFS (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000430)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_2_ADDR (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000434)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_2_PHYS (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000434)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_2_OFFS (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000434)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_1_ADDR (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000438)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_1_PHYS (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000438)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_PERIPH_IF_TLV_IN_1_OFFS (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000438)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_TIMER_EXPIRED_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000043c)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_TIMER_EXPIRED_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000043c)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_TIMER_EXPIRED_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000043c)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_TIMER_EXPIRED_RMSK 0xfff
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_TIMER_EXPIRED_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_TIMER_EXPIRED_IN in_dword_masked(	\
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_TIMER_EXPIRED_ADDR, \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_TIMER_EXPIRED_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_TIMER_EXPIRED_INM(m) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_TIMER_EXPIRED_ADDR, \
-		m)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_TIMER_EXPIRED_OUT(v) out_dword( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_TIMER_EXPIRED_ADDR, \
-		v)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_TIMER_EXPIRED_OUTM(m, \
-							 v) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_TIMER_EXPIRED_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_TIMER_EXPIRED_IN)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_TIMER_EXPIRED_IRAM_PTR_BMSK 0xfff
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_TIMER_EXPIRED_IRAM_PTR_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_WRITE_ENG_COMP_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000440)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_WRITE_ENG_COMP_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000440)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_WRITE_ENG_COMP_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000440)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_WRITE_ENG_COMP_RMSK 0xfff
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_WRITE_ENG_COMP_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_WRITE_ENG_COMP_IN in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_WRITE_ENG_COMP_ADDR, \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_WRITE_ENG_COMP_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_WRITE_ENG_COMP_INM(m) \
-	in_dword_masked(HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_WRITE_ENG_COMP_ADDR, \
-			m)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_WRITE_ENG_COMP_OUT(v) out_dword( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_WRITE_ENG_COMP_ADDR, \
-		v)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_WRITE_ENG_COMP_OUTM(m, \
-							  v) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_WRITE_ENG_COMP_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_WRITE_ENG_COMP_IN)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_WRITE_ENG_COMP_IRAM_PTR_BMSK 0xfff
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_WRITE_ENG_COMP_IRAM_PTR_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_READ_ENG_COMP_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000444)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_READ_ENG_COMP_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000444)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_READ_ENG_COMP_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000444)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_READ_ENG_COMP_RMSK 0xfff
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_READ_ENG_COMP_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_READ_ENG_COMP_IN in_dword_masked(	\
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_READ_ENG_COMP_ADDR, \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_READ_ENG_COMP_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_READ_ENG_COMP_INM(m) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_READ_ENG_COMP_ADDR, \
-		m)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_READ_ENG_COMP_OUT(v) out_dword( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_READ_ENG_COMP_ADDR, \
-		v)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_READ_ENG_COMP_OUTM(m, \
-							 v) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_READ_ENG_COMP_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_READ_ENG_COMP_IN)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_READ_ENG_COMP_IRAM_PTR_BMSK 0xfff
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_READ_ENG_COMP_IRAM_PTR_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_UC_GP_INT_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000448)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_UC_GP_INT_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000448)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_UC_GP_INT_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000448)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_UC_GP_INT_RMSK 0xfff
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_UC_GP_INT_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_UC_GP_INT_IN in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_UC_GP_INT_ADDR, \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_UC_GP_INT_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_UC_GP_INT_INM(m) in_dword_masked(	\
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_UC_GP_INT_ADDR, \
-		m)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_UC_GP_INT_OUT(v) out_dword( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_UC_GP_INT_ADDR, \
-		v)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_UC_GP_INT_OUTM(m,	\
-						     v)	\
-	out_dword_masked_ns(HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_UC_GP_INT_ADDR, \
-			    m, \
-			    v, \
-			    HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_UC_GP_INT_IN)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_UC_GP_INT_IRAM_PTR_BMSK 0xfff
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_UC_GP_INT_IRAM_PTR_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_INT_MOD_STOPPED_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000044c)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_INT_MOD_STOPPED_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000044c)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_INT_MOD_STOPPED_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000044c)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_INT_MOD_STOPPED_RMSK 0xfff
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_INT_MOD_STOPPED_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_INT_MOD_STOPPED_IN in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_INT_MOD_STOPPED_ADDR, \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_INT_MOD_STOPPED_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_INT_MOD_STOPPED_INM(m) \
-	in_dword_masked(HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_INT_MOD_STOPPED_ADDR, \
-			m)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_INT_MOD_STOPPED_OUT(v) out_dword( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_INT_MOD_STOPPED_ADDR, \
-		v)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_INT_MOD_STOPPED_OUTM(m, \
-							   v) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_INT_MOD_STOPPED_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_INT_MOD_STOPPED_IN)
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_INT_MOD_STOPPED_IRAM_PTR_BMSK 0xfff
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_INT_MOD_STOPPED_IRAM_PTR_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_SDMA_INT_n_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000450 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_SDMA_INT_n_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000450 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_IRAM_PTR_SDMA_INT_n_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000450 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_INST_RAM_n_ADDR(n) (IPA_GSI_TOP_GSI_REG_BASE \
-						 + 0x0001b000 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_INST_RAM_n_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0001b000 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_INST_RAM_n_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0001b000 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_INST_RAM_n_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_GSI_INST_RAM_n_MAXn 8191
-#define HWIO_IPA_GSI_TOP_GSI_INST_RAM_n_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_GSI_INST_RAM_n_INI(n) \
-	in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_INST_RAM_n_ADDR(n), \
-		HWIO_IPA_GSI_TOP_GSI_INST_RAM_n_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_INST_RAM_n_INMI(n, mask) \
-	in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_INST_RAM_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_GSI_TOP_GSI_INST_RAM_n_OUTI(n, val) \
-	out_dword( \
-		HWIO_IPA_GSI_TOP_GSI_INST_RAM_n_ADDR(n), \
-		val)
-#define HWIO_IPA_GSI_TOP_GSI_INST_RAM_n_OUTMI(n, mask, \
-					      val) out_dword_masked_ns(	\
-		HWIO_IPA_GSI_TOP_GSI_INST_RAM_n_ADDR( \
-			n), \
-		mask, \
-		val, \
-		HWIO_IPA_GSI_TOP_GSI_INST_RAM_n_INI(n))
-#define HWIO_IPA_GSI_TOP_GSI_INST_RAM_n_INST_BYTE_3_BMSK 0xff000000
-#define HWIO_IPA_GSI_TOP_GSI_INST_RAM_n_INST_BYTE_3_SHFT 0x18
-#define HWIO_IPA_GSI_TOP_GSI_INST_RAM_n_INST_BYTE_2_BMSK 0xff0000
-#define HWIO_IPA_GSI_TOP_GSI_INST_RAM_n_INST_BYTE_2_SHFT 0x10
-#define HWIO_IPA_GSI_TOP_GSI_INST_RAM_n_INST_BYTE_1_BMSK 0xff00
-#define HWIO_IPA_GSI_TOP_GSI_INST_RAM_n_INST_BYTE_1_SHFT 0x8
-#define HWIO_IPA_GSI_TOP_GSI_INST_RAM_n_INST_BYTE_0_BMSK 0xff
-#define HWIO_IPA_GSI_TOP_GSI_INST_RAM_n_INST_BYTE_0_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_n_ADDR(n) (IPA_GSI_TOP_GSI_REG_BASE + \
-					      0x00002000 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_n_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00002000 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_n_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00002000 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_n_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_n_MAXn 1343
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_n_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_n_INI(n) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_n_ADDR(n), \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_n_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_n_ADDR(n), \
-		val)
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_n_OUTMI(n, mask, \
-					   val) out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_n_ADDR( \
-			n), \
-		mask, \
-		val, \
-		HWIO_IPA_GSI_TOP_GSI_SHRAM_n_INI(n))
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_n_SHRAM_BMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_GSI_SHRAM_n_SHRAM_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_GSI_MAP_EE_n_CH_k_VP_TABLE_ADDR(n, \
-							 k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00003800 + 0x80 * (n) + 0x4 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_GSI_MAP_EE_n_CH_k_VP_TABLE_PHYS(n, \
-							 k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00003800 + 0x80 * (n) + \
-		0x4 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_GSI_MAP_EE_n_CH_k_VP_TABLE_OFFS(n, \
-							 k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00003800 + 0x80 * (n) + \
-		0x4 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_GSI_MAP_EE_n_CH_k_VP_TABLE_RMSK 0x3f
-#define HWIO_IPA_GSI_TOP_GSI_MAP_EE_n_CH_k_VP_TABLE_MAXn 2
-#define HWIO_IPA_GSI_TOP_GSI_MAP_EE_n_CH_k_VP_TABLE_MAXk 22
-#define HWIO_IPA_GSI_TOP_GSI_MAP_EE_n_CH_k_VP_TABLE_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_GSI_MAP_EE_n_CH_k_VP_TABLE_INI2(n, \
-							 k) \
-	in_dword_masked(HWIO_IPA_GSI_TOP_GSI_MAP_EE_n_CH_k_VP_TABLE_ADDR( \
-				n, \
-				k), \
-			HWIO_IPA_GSI_TOP_GSI_MAP_EE_n_CH_k_VP_TABLE_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_MAP_EE_n_CH_k_VP_TABLE_INMI2(n, k,	\
-							  mask)	\
-	in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_MAP_EE_n_CH_k_VP_TABLE_ADDR(n, k),	\
-		mask)
-#define HWIO_IPA_GSI_TOP_GSI_MAP_EE_n_CH_k_VP_TABLE_OUTI2(n, k,	\
-							  val) out_dword( \
-		HWIO_IPA_GSI_TOP_GSI_MAP_EE_n_CH_k_VP_TABLE_ADDR(n, \
-								 k), \
-		val)
-#define HWIO_IPA_GSI_TOP_GSI_MAP_EE_n_CH_k_VP_TABLE_OUTMI2(n, k, mask, \
-							   val)	\
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_GSI_MAP_EE_n_CH_k_VP_TABLE_ADDR(n, k),	\
-		mask, \
-		val, \
-		HWIO_IPA_GSI_TOP_GSI_MAP_EE_n_CH_k_VP_TABLE_INI2(n, k))
-#define HWIO_IPA_GSI_TOP_GSI_MAP_EE_n_CH_k_VP_TABLE_VALID_BMSK 0x20
-#define HWIO_IPA_GSI_TOP_GSI_MAP_EE_n_CH_k_VP_TABLE_VALID_SHFT 0x5
-#define HWIO_IPA_GSI_TOP_GSI_MAP_EE_n_CH_k_VP_TABLE_PHY_CH_BMSK 0x1f
-#define HWIO_IPA_GSI_TOP_GSI_MAP_EE_n_CH_k_VP_TABLE_PHY_CH_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_ADDR (IPA_GSI_TOP_GSI_REG_BASE + \
-						0x00001000)
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00001000)
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00001000)
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_RMSK 0xf00ff
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_IN in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_ADDR,	\
-		HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_INM(m) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_ADDR,	\
-		m)
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_OUT(v) out_dword( \
-		HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_ADDR,	\
-		v)
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_ADDR,	\
-		m, \
-		v, \
-		HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_IN)
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_HW_EVENTS_SEL_BMSK 0xf0000
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_HW_EVENTS_SEL_SHFT 0x10
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_BMSK 0xff
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_ZEROS_FVAL 0x0
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_MCS_0_FVAL 0x1
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_MCS_1_FVAL 0x2
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_MCS_2_FVAL 0x3
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_MCS_3_FVAL 0x4
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_MCS_4_FVAL 0x5
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_DB_ENG_FVAL 0x9
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_REE_0_FVAL 0xb
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_REE_1_FVAL 0xc
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_REE_2_FVAL 0xd
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_REE_3_FVAL 0xe
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_REE_4_FVAL 0xf
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_REE_5_FVAL 0x10
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_REE_6_FVAL 0x11
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_REE_7_FVAL 0x12
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_EVE_0_FVAL 0x13
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_EVE_1_FVAL 0x14
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_EVE_2_FVAL 0x15
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_EVE_3_FVAL 0x16
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_EVE_4_FVAL 0x17
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_EVE_5_FVAL 0x18
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_IE_0_FVAL 0x1b
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_IE_1_FVAL 0x1c
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_IE_2_FVAL 0x1d
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_IC_0_FVAL 0x1f
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_IC_1_FVAL 0x20
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_IC_2_FVAL 0x21
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_IC_3_FVAL 0x22
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_IC_4_FVAL 0x23
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_MOQA_0_FVAL 0x27
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_MOQA_1_FVAL 0x28
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_MOQA_2_FVAL 0x29
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_MOQA_3_FVAL 0x2a
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_TMR_0_FVAL 0x2b
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_TMR_1_FVAL 0x2c
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_TMR_2_FVAL 0x2d
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_TMR_3_FVAL 0x2e
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_RD_WR_0_FVAL \
-	0x33
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_RD_WR_1_FVAL \
-	0x34
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_RD_WR_2_FVAL \
-	0x35
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_RD_WR_3_FVAL \
-	0x36
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_CSR_FVAL 0x3a
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_SDMA_0_FVAL 0x3c
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_SMDA_1_FVAL 0x3d
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_CSR_1_FVAL 0x3e
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_CSR_2_FVAL 0x3f
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_MCS_5_FVAL 0x40
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_IC_5_FVAL 0x41
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_CSR_3_FVAL 0x42
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_TLV_0_FVAL 0x43
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_SEL_GSI_TESTBUS_SEL_REE_8_FVAL 0x44
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_REG_ADDR (IPA_GSI_TOP_GSI_REG_BASE + \
-						0x00001008)
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_REG_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00001008)
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_REG_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00001008)
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_REG_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_REG_ATTR 0x1
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_REG_IN in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_TEST_BUS_REG_ADDR,	\
-		HWIO_IPA_GSI_TOP_GSI_TEST_BUS_REG_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_REG_INM(m) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_TEST_BUS_REG_ADDR,	\
-		m)
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_REG_GSI_TESTBUS_REG_BMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_GSI_TEST_BUS_REG_GSI_TESTBUS_REG_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_ADDR (IPA_GSI_TOP_GSI_REG_BASE \
-						  + 0x00001010)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00001010)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00001010)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_RMSK 0x1fff
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_ATTR 0x1
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_IN in_dword_masked(	\
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_ADDR, \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_INM(m) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_ADDR, \
-		m)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_SDMA_BUSY_BMSK 0x1000
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_SDMA_BUSY_SHFT 0xc
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_IC_BUSY_BMSK 0x800
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_IC_BUSY_SHFT 0xb
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_UC_BUSY_BMSK 0x400
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_UC_BUSY_SHFT 0xa
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_DBG_CNT_BUSY_BMSK 0x200
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_DBG_CNT_BUSY_SHFT 0x9
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_DB_ENG_BUSY_BMSK 0x100
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_DB_ENG_BUSY_SHFT 0x8
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_REE_PWR_CLPS_BUSY_BMSK 0x80
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_REE_PWR_CLPS_BUSY_SHFT 0x7
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_INT_ENG_BUSY_BMSK 0x40
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_INT_ENG_BUSY_SHFT 0x6
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_EV_ENG_BUSY_BMSK 0x20
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_EV_ENG_BUSY_SHFT 0x5
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_RD_WR_BUSY_BMSK 0x10
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_RD_WR_BUSY_SHFT 0x4
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_TIMER_BUSY_BMSK 0x8
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_TIMER_BUSY_SHFT 0x3
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_MCS_BUSY_BMSK 0x4
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_MCS_BUSY_SHFT 0x2
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_REE_BUSY_BMSK 0x2
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_REE_BUSY_SHFT 0x1
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_CSR_BUSY_BMSK 0x1
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_BUSY_REG_CSR_BUSY_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_EVENT_PENDING_ADDR (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00001014)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_EVENT_PENDING_PHYS (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00001014)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_EVENT_PENDING_OFFS (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00001014)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_EVENT_PENDING_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_EVENT_PENDING_ATTR 0x1
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_EVENT_PENDING_IN in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_EVENT_PENDING_ADDR, \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_EVENT_PENDING_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_EVENT_PENDING_INM(m) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_EVENT_PENDING_ADDR, \
-		m)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_EVENT_PENDING_CHID_BIT_MAP_BMSK \
-	0xffffffff
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_EVENT_PENDING_CHID_BIT_MAP_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_TIMER_PENDING_ADDR (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00001018)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_TIMER_PENDING_PHYS (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00001018)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_TIMER_PENDING_OFFS (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00001018)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_TIMER_PENDING_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_TIMER_PENDING_ATTR 0x1
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_TIMER_PENDING_IN in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_TIMER_PENDING_ADDR, \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_TIMER_PENDING_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_TIMER_PENDING_INM(m) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_TIMER_PENDING_ADDR, \
-		m)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_TIMER_PENDING_CHID_BIT_MAP_BMSK \
-	0xffffffff
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_TIMER_PENDING_CHID_BIT_MAP_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_RD_WR_PENDING_ADDR (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000101c)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_RD_WR_PENDING_PHYS (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000101c)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_RD_WR_PENDING_OFFS (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000101c)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_RD_WR_PENDING_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_RD_WR_PENDING_ATTR 0x1
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_RD_WR_PENDING_IN in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_RD_WR_PENDING_ADDR, \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_RD_WR_PENDING_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_RD_WR_PENDING_INM(m) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_RD_WR_PENDING_ADDR, \
-		m)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_RD_WR_PENDING_CHID_BIT_MAP_BMSK \
-	0xffffffff
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_RD_WR_PENDING_CHID_BIT_MAP_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_COUNTER_CFGn_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00001200 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_COUNTER_CFGn_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00001200 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_COUNTER_CFGn_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00001200 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_COUNTERn_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00001240 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_COUNTERn_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00001240 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_COUNTERn_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00001240 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_COUNTERn_RMSK 0xffff
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_COUNTERn_MAXn 7
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_COUNTERn_ATTR 0x1
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_COUNTERn_INI(n) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_COUNTERn_ADDR(n), \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_COUNTERn_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_COUNTERn_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_COUNTERn_ADDR(n), \
-		mask)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_COUNTERn_COUNTER_VALUE_BMSK 0xffff
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_COUNTERn_COUNTER_VALUE_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_PC_FROM_SW_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00001040)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_PC_FROM_SW_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00001040)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_PC_FROM_SW_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00001040)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_PC_FROM_SW_RMSK 0xfff
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_PC_FROM_SW_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_PC_FROM_SW_IN in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_PC_FROM_SW_ADDR, \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_PC_FROM_SW_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_PC_FROM_SW_INM(m) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_PC_FROM_SW_ADDR, \
-		m)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_PC_FROM_SW_OUT(v) out_dword(	\
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_PC_FROM_SW_ADDR, \
-		v)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_PC_FROM_SW_OUTM(m, \
-						   v) out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_PC_FROM_SW_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_PC_FROM_SW_IN)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_PC_FROM_SW_IRAM_PTR_BMSK 0xfff
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_PC_FROM_SW_IRAM_PTR_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_STALL_ADDR (IPA_GSI_TOP_GSI_REG_BASE \
-						  + 0x00001044)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_STALL_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00001044)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_STALL_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00001044)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_STALL_RMSK 0x1
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_STALL_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_STALL_IN in_dword_masked(	\
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_STALL_ADDR, \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_STALL_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_STALL_INM(m) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_STALL_ADDR, \
-		m)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_STALL_OUT(v) out_dword( \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_STALL_ADDR, \
-		v)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_STALL_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_STALL_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_STALL_IN)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_STALL_MCS_STALL_BMSK 0x1
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_STALL_MCS_STALL_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_PC_FOR_DEBUG_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00001048)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_PC_FOR_DEBUG_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00001048)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_PC_FOR_DEBUG_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00001048)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_PC_FOR_DEBUG_RMSK 0xfff
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_PC_FOR_DEBUG_ATTR 0x1
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_PC_FOR_DEBUG_IN in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_PC_FOR_DEBUG_ADDR, \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_PC_FOR_DEBUG_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_PC_FOR_DEBUG_INM(m) in_dword_masked(	\
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_PC_FOR_DEBUG_ADDR, \
-		m)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_PC_FOR_DEBUG_IRAM_PTR_BMSK 0xfff
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_PC_FOR_DEBUG_IRAM_PTR_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_SEL_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00001050)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_SEL_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00001050)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_SEL_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00001050)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_CLR_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00001058)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_CLR_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00001058)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_CLR_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00001058)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_ERR_TRNS_ID_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00001060)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_ERR_TRNS_ID_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00001060)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_ERR_TRNS_ID_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00001060)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_ERR_TRNS_ID_RMSK 0x1ffff01
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_ERR_TRNS_ID_ATTR 0x1
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_ERR_TRNS_ID_IN in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_ERR_TRNS_ID_ADDR, \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_ERR_TRNS_ID_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_ERR_TRNS_ID_INM(m) \
-	in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_ERR_TRNS_ID_ADDR, \
-		m)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_ERR_TRNS_ID_ERR_SAVED_BMSK \
-	0x1000000
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_ERR_TRNS_ID_ERR_SAVED_SHFT 0x18
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_ERR_TRNS_ID_ERR_MID_BMSK \
-	0xff0000
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_ERR_TRNS_ID_ERR_MID_SHFT 0x10
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_ERR_TRNS_ID_ERR_TID_BMSK 0xff00
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_ERR_TRNS_ID_ERR_TID_SHFT 0x8
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_ERR_TRNS_ID_ERR_WRITE_BMSK 0x1
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_ERR_TRNS_ID_ERR_WRITE_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_0_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00001064)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_0_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00001064)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_0_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00001064)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_1_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00001068)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_1_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00001068)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_1_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00001068)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_2_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000106c)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_2_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000106c)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_2_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000106c)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_LAST_MISC_IDn_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00001070 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_LAST_MISC_IDn_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00001070 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_LAST_MISC_IDn_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00001070 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_RF_n_WRITE_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00001080 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_RF_n_WRITE_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00001080 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_RF_n_WRITE_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00001080 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_RF_n_READ_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00001100 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_RF_n_READ_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00001100 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_RF_n_READ_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00001100 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_RF_n_READ_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_RF_n_READ_MAXn 31
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_RF_n_READ_ATTR 0x1
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_RF_n_READ_INI(n) in_dword_masked(	\
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_RF_n_READ_ADDR(n), \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_RF_n_READ_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_RF_n_READ_INMI(n,	\
-						     mask) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_RF_n_READ_ADDR( \
-			n), \
-		mask)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_RF_n_READ_RF_REG_BMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_SW_RF_n_READ_RF_REG_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_EE_n_CH_k_VP_TABLE_ADDR(n, \
-							   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00001400 + 0x80 * (n) + 0x4 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_EE_n_CH_k_VP_TABLE_PHYS(n, \
-							   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00001400 + 0x80 * (n) + \
-		0x4 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_EE_n_CH_k_VP_TABLE_OFFS(n, \
-							   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00001400 + 0x80 * (n) + \
-		0x4 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_EE_n_EV_k_VP_TABLE_ADDR(n, \
-							   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00001600 + 0x80 * (n) + 0x4 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_EE_n_EV_k_VP_TABLE_PHYS(n, \
-							   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00001600 + 0x80 * (n) + \
-		0x4 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_EE_n_EV_k_VP_TABLE_OFFS(n, \
-							   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00001600 + 0x80 * (n) + \
-		0x4 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_EE_n_EV_k_VP_TABLE_RMSK 0x3f
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_EE_n_EV_k_VP_TABLE_MAXn 3
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_EE_n_EV_k_VP_TABLE_MAXk 19
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_EE_n_EV_k_VP_TABLE_ATTR 0x1
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_EE_n_EV_k_VP_TABLE_INI2(n, \
-							   k) \
-	in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_EE_n_EV_k_VP_TABLE_ADDR(n, k), \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_EE_n_EV_k_VP_TABLE_RMSK)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_EE_n_EV_k_VP_TABLE_INMI2(n, k, \
-							    mask) \
-	in_dword_masked( \
-		HWIO_IPA_GSI_TOP_GSI_DEBUG_EE_n_EV_k_VP_TABLE_ADDR(n, k), \
-		mask)
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_EE_n_EV_k_VP_TABLE_VALID_BMSK 0x20
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_EE_n_EV_k_VP_TABLE_VALID_SHFT 0x5
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_EE_n_EV_k_VP_TABLE_PHY_EV_CH_BMSK 0x1f
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_EE_n_EV_k_VP_TABLE_PHY_EV_CH_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_SDMA_TRANS_DB_n_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00001800 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_SDMA_TRANS_DB_n_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00001800 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_DEBUG_SDMA_TRANS_DB_n_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00001800 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_UC_SRC_IRQ_ADDR (IPA_GSI_TOP_GSI_REG_BASE + \
-					      0x00000500)
-#define HWIO_IPA_GSI_TOP_GSI_UC_SRC_IRQ_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000500)
-#define HWIO_IPA_GSI_TOP_GSI_UC_SRC_IRQ_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000500)
-#define HWIO_IPA_GSI_TOP_GSI_UC_SRC_IRQ_MSK_ADDR (IPA_GSI_TOP_GSI_REG_BASE \
-						  + 0x00000504)
-#define HWIO_IPA_GSI_TOP_GSI_UC_SRC_IRQ_MSK_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000504)
-#define HWIO_IPA_GSI_TOP_GSI_UC_SRC_IRQ_MSK_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000504)
-#define HWIO_IPA_GSI_TOP_GSI_UC_SRC_IRQ_CLR_ADDR (IPA_GSI_TOP_GSI_REG_BASE \
-						  + 0x00000508)
-#define HWIO_IPA_GSI_TOP_GSI_UC_SRC_IRQ_CLR_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000508)
-#define HWIO_IPA_GSI_TOP_GSI_UC_SRC_IRQ_CLR_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000508)
-#define HWIO_IPA_GSI_TOP_GSI_ACC_ARGS_n_ADDR(n) (IPA_GSI_TOP_GSI_REG_BASE \
-						 + 0x0000050c + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_ACC_ARGS_n_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000050c + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_ACC_ARGS_n_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000050c + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_ACC_ROUTINE_ADDR (IPA_GSI_TOP_GSI_REG_BASE + \
-					       0x00000524)
-#define HWIO_IPA_GSI_TOP_GSI_ACC_ROUTINE_PHYS (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000524)
-#define HWIO_IPA_GSI_TOP_GSI_ACC_ROUTINE_OFFS (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000524)
-#define HWIO_IPA_GSI_TOP_GSI_ACC_GO_ADDR (IPA_GSI_TOP_GSI_REG_BASE + \
-					  0x00000528)
-#define HWIO_IPA_GSI_TOP_GSI_ACC_GO_PHYS (IPA_GSI_TOP_GSI_REG_BASE_PHYS + \
-					  0x00000528)
-#define HWIO_IPA_GSI_TOP_GSI_ACC_GO_OFFS (IPA_GSI_TOP_GSI_REG_BASE_OFFS + \
-					  0x00000528)
-#define HWIO_IPA_GSI_TOP_GSI_ACC_2_UC_MCS_STTS_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000052c)
-#define HWIO_IPA_GSI_TOP_GSI_ACC_2_UC_MCS_STTS_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000052c)
-#define HWIO_IPA_GSI_TOP_GSI_ACC_2_UC_MCS_STTS_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000052c)
-#define HWIO_IPA_GSI_TOP_GSI_ACC_2_UC_MCS_RET_VAL_LSB_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000530)
-#define HWIO_IPA_GSI_TOP_GSI_ACC_2_UC_MCS_RET_VAL_LSB_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000530)
-#define HWIO_IPA_GSI_TOP_GSI_ACC_2_UC_MCS_RET_VAL_LSB_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000530)
-#define HWIO_IPA_GSI_TOP_GSI_ACC_2_UC_MCS_RET_VAL_MSB_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000534)
-#define HWIO_IPA_GSI_TOP_GSI_ACC_2_UC_MCS_RET_VAL_MSB_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000534)
-#define HWIO_IPA_GSI_TOP_GSI_ACC_2_UC_MCS_RET_VAL_MSB_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000534)
-#define HWIO_IPA_GSI_TOP_GSI_IC_2_UC_MCS_VLD_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000538)
-#define HWIO_IPA_GSI_TOP_GSI_IC_2_UC_MCS_VLD_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000538)
-#define HWIO_IPA_GSI_TOP_GSI_IC_2_UC_MCS_VLD_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000538)
-#define HWIO_IPA_GSI_TOP_GSI_IC_2_UC_MCS_PC_ADDR (IPA_GSI_TOP_GSI_REG_BASE \
-						  + 0x0000053c)
-#define HWIO_IPA_GSI_TOP_GSI_IC_2_UC_MCS_PC_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000053c)
-#define HWIO_IPA_GSI_TOP_GSI_IC_2_UC_MCS_PC_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000053c)
-#define HWIO_IPA_GSI_TOP_GSI_IC_2_UC_MCS_ARGS_n_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000540 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_IC_2_UC_MCS_ARGS_n_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000540 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_IC_2_UC_MCS_ARGS_n_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000540 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_UC_TLV_IN_VLD_ADDR (IPA_GSI_TOP_GSI_REG_BASE \
-						 + 0x00000558)
-#define HWIO_IPA_GSI_TOP_GSI_UC_TLV_IN_VLD_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000558)
-#define HWIO_IPA_GSI_TOP_GSI_UC_TLV_IN_VLD_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000558)
-#define HWIO_IPA_GSI_TOP_GSI_UC_TLV_IN_ROUTINE_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000055c)
-#define HWIO_IPA_GSI_TOP_GSI_UC_TLV_IN_ROUTINE_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000055c)
-#define HWIO_IPA_GSI_TOP_GSI_UC_TLV_IN_ROUTINE_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000055c)
-#define HWIO_IPA_GSI_TOP_GSI_UC_TLV_IN_ARGS_n_ADDR(n) (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00000560 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_UC_TLV_IN_ARGS_n_PHYS(n) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00000560 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_GSI_UC_TLV_IN_ARGS_n_OFFS(n) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00000560 + 0x4 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_ADDR(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000f000 + 0x4000 * (n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_PHYS(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000f000 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_OFFS(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000f000 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_RMSK 0xfff7ffff
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_MAXk 22
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_INI2(n, k) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_ADDR(n, k), \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_INMI2(n, k, \
-						     mask) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_ADDR(n, \
-							    k),	\
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_OUTI2(n, k, val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_ADDR(n, k), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_OUTMI2(n, k, mask, \
-						      val) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_ADDR(n, k), \
-		mask, \
-		val, \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_INI2(n, k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_ELEMENT_SIZE_BMSK \
-	0xff000000
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_ELEMENT_SIZE_SHFT 0x18
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_CHSTATE_BMSK 0xf00000
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_CHSTATE_SHFT 0x14
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_CHSTATE_NOT_ALLOCATED_FVAL \
-	0x0
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_CHSTATE_ALLOCATED_FVAL 0x1
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_CHSTATE_STARTED_FVAL 0x2
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_CHSTATE_STOPPED_FVAL 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_CHSTATE_STOP_IN_PROC_FVAL \
-	0x4
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_CHSTATE_ERROR_FVAL 0xf
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_ERINDEX_BMSK 0x7c000
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_ERINDEX_SHFT 0xe
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_PROTOCOL_MSB_BMSK	\
-	0x2000
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_PROTOCOL_MSB_SHFT	\
-	0xd
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_CHID_BMSK 0x1f00
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_CHID_SHFT 0x8
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_EE_BMSK 0xf0
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_EE_SHFT 0x4
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_DIR_BMSK 0x8
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_DIR_SHFT 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_DIR_INBOUND_FVAL 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_DIR_OUTBOUND_FVAL	\
-	0x1
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_PROTOCOL_BMSK 0x7
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_PROTOCOL_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_PROTOCOL_MHI_FVAL	\
-	0x0
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_PROTOCOL_XHCI_FVAL \
-	0x1
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_PROTOCOL_GPI_FVAL	\
-	0x2
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_0_CHTYPE_PROTOCOL_XDCI_FVAL \
-	0x3
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_1_ADDR(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000f004 + 0x4000 * (n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_1_PHYS(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000f004 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_1_OFFS(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000f004 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_1_RMSK 0xffff
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_1_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_1_MAXk 22
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_1_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_1_INI2(n, k) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_1_ADDR(n, k), \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_1_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_1_INMI2(n, k, \
-						     mask) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_1_ADDR(n, \
-							    k),	\
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_1_OUTI2(n, k, val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_1_ADDR(n, k), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_1_OUTMI2(n, k, mask, \
-						      val) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_1_ADDR(n, k), \
-		mask, \
-		val, \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_1_INI2(n, k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_1_R_LENGTH_BMSK 0xffff
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_1_R_LENGTH_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_2_ADDR(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000f008 + 0x4000 * (n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_2_PHYS(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000f008 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_2_OFFS(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000f008 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_2_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_2_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_2_MAXk 22
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_2_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_2_INI2(n, k) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_2_ADDR(n, k), \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_2_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_2_INMI2(n, k, \
-						     mask) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_2_ADDR(n, \
-							    k),	\
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_2_OUTI2(n, k, val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_2_ADDR(n, k), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_2_OUTMI2(n, k, mask, \
-						      val) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_2_ADDR(n, k), \
-		mask, \
-		val, \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_2_INI2(n, k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_2_R_BASE_ADDR_LSBS_BMSK \
-	0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_2_R_BASE_ADDR_LSBS_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_3_ADDR(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000f00c + 0x4000 * (n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_3_PHYS(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000f00c + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_3_OFFS(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000f00c + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_3_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_3_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_3_MAXk 22
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_3_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_3_INI2(n, k) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_3_ADDR(n, k), \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_3_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_3_INMI2(n, k, \
-						     mask) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_3_ADDR(n, \
-							    k),	\
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_3_OUTI2(n, k, val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_3_ADDR(n, k), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_3_OUTMI2(n, k, mask, \
-						      val) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_3_ADDR(n, k), \
-		mask, \
-		val, \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_3_INI2(n, k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_3_R_BASE_ADDR_MSBS_BMSK \
-	0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_3_R_BASE_ADDR_MSBS_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_4_ADDR(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000f010 + 0x4000 * (n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_4_PHYS(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000f010 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_4_OFFS(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000f010 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_4_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_4_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_4_MAXk 22
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_4_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_4_INI2(n, k) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_4_ADDR(n, k), \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_4_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_4_INMI2(n, k, \
-						     mask) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_4_ADDR(n, \
-							    k),	\
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_4_OUTI2(n, k, val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_4_ADDR(n, k), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_4_OUTMI2(n, k, mask, \
-						      val) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_4_ADDR(n, k), \
-		mask, \
-		val, \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_4_INI2(n, k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_4_READ_PTR_LSB_BMSK \
-	0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_4_READ_PTR_LSB_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_5_ADDR(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000f014 + 0x4000 * (n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_5_PHYS(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000f014 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_5_OFFS(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000f014 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_5_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_5_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_5_MAXk 22
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_5_ATTR 0x1
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_5_INI2(n, k) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_5_ADDR(n, k), \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_5_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_5_INMI2(n, k, \
-						     mask) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_5_ADDR(n, \
-							    k),	\
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_5_READ_PTR_MSB_BMSK \
-	0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_5_READ_PTR_MSB_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_6_ADDR(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000f018 + 0x4000 * (n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_6_PHYS(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000f018 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_6_OFFS(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000f018 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_6_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_6_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_6_MAXk 22
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_6_ATTR 0x1
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_6_INI2(n, k) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_6_ADDR(n, k), \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_6_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_6_INMI2(n, k, \
-						     mask) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_6_ADDR(n, \
-							    k),	\
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_6_WRITE_PTR_LSB_BMSK \
-	0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_6_WRITE_PTR_LSB_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_7_ADDR(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000f01c + 0x4000 * (n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_7_PHYS(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000f01c + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_7_OFFS(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000f01c + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_7_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_7_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_7_MAXk 22
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_7_ATTR 0x1
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_7_INI2(n, k) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_7_ADDR(n, k), \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_7_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_7_INMI2(n, k, \
-						     mask) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_7_ADDR(n, \
-							    k),	\
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_7_WRITE_PTR_MSB_BMSK \
-	0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_CNTXT_7_WRITE_PTR_MSB_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_ADDR(n, \
-							      k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000f054 + 0x4000 * (n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_PHYS(n, \
-							      k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000f054 + 0x4000 * \
-		(n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_OFFS(n, \
-							      k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000f054 + 0x4000 * \
-		(n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_RMSK 0xffff
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_MAXk 22
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_INI2(n, \
-							      k) \
-	in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_ADDR(n, k), \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_INMI2(n, k, \
-							       mask) \
-	in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_ADDR(n, k), \
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_OUTI2(n, k, \
-							       val) \
-	out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_ADDR(n, k), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_OUTMI2(n, \
-								k, \
-								mask, \
-								val) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_ADDR(n, k), \
-		mask, \
-		val, \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_INI2(n, k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_READ_PTR_BMSK \
-	0xffff
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_READ_PTR_READ_PTR_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_ADDR(n, \
-							       k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000f058 + 0x4000 * (n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_PHYS(n, \
-							       k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000f058 + 0x4000 * \
-		(n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_OFFS(n, \
-							       k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000f058 + 0x4000 * \
-		(n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_RMSK 0xffff
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_MAXk 22
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_INI2(n, \
-							       k) \
-	in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_ADDR(n, \
-								       k), \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_INMI2(n, k, \
-								mask) \
-	in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_ADDR(n, \
-								       k), \
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_OUTI2(n, k, \
-								val) \
-	out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_ADDR(n, \
-								       k), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_OUTMI2(n, \
-								 k, \
-								 mask, \
-								 val) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_ADDR(n, \
-								       k), \
-		mask, \
-		val, \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_INI2(n, \
-								       k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_RE_INTR_DB_BMSK \
-	0xffff
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR_RE_INTR_DB_SHFT \
-	0x0
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_QOS_ADDR(n, \
-						k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000f05c + 0x4000 * (n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_QOS_PHYS(n, \
-						k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000f05c + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_QOS_OFFS(n, \
-						k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000f05c + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_QOS_RMSK 0xff3f0f
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_QOS_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_QOS_MAXk 22
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_QOS_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_QOS_INI2(n, k) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_QOS_ADDR(n, k), \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_QOS_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_QOS_INMI2(n, k, \
-						 mask) in_dword_masked(	\
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_QOS_ADDR(n, \
-							k), \
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_QOS_OUTI2(n, k, val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_QOS_ADDR(n, k), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_QOS_OUTMI2(n, k, mask, \
-						  val) out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_QOS_ADDR(n, \
-							k), \
-		mask, \
-		val, \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_QOS_INI2(n, k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_QOS_EMPTY_LVL_THRSHOLD_BMSK \
-	0xff0000
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_QOS_EMPTY_LVL_THRSHOLD_SHFT 0x10
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_QOS_PREFETCH_MODE_BMSK 0x3c00
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_QOS_PREFETCH_MODE_SHFT 0xa
-#define	\
-	HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_QOS_PREFETCH_MODE_ESCAPE_BUF_ONLY_FVAL \
-	0x1
-#define	\
-	HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_QOS_PREFETCH_MODE_SMART_PRE_FETCH_FVAL \
-	0x2
-#define	\
-	HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_QOS_PREFETCH_MODE_FREE_PRE_FETCH_FVAL \
-	0x3
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_QOS_USE_DB_ENG_BMSK 0x200
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_QOS_USE_DB_ENG_SHFT 0x9
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_QOS_MAX_PREFETCH_BMSK 0x100
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_QOS_MAX_PREFETCH_SHFT 0x8
-#define	\
-	HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_QOS_MAX_PREFETCH_ONE_PREFETCH_SEG_FVAL \
-	0x0
-#define	\
-	HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_QOS_MAX_PREFETCH_TWO_PREFETCH_SEG_FVAL \
-	0x1
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_QOS_WRR_WEIGHT_BMSK 0xf
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_QOS_WRR_WEIGHT_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_0_ADDR(n, \
-						      k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000f060 + 0x4000 * (n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_0_PHYS(n, \
-						      k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000f060 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_0_OFFS(n, \
-						      k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000f060 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_0_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_0_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_0_MAXk 22
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_0_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_0_INI2(n, \
-						      k) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_0_ADDR(n, \
-							      k), \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_0_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_0_INMI2(n, k, \
-						       mask) \
-	in_dword_masked(HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_0_ADDR( \
-				n, \
-				k), mask)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_0_OUTI2(n, k, \
-						       val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_0_ADDR(n, \
-							      k), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_0_OUTMI2(n, k, mask, \
-							val) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_0_ADDR(n, k), \
-		mask, \
-		val, \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_0_INI2(n, k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_0_SCRATCH_BMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_0_SCRATCH_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_1_ADDR(n, \
-						      k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000f064 + 0x4000 * (n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_1_PHYS(n, \
-						      k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000f064 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_1_OFFS(n, \
-						      k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000f064 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_1_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_1_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_1_MAXk 22
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_1_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_1_INI2(n, \
-						      k) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_1_ADDR(n, \
-							      k), \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_1_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_1_INMI2(n, k, \
-						       mask) \
-	in_dword_masked(HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_1_ADDR( \
-				n, \
-				k), mask)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_1_OUTI2(n, k, \
-						       val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_1_ADDR(n, \
-							      k), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_1_OUTMI2(n, k, mask, \
-							val) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_1_ADDR(n, k), \
-		mask, \
-		val, \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_1_INI2(n, k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_1_SCRATCH_BMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_1_SCRATCH_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_2_ADDR(n, \
-						      k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000f068 + 0x4000 * (n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_2_PHYS(n, \
-						      k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000f068 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_2_OFFS(n, \
-						      k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000f068 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_2_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_2_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_2_MAXk 22
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_2_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_2_INI2(n, \
-						      k) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_2_ADDR(n, \
-							      k), \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_2_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_2_INMI2(n, k, \
-						       mask) \
-	in_dword_masked(HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_2_ADDR( \
-				n, \
-				k), mask)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_2_OUTI2(n, k, \
-						       val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_2_ADDR(n, \
-							      k), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_2_OUTMI2(n, k, mask, \
-							val) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_2_ADDR(n, k), \
-		mask, \
-		val, \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_2_INI2(n, k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_2_SCRATCH_BMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_2_SCRATCH_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_3_ADDR(n, \
-						      k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000f06c + 0x4000 * (n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_3_PHYS(n, \
-						      k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000f06c + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_3_OFFS(n, \
-						      k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000f06c + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_3_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_3_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_3_MAXk 22
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_3_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_3_INI2(n, \
-						      k) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_3_ADDR(n, \
-							      k), \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_3_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_3_INMI2(n, k, \
-						       mask) \
-	in_dword_masked(HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_3_ADDR( \
-				n, \
-				k), mask)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_3_OUTI2(n, k, \
-						       val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_3_ADDR(n, \
-							      k), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_3_OUTMI2(n, k, mask, \
-							val) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_3_ADDR(n, k), \
-		mask, \
-		val, \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_3_INI2(n, k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_3_SCRATCH_BMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_SCRATCH_3_SCRATCH_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_DB_ENG_WRITE_PTR_ADDR(n,	\
-							     k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000f070 + 0x4000 * (n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_DB_ENG_WRITE_PTR_PHYS(n,	\
-							     k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000f070 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_DB_ENG_WRITE_PTR_OFFS(n,	\
-							     k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000f070 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_ADDR(n, \
-						   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00010000 + 0x4000 * (n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_PHYS(n, \
-						   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00010000 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_OFFS(n, \
-						   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00010000 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_RMSK 0xfff1ffff
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_MAXk 19
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_INI2(n, k) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_ADDR(n, k), \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_INMI2(n, k, \
-						    mask) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_ADDR(n, \
-							   k), \
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_OUTI2(n, k, val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_ADDR(n, k), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_OUTMI2(n, k, mask, \
-						     val) \
-	out_dword_masked_ns(HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_ADDR(	\
-				    n, \
-				    k), mask, val, \
-			    HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_INI2(n, \
-								       k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_ELEMENT_SIZE_BMSK 0xff000000
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_ELEMENT_SIZE_SHFT 0x18
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_CHSTATE_BMSK 0xf00000
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_CHSTATE_SHFT 0x14
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_CHSTATE_NOT_ALLOCATED_FVAL \
-	0x0
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_CHSTATE_ALLOCATED_FVAL 0x1
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_INTYPE_BMSK 0x10000
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_INTYPE_SHFT 0x10
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_INTYPE_MSI_FVAL 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_INTYPE_IRQ_FVAL 0x1
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_EVCHID_BMSK 0xff00
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_EVCHID_SHFT 0x8
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_EE_BMSK 0xf0
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_EE_SHFT 0x4
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_CHTYPE_BMSK 0xf
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_CHTYPE_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_CHTYPE_MHI_EV_FVAL 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_CHTYPE_XHCI_EV_FVAL 0x1
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_CHTYPE_GPI_EV_FVAL 0x2
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_0_CHTYPE_XDCI_FVAL 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_1_ADDR(n, \
-						   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00010004 + 0x4000 * (n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_1_PHYS(n, \
-						   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00010004 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_1_OFFS(n, \
-						   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00010004 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_1_RMSK 0xffff
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_1_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_1_MAXk 19
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_1_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_1_INI2(n, k) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_1_ADDR(n, k), \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_1_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_1_INMI2(n, k, \
-						    mask) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_1_ADDR(n, \
-							   k), \
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_1_OUTI2(n, k, val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_1_ADDR(n, k), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_1_OUTMI2(n, k, mask, \
-						     val) \
-	out_dword_masked_ns(HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_1_ADDR(	\
-				    n, \
-				    k), mask, val, \
-			    HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_1_INI2(n, \
-								       k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_1_R_LENGTH_BMSK 0xffff
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_1_R_LENGTH_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_2_ADDR(n, \
-						   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00010008 + 0x4000 * (n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_2_PHYS(n, \
-						   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00010008 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_2_OFFS(n, \
-						   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00010008 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_2_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_2_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_2_MAXk 19
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_2_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_2_INI2(n, k) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_2_ADDR(n, k), \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_2_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_2_INMI2(n, k, \
-						    mask) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_2_ADDR(n, \
-							   k), \
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_2_OUTI2(n, k, val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_2_ADDR(n, k), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_2_OUTMI2(n, k, mask, \
-						     val) \
-	out_dword_masked_ns(HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_2_ADDR(	\
-				    n, \
-				    k), mask, val, \
-			    HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_2_INI2(n, \
-								       k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_2_R_BASE_ADDR_LSBS_BMSK \
-	0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_2_R_BASE_ADDR_LSBS_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_3_ADDR(n, \
-						   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0001000c + 0x4000 * (n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_3_PHYS(n, \
-						   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0001000c + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_3_OFFS(n, \
-						   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0001000c + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_3_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_3_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_3_MAXk 19
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_3_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_3_INI2(n, k) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_3_ADDR(n, k), \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_3_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_3_INMI2(n, k, \
-						    mask) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_3_ADDR(n, \
-							   k), \
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_3_OUTI2(n, k, val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_3_ADDR(n, k), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_3_OUTMI2(n, k, mask, \
-						     val) \
-	out_dword_masked_ns(HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_3_ADDR(	\
-				    n, \
-				    k), mask, val, \
-			    HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_3_INI2(n, \
-								       k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_3_R_BASE_ADDR_MSBS_BMSK \
-	0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_3_R_BASE_ADDR_MSBS_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_4_ADDR(n, \
-						   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00010010 + 0x4000 * (n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_4_PHYS(n, \
-						   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00010010 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_4_OFFS(n, \
-						   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00010010 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_4_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_4_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_4_MAXk 19
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_4_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_4_INI2(n, k) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_4_ADDR(n, k), \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_4_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_4_INMI2(n, k, \
-						    mask) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_4_ADDR(n, \
-							   k), \
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_4_OUTI2(n, k, val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_4_ADDR(n, k), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_4_OUTMI2(n, k, mask, \
-						     val) \
-	out_dword_masked_ns(HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_4_ADDR(	\
-				    n, \
-				    k), mask, val, \
-			    HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_4_INI2(n, \
-								       k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_4_READ_PTR_LSB_BMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_4_READ_PTR_LSB_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_5_ADDR(n, \
-						   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00010014 + 0x4000 * (n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_5_PHYS(n, \
-						   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00010014 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_5_OFFS(n, \
-						   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00010014 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_5_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_5_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_5_MAXk 19
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_5_ATTR 0x1
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_5_INI2(n, k) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_5_ADDR(n, k), \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_5_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_5_INMI2(n, k, \
-						    mask) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_5_ADDR(n, \
-							   k), \
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_5_READ_PTR_MSB_BMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_5_READ_PTR_MSB_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_6_ADDR(n, \
-						   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00010018 + 0x4000 * (n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_6_PHYS(n, \
-						   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00010018 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_6_OFFS(n, \
-						   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00010018 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_6_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_6_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_6_MAXk 19
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_6_ATTR 0x1
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_6_INI2(n, k) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_6_ADDR(n, k), \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_6_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_6_INMI2(n, k, \
-						    mask) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_6_ADDR(n, \
-							   k), \
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_6_WRITE_PTR_LSB_BMSK \
-	0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_6_WRITE_PTR_LSB_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_7_ADDR(n, \
-						   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0001001c + 0x4000 * (n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_7_PHYS(n, \
-						   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0001001c + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_7_OFFS(n, \
-						   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0001001c + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_7_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_7_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_7_MAXk 19
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_7_ATTR 0x1
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_7_INI2(n, k) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_7_ADDR(n, k), \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_7_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_7_INMI2(n, k, \
-						    mask) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_7_ADDR(n, \
-							   k), \
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_7_WRITE_PTR_MSB_BMSK \
-	0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_7_WRITE_PTR_MSB_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_8_ADDR(n, \
-						   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00010020 + 0x4000 * (n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_8_PHYS(n, \
-						   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00010020 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_8_OFFS(n, \
-						   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00010020 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_8_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_8_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_8_MAXk 19
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_8_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_8_INI2(n, k) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_8_ADDR(n, k), \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_8_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_8_INMI2(n, k, \
-						    mask) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_8_ADDR(n, \
-							   k), \
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_8_OUTI2(n, k, val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_8_ADDR(n, k), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_8_OUTMI2(n, k, mask, \
-						     val) \
-	out_dword_masked_ns(HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_8_ADDR(	\
-				    n, \
-				    k), mask, val, \
-			    HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_8_INI2(n, \
-								       k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_8_INT_MOD_CNT_BMSK 0xff000000
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_8_INT_MOD_CNT_SHFT 0x18
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_8_INT_MODC_BMSK 0xff0000
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_8_INT_MODC_SHFT 0x10
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_8_INT_MODT_BMSK 0xffff
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_8_INT_MODT_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_9_ADDR(n, \
-						   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00010024 + 0x4000 * (n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_9_PHYS(n, \
-						   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00010024 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_9_OFFS(n, \
-						   k) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00010024 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_9_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_9_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_9_MAXk 19
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_9_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_9_INI2(n, k) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_9_ADDR(n, k), \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_9_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_9_INMI2(n, k, \
-						    mask) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_9_ADDR(n, \
-							   k), \
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_9_OUTI2(n, k, val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_9_ADDR(n, k), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_9_OUTMI2(n, k, mask, \
-						     val) \
-	out_dword_masked_ns(HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_9_ADDR(	\
-				    n, \
-				    k), mask, val, \
-			    HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_9_INI2(n, \
-								       k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_9_INTVEC_BMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_9_INTVEC_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_10_ADDR(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00010028 + 0x4000 * (n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_10_PHYS(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00010028 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_10_OFFS(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00010028 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_10_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_10_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_10_MAXk 19
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_10_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_10_INI2(n, k) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_10_ADDR(n, k), \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_10_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_10_INMI2(n, k, \
-						     mask) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_10_ADDR(n, \
-							    k),	\
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_10_OUTI2(n, k, val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_10_ADDR(n, k), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_10_OUTMI2(n, k, mask, \
-						      val) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_10_ADDR(n, k), \
-		mask, \
-		val, \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_10_INI2(n, k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_10_MSI_ADDR_LSB_BMSK \
-	0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_10_MSI_ADDR_LSB_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_11_ADDR(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0001002c + 0x4000 * (n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_11_PHYS(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0001002c + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_11_OFFS(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0001002c + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_11_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_11_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_11_MAXk 19
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_11_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_11_INI2(n, k) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_11_ADDR(n, k), \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_11_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_11_INMI2(n, k, \
-						     mask) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_11_ADDR(n, \
-							    k),	\
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_11_OUTI2(n, k, val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_11_ADDR(n, k), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_11_OUTMI2(n, k, mask, \
-						      val) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_11_ADDR(n, k), \
-		mask, \
-		val, \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_11_INI2(n, k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_11_MSI_ADDR_MSB_BMSK \
-	0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_11_MSI_ADDR_MSB_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_12_ADDR(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00010030 + 0x4000 * (n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_12_PHYS(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00010030 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_12_OFFS(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00010030 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_12_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_12_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_12_MAXk 19
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_12_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_12_INI2(n, k) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_12_ADDR(n, k), \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_12_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_12_INMI2(n, k, \
-						     mask) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_12_ADDR(n, \
-							    k),	\
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_12_OUTI2(n, k, val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_12_ADDR(n, k), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_12_OUTMI2(n, k, mask, \
-						      val) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_12_ADDR(n, k), \
-		mask, \
-		val, \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_12_INI2(n, k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_12_RP_UPDATE_ADDR_LSB_BMSK \
-	0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_12_RP_UPDATE_ADDR_LSB_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_13_ADDR(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00010034 + 0x4000 * (n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_13_PHYS(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00010034 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_13_OFFS(n, \
-						    k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00010034 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_13_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_13_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_13_MAXk 19
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_13_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_13_INI2(n, k) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_13_ADDR(n, k), \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_13_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_13_INMI2(n, k, \
-						     mask) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_13_ADDR(n, \
-							    k),	\
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_13_OUTI2(n, k, val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_13_ADDR(n, k), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_13_OUTMI2(n, k, mask, \
-						      val) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_13_ADDR(n, k), \
-		mask, \
-		val, \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_13_INI2(n, k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_13_RP_UPDATE_ADDR_MSB_BMSK \
-	0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_CNTXT_13_RP_UPDATE_ADDR_MSB_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_0_ADDR(n,	\
-						     k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00010048 + 0x4000 * (n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_0_PHYS(n,	\
-						     k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00010048 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_0_OFFS(n,	\
-						     k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00010048 + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_0_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_0_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_0_MAXk 19
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_0_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_0_INI2(n, k) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_0_ADDR(n, k), \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_0_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_0_INMI2(n, k, \
-						      mask) \
-	in_dword_masked(HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_0_ADDR( \
-				n, \
-				k), mask)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_0_OUTI2(n, k, val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_0_ADDR(n, k), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_0_OUTMI2(n, k, mask, \
-						       val) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_0_ADDR(n, k), \
-		mask, \
-		val, \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_0_INI2(n, k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_0_SCRATCH_BMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_0_SCRATCH_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_1_ADDR(n,	\
-						     k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0001004c + 0x4000 * (n) + \
-		0x80 * \
-		(k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_1_PHYS(n,	\
-						     k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0001004c + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_1_OFFS(n,	\
-						     k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0001004c + 0x4000 * \
-		(n) + \
-		0x80 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_1_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_1_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_1_MAXk 19
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_1_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_1_INI2(n, k) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_1_ADDR(n, k), \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_1_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_1_INMI2(n, k, \
-						      mask) \
-	in_dword_masked(HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_1_ADDR( \
-				n, \
-				k), mask)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_1_OUTI2(n, k, val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_1_ADDR(n, k), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_1_OUTMI2(n, k, mask, \
-						       val) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_1_ADDR(n, k), \
-		mask, \
-		val, \
-		HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_1_INI2(n, k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_1_SCRATCH_BMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_SCRATCH_1_SCRATCH_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_DOORBELL_0_ADDR(n, \
-						       k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00011000 + 0x4000 * (n) + \
-		0x8 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_DOORBELL_0_PHYS(n, \
-						       k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00011000 + 0x4000 * \
-		(n) + \
-		0x8 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_DOORBELL_0_OFFS(n, \
-						       k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00011000 + 0x4000 * \
-		(n) + \
-		0x8 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_DOORBELL_1_ADDR(n, \
-						       k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00011004 + 0x4000 * (n) + \
-		0x8 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_DOORBELL_1_PHYS(n, \
-						       k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00011004 + 0x4000 * \
-		(n) + \
-		0x8 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_k_DOORBELL_1_OFFS(n, \
-						       k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00011004 + 0x4000 * \
-		(n) + \
-		0x8 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_DOORBELL_0_ADDR(n, \
-						      k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00011100 + 0x4000 * (n) + \
-		0x8 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_DOORBELL_0_PHYS(n, \
-						      k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00011100 + 0x4000 * \
-		(n) + \
-		0x8 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_DOORBELL_0_OFFS(n, \
-						      k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00011100 + 0x4000 * \
-		(n) + \
-		0x8 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_DOORBELL_1_ADDR(n, \
-						      k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00011104 + 0x4000 * (n) + \
-		0x8 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_DOORBELL_1_PHYS(n, \
-						      k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00011104 + 0x4000 * \
-		(n) + \
-		0x8 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_k_DOORBELL_1_OFFS(n, \
-						      k) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00011104 + 0x4000 * \
-		(n) + \
-		0x8 * (k))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_STATUS_ADDR(n) (IPA_GSI_TOP_GSI_REG_BASE \
-						  + 0x00012000 + 0x4000 * \
-						  (n))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_STATUS_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00012000 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_STATUS_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00012000 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_STATUS_RMSK 0x1
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_STATUS_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_STATUS_ATTR 0x1
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_STATUS_INI(n) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_STATUS_ADDR(n), \
-		HWIO_IPA_GSI_TOP_EE_n_GSI_STATUS_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_STATUS_INMI(n, mask) in_dword_masked(	\
-		HWIO_IPA_GSI_TOP_EE_n_GSI_STATUS_ADDR(n), \
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_STATUS_ENABLED_BMSK 0x1
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_STATUS_ENABLED_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_CMD_ADDR(n) (IPA_GSI_TOP_GSI_REG_BASE \
-						  + 0x00012008 + 0x4000 * \
-						  (n))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_CMD_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00012008 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_CH_CMD_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00012008 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_CMD_ADDR(n) (IPA_GSI_TOP_GSI_REG_BASE \
-						 + 0x00012010 + 0x4000 * \
-						 (n))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_CMD_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00012010 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_EV_CH_CMD_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00012010 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_EE_GENERIC_CMD_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00012018 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_EE_GENERIC_CMD_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00012018 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_EE_GENERIC_CMD_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00012018 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_HW_PARAM_0_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00012038 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_HW_PARAM_0_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00012038 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_HW_PARAM_0_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00012038 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_HW_PARAM_1_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0001203c + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_HW_PARAM_1_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0001203c + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_HW_PARAM_1_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0001203c + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_HW_PARAM_2_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00012040 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_HW_PARAM_2_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00012040 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_HW_PARAM_2_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00012040 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_SW_VERSION_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00012044 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_SW_VERSION_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00012044 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_SW_VERSION_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00012044 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_MCS_CODE_VER_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00012048 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_MCS_CODE_VER_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00012048 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_MCS_CODE_VER_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00012048 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_HW_PARAM_3_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0001204c + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_HW_PARAM_3_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0001204c + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_GSI_HW_PARAM_3_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0001204c + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00012080 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00012080 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00012080 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_RMSK 0x7f
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_ATTR 0x1
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_INI(n) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_ADDR(n), \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_ADDR(n), \
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_GENERAL_BMSK 0x40
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_GENERAL_SHFT 0x6
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_INTER_EE_EV_CTRL_BMSK 0x20
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_INTER_EE_EV_CTRL_SHFT 0x5
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_INTER_EE_CH_CTRL_BMSK 0x10
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_INTER_EE_CH_CTRL_SHFT 0x4
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_IEOB_BMSK 0x8
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_IEOB_SHFT 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_GLOB_EE_BMSK 0x4
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_GLOB_EE_SHFT 0x2
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_EV_CTRL_BMSK 0x2
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_EV_CTRL_SHFT 0x1
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_CH_CTRL_BMSK 0x1
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_CH_CTRL_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_MSK_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00012088 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_MSK_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00012088 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_MSK_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00012088 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_MSK_RMSK 0x7f
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_MSK_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_MSK_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_MSK_INI(n) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_MSK_ADDR(n), \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_MSK_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_MSK_INMI(n, \
-						      mask) \
-	in_dword_masked(HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_MSK_ADDR( \
-				n), \
-			mask)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_MSK_OUTI(n, val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_MSK_ADDR(n), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_MSK_OUTMI(n, mask,	\
-						       val) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_MSK_ADDR(n), \
-		mask, \
-		val, \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_MSK_INI(n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_MSK_GENERAL_BMSK 0x40
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_MSK_GENERAL_SHFT 0x6
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_MSK_INTER_EE_EV_CTRL_BMSK \
-	0x20
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_MSK_INTER_EE_EV_CTRL_SHFT 0x5
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_MSK_INTER_EE_CH_CTRL_BMSK \
-	0x10
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_MSK_INTER_EE_CH_CTRL_SHFT 0x4
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_MSK_IEOB_BMSK 0x8
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_MSK_IEOB_SHFT 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_MSK_GLOB_EE_BMSK 0x4
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_MSK_GLOB_EE_SHFT 0x2
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_MSK_EV_CTRL_BMSK 0x2
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_MSK_EV_CTRL_SHFT 0x1
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_MSK_CH_CTRL_BMSK 0x1
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_TYPE_IRQ_MSK_CH_CTRL_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00012090 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00012090 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00012090 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_ATTR 0x1
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_INI(n) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_ADDR(n), \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_INMI(n, \
-							mask) \
-	in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_ADDR(n), \
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_GSI_CH_BIT_MAP_BMSK \
-	0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_GSI_CH_BIT_MAP_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00012094 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00012094 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00012094 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_ATTR 0x1
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_INI(n) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_ADDR(n), \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_INMI(n, \
-						       mask) \
-	in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_ADDR(n), \
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_EV_CH_BIT_MAP_BMSK \
-	0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_EV_CH_BIT_MAP_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00012098 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00012098 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00012098 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_RMSK 0x7fffff
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_INI(n) \
-	in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_ADDR( \
-			n), \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_INMI(n, \
-							    mask) \
-	in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_ADDR(n),	\
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_OUTI(n, \
-							    val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_ADDR( \
-			n), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_OUTMI(n, mask, \
-							     val) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_ADDR(n),	\
-		mask, \
-		val, \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_INI(n))
-#define	\
-	HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_GSI_CH_BIT_MAP_MSK_BMSK \
-	0x7fffff
-#define	\
-	HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK_GSI_CH_BIT_MAP_MSK_SHFT \
-	0x0
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_ADDR(n) (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0001209c + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_PHYS(n) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0001209c + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_OFFS(n) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0001209c + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_RMSK 0xfffff
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_INI(n) \
-	in_dword_masked(HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_ADDR( \
-				n), \
-			HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_INMI(n, \
-							   mask) \
-	in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_ADDR(n), \
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_OUTI(n, \
-							   val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_ADDR( \
-			n), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_OUTMI(n, mask, \
-							    val) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_ADDR(n), \
-		mask, \
-		val, \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_INI(n))
-#define	\
-	HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_EV_CH_BIT_MAP_MSK_BMSK \
-	0xfffff
-#define	\
-	HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_MSK_EV_CH_BIT_MAP_MSK_SHFT \
-	0x0
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_CLR_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x000120a0 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_CLR_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x000120a0 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_CLR_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x000120a0 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_CLR_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_CLR_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_CLR_ATTR 0x2
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_CLR_OUTI(n, \
-							    val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_CLR_ADDR( \
-			n), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_CLR_GSI_CH_BIT_MAP_BMSK \
-	0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_GSI_CH_IRQ_CLR_GSI_CH_BIT_MAP_SHFT \
-	0x0
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_CLR_ADDR(n) (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x000120a4 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_CLR_PHYS(n) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x000120a4 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_CLR_OFFS(n) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x000120a4 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_CLR_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_CLR_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_CLR_ATTR 0x2
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_CLR_OUTI(n, \
-							   val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_CLR_ADDR( \
-			n), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_CLR_EV_CH_BIT_MAP_BMSK \
-	0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_EV_CH_IRQ_CLR_EV_CH_BIT_MAP_SHFT \
-	0x0
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x000120b0 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x000120b0 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x000120b0 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_ATTR 0x1
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_INI(n) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_ADDR(n), \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_INMI(n, \
-						      mask) \
-	in_dword_masked(HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_ADDR( \
-				n), \
-			mask)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_EV_CH_BIT_MAP_BMSK \
-	0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_EV_CH_BIT_MAP_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x000120b8 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x000120b8 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x000120b8 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_RMSK 0xfffff
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_INI(n) \
-	in_dword_masked(HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_ADDR( \
-				n), \
-			HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_INMI(n, \
-							  mask)	\
-	in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_ADDR(n), \
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_OUTI(n, \
-							  val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_ADDR( \
-			n), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_OUTMI(n, mask, \
-							   val)	\
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_ADDR(n), \
-		mask, \
-		val, \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_INI(n))
-#define	\
-	HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_EV_CH_BIT_MAP_MSK_BMSK \
-	0xfffff
-#define	\
-	HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_MSK_EV_CH_BIT_MAP_MSK_SHFT \
-	0x0
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_CLR_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x000120c0 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_CLR_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x000120c0 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_CLR_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x000120c0 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_CLR_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_CLR_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_CLR_ATTR 0x2
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_CLR_OUTI(n, \
-							  val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_CLR_ADDR( \
-			n), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_CLR_EV_CH_BIT_MAP_BMSK	\
-	0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SRC_IEOB_IRQ_CLR_EV_CH_BIT_MAP_SHFT	\
-	0x0
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GLOB_IRQ_STTS_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00012100 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GLOB_IRQ_STTS_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00012100 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GLOB_IRQ_STTS_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00012100 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GLOB_IRQ_STTS_RMSK 0xf
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GLOB_IRQ_STTS_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GLOB_IRQ_STTS_ATTR 0x1
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GLOB_IRQ_STTS_INI(n) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_GLOB_IRQ_STTS_ADDR(n), \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_GLOB_IRQ_STTS_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GLOB_IRQ_STTS_INMI(n, \
-						       mask) \
-	in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_GLOB_IRQ_STTS_ADDR(n), \
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GLOB_IRQ_STTS_GP_INT3_BMSK 0x8
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GLOB_IRQ_STTS_GP_INT3_SHFT 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GLOB_IRQ_STTS_GP_INT2_BMSK 0x4
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GLOB_IRQ_STTS_GP_INT2_SHFT 0x2
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GLOB_IRQ_STTS_GP_INT1_BMSK 0x2
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GLOB_IRQ_STTS_GP_INT1_SHFT 0x1
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GLOB_IRQ_STTS_ERROR_INT_BMSK 0x1
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GLOB_IRQ_STTS_ERROR_INT_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GLOB_IRQ_EN_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00012108 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GLOB_IRQ_EN_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00012108 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GLOB_IRQ_EN_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00012108 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GLOB_IRQ_CLR_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00012110 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GLOB_IRQ_CLR_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00012110 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GLOB_IRQ_CLR_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00012110 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GSI_IRQ_STTS_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00012118 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GSI_IRQ_STTS_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00012118 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GSI_IRQ_STTS_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00012118 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GSI_IRQ_STTS_RMSK 0xf
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GSI_IRQ_STTS_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GSI_IRQ_STTS_ATTR 0x1
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GSI_IRQ_STTS_INI(n) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_GSI_IRQ_STTS_ADDR(n), \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_GSI_IRQ_STTS_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GSI_IRQ_STTS_INMI(n, \
-						      mask) \
-	in_dword_masked(HWIO_IPA_GSI_TOP_EE_n_CNTXT_GSI_IRQ_STTS_ADDR( \
-				n), \
-			mask)
-#define	\
-	HWIO_IPA_GSI_TOP_EE_n_CNTXT_GSI_IRQ_STTS_GSI_MCS_STACK_OVRFLOW_BMSK \
-	0x8
-#define	\
-	HWIO_IPA_GSI_TOP_EE_n_CNTXT_GSI_IRQ_STTS_GSI_MCS_STACK_OVRFLOW_SHFT \
-	0x3
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GSI_IRQ_STTS_GSI_CMD_FIFO_OVRFLOW_BMSK \
-	0x4
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GSI_IRQ_STTS_GSI_CMD_FIFO_OVRFLOW_SHFT \
-	0x2
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GSI_IRQ_STTS_GSI_BUS_ERROR_BMSK 0x2
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GSI_IRQ_STTS_GSI_BUS_ERROR_SHFT 0x1
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GSI_IRQ_STTS_GSI_BREAK_POINT_BMSK 0x1
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GSI_IRQ_STTS_GSI_BREAK_POINT_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GSI_IRQ_EN_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00012120 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GSI_IRQ_EN_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00012120 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GSI_IRQ_EN_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00012120 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GSI_IRQ_CLR_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00012128 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GSI_IRQ_CLR_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00012128 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_GSI_IRQ_CLR_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00012128 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_INTSET_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00012180 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_INTSET_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00012180 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_INTSET_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00012180 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_INTSET_RMSK 0x1
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_INTSET_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_INTSET_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_INTSET_INI(n) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_INTSET_ADDR(n), \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_INTSET_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_INTSET_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_INTSET_ADDR(n), \
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_INTSET_OUTI(n, val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_INTSET_ADDR(n), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_INTSET_OUTMI(n, mask, \
-						 val) out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_INTSET_ADDR( \
-			n), \
-		mask, \
-		val, \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_INTSET_INI(n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_INTSET_INTYPE_BMSK 0x1
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_INTSET_INTYPE_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_INTSET_INTYPE_MSI_FVAL 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_INTSET_INTYPE_IRQ_FVAL 0x1
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_LSB_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00012188 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_LSB_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00012188 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_LSB_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00012188 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_LSB_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_LSB_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_LSB_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_LSB_INI(n) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_LSB_ADDR(n), \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_LSB_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_LSB_INMI(n, \
-						      mask) \
-	in_dword_masked(HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_LSB_ADDR( \
-				n), \
-			mask)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_LSB_OUTI(n, val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_LSB_ADDR(n), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_LSB_OUTMI(n, mask,	\
-						       val) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_LSB_ADDR(n), \
-		mask, \
-		val, \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_LSB_INI(n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_LSB_MSI_ADDR_LSB_BMSK \
-	0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_LSB_MSI_ADDR_LSB_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_MSB_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0001218c + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_MSB_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0001218c + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_MSB_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0001218c + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_MSB_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_MSB_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_MSB_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_MSB_INI(n) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_MSB_ADDR(n), \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_MSB_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_MSB_INMI(n, \
-						      mask) \
-	in_dword_masked(HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_MSB_ADDR( \
-				n), \
-			mask)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_MSB_OUTI(n, val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_MSB_ADDR(n), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_MSB_OUTMI(n, mask,	\
-						       val) \
-	out_dword_masked_ns( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_MSB_ADDR(n), \
-		mask, \
-		val, \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_MSB_INI(n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_MSB_MSI_ADDR_MSB_BMSK \
-	0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_MSI_BASE_MSB_MSI_ADDR_MSB_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_INT_VEC_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00012190 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_INT_VEC_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00012190 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_INT_VEC_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00012190 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_ERROR_LOG_ADDR(n) (IPA_GSI_TOP_GSI_REG_BASE \
-						 + 0x00012200 + 0x4000 * \
-						 (n))
-#define HWIO_IPA_GSI_TOP_EE_n_ERROR_LOG_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00012200 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_ERROR_LOG_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00012200 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_ERROR_LOG_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_ERROR_LOG_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_ERROR_LOG_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_ERROR_LOG_INI(n) in_dword_masked(	\
-		HWIO_IPA_GSI_TOP_EE_n_ERROR_LOG_ADDR(n), \
-		HWIO_IPA_GSI_TOP_EE_n_ERROR_LOG_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_ERROR_LOG_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_ERROR_LOG_ADDR(n), \
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_ERROR_LOG_OUTI(n, val) out_dword(	\
-		HWIO_IPA_GSI_TOP_EE_n_ERROR_LOG_ADDR(n), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_ERROR_LOG_OUTMI(n, mask, \
-					      val) out_dword_masked_ns(	\
-		HWIO_IPA_GSI_TOP_EE_n_ERROR_LOG_ADDR( \
-			n), \
-		mask, \
-		val, \
-		HWIO_IPA_GSI_TOP_EE_n_ERROR_LOG_INI(n))
-#define HWIO_IPA_GSI_TOP_EE_n_ERROR_LOG_ERROR_LOG_BMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_ERROR_LOG_ERROR_LOG_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_ERROR_LOG_CLR_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00012210 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_ERROR_LOG_CLR_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00012210 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_ERROR_LOG_CLR_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00012210 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_ERROR_LOG_CLR_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_ERROR_LOG_CLR_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_ERROR_LOG_CLR_ATTR 0x2
-#define HWIO_IPA_GSI_TOP_EE_n_ERROR_LOG_CLR_OUTI(n, val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_ERROR_LOG_CLR_ADDR(n), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_ERROR_LOG_CLR_ERROR_LOG_CLR_BMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_ERROR_LOG_CLR_ERROR_LOG_CLR_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_0_ADDR(n) (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00012400 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_0_PHYS(n) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00012400 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_0_OFFS(n) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00012400 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_0_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_0_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_0_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_0_INI(n) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_0_ADDR(n), \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_0_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_0_INMI(n, \
-						   mask) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_0_ADDR( \
-			n), \
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_0_OUTI(n, val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_0_ADDR(n), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_0_OUTMI(n, mask, \
-						    val) \
-	out_dword_masked_ns(HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_0_ADDR(	\
-				    n),	\
-			    mask, val, \
-			    HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_0_INI(n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_0_SCRATCH_BMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_0_SCRATCH_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_1_ADDR(n) (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x00012404 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_1_PHYS(n) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x00012404 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_1_OFFS(n) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x00012404 + 0x4000 * (n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_1_RMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_1_MAXn 2
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_1_ATTR 0x3
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_1_INI(n) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_1_ADDR(n), \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_1_RMSK)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_1_INMI(n, \
-						   mask) in_dword_masked( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_1_ADDR( \
-			n), \
-		mask)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_1_OUTI(n, val) out_dword( \
-		HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_1_ADDR(n), \
-		val)
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_1_OUTMI(n, mask, \
-						    val) \
-	out_dword_masked_ns(HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_1_ADDR(	\
-				    n),	\
-			    mask, val, \
-			    HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_1_INI(n))
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_1_SCRATCH_BMSK 0xffffffff
-#define HWIO_IPA_GSI_TOP_EE_n_CNTXT_SCRATCH_1_SCRATCH_SHFT 0x0
-#define HWIO_IPA_GSI_TOP_GSI_MCS_CFG_ADDR (IPA_GSI_TOP_GSI_REG_BASE + \
-					   0x0000b000)
-#define HWIO_IPA_GSI_TOP_GSI_MCS_CFG_PHYS (IPA_GSI_TOP_GSI_REG_BASE_PHYS + \
-					   0x0000b000)
-#define HWIO_IPA_GSI_TOP_GSI_MCS_CFG_OFFS (IPA_GSI_TOP_GSI_REG_BASE_OFFS + \
-					   0x0000b000)
-#define HWIO_IPA_GSI_TOP_GSI_TZ_FW_AUTH_LOCK_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000b008)
-#define HWIO_IPA_GSI_TOP_GSI_TZ_FW_AUTH_LOCK_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000b008)
-#define HWIO_IPA_GSI_TOP_GSI_TZ_FW_AUTH_LOCK_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000b008)
-#define HWIO_IPA_GSI_TOP_GSI_MSA_FW_AUTH_LOCK_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000b010)
-#define HWIO_IPA_GSI_TOP_GSI_MSA_FW_AUTH_LOCK_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000b010)
-#define HWIO_IPA_GSI_TOP_GSI_MSA_FW_AUTH_LOCK_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000b010)
-#define HWIO_IPA_GSI_TOP_GSI_SP_FW_AUTH_LOCK_ADDR ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000b018)
-#define HWIO_IPA_GSI_TOP_GSI_SP_FW_AUTH_LOCK_PHYS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000b018)
-#define HWIO_IPA_GSI_TOP_GSI_SP_FW_AUTH_LOCK_OFFS ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000b018)
-#define HWIO_IPA_GSI_TOP_INTER_EE_n_ORIGINATOR_EE_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000c000 + 0x1000 * (n))
-#define HWIO_IPA_GSI_TOP_INTER_EE_n_ORIGINATOR_EE_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000c000 + 0x1000 * (n))
-#define HWIO_IPA_GSI_TOP_INTER_EE_n_ORIGINATOR_EE_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000c000 + 0x1000 * (n))
-#define HWIO_IPA_GSI_TOP_INTER_EE_n_GSI_CH_CMD_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000c008 + 0x1000 * (n))
-#define HWIO_IPA_GSI_TOP_INTER_EE_n_GSI_CH_CMD_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000c008 + 0x1000 * (n))
-#define HWIO_IPA_GSI_TOP_INTER_EE_n_GSI_CH_CMD_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000c008 + 0x1000 * (n))
-#define HWIO_IPA_GSI_TOP_INTER_EE_n_EV_CH_CMD_ADDR(n) (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000c010 + 0x1000 * (n))
-#define HWIO_IPA_GSI_TOP_INTER_EE_n_EV_CH_CMD_PHYS(n) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000c010 + 0x1000 * (n))
-#define HWIO_IPA_GSI_TOP_INTER_EE_n_EV_CH_CMD_OFFS(n) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000c010 + 0x1000 * (n))
-#define HWIO_IPA_GSI_TOP_INTER_EE_n_SRC_GSI_CH_IRQ_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000c018 + 0x1000 * (n))
-#define HWIO_IPA_GSI_TOP_INTER_EE_n_SRC_GSI_CH_IRQ_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000c018 + 0x1000 * (n))
-#define HWIO_IPA_GSI_TOP_INTER_EE_n_SRC_GSI_CH_IRQ_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000c018 + 0x1000 * (n))
-#define HWIO_IPA_GSI_TOP_INTER_EE_n_SRC_EV_CH_IRQ_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000c01c + 0x1000 * (n))
-#define HWIO_IPA_GSI_TOP_INTER_EE_n_SRC_EV_CH_IRQ_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000c01c + 0x1000 * (n))
-#define HWIO_IPA_GSI_TOP_INTER_EE_n_SRC_EV_CH_IRQ_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000c01c + 0x1000 * (n))
-#define HWIO_IPA_GSI_TOP_INTER_EE_n_SRC_GSI_CH_IRQ_MSK_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000c020 + 0x1000 * (n))
-#define HWIO_IPA_GSI_TOP_INTER_EE_n_SRC_GSI_CH_IRQ_MSK_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000c020 + 0x1000 * (n))
-#define HWIO_IPA_GSI_TOP_INTER_EE_n_SRC_GSI_CH_IRQ_MSK_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000c020 + 0x1000 * (n))
-#define HWIO_IPA_GSI_TOP_INTER_EE_n_SRC_EV_CH_IRQ_MSK_ADDR(n) (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000c024 + 0x1000 * (n))
-#define HWIO_IPA_GSI_TOP_INTER_EE_n_SRC_EV_CH_IRQ_MSK_PHYS(n) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000c024 + 0x1000 * (n))
-#define HWIO_IPA_GSI_TOP_INTER_EE_n_SRC_EV_CH_IRQ_MSK_OFFS(n) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000c024 + 0x1000 * (n))
-#define HWIO_IPA_GSI_TOP_INTER_EE_n_SRC_GSI_CH_IRQ_CLR_ADDR(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000c028 + 0x1000 * (n))
-#define HWIO_IPA_GSI_TOP_INTER_EE_n_SRC_GSI_CH_IRQ_CLR_PHYS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000c028 + 0x1000 * (n))
-#define HWIO_IPA_GSI_TOP_INTER_EE_n_SRC_GSI_CH_IRQ_CLR_OFFS(n) ( \
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000c028 + 0x1000 * (n))
-#define HWIO_IPA_GSI_TOP_INTER_EE_n_SRC_EV_CH_IRQ_CLR_ADDR(n) (	\
-		IPA_GSI_TOP_GSI_REG_BASE + 0x0000c02c + 0x1000 * (n))
-#define HWIO_IPA_GSI_TOP_INTER_EE_n_SRC_EV_CH_IRQ_CLR_PHYS(n) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_PHYS + 0x0000c02c + 0x1000 * (n))
-#define HWIO_IPA_GSI_TOP_INTER_EE_n_SRC_EV_CH_IRQ_CLR_OFFS(n) (	\
-		IPA_GSI_TOP_GSI_REG_BASE_OFFS + 0x0000c02c + 0x1000 * (n))
-#define IPA_CFG_REG_BASE (IPA_0_IPA_WRAPPER_BASE + 0x00040000)
-#define IPA_CFG_REG_BASE_PHYS (IPA_0_IPA_WRAPPER_BASE_PHYS + 0x00040000)
-#define IPA_CFG_REG_BASE_OFFS 0x00040000
-#define HWIO_IPA_COMP_HW_VERSION_ADDR (IPA_CFG_REG_BASE + 0x00000030)
-#define HWIO_IPA_COMP_HW_VERSION_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00000030)
-#define HWIO_IPA_COMP_HW_VERSION_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00000030)
-#define HWIO_IPA_COMP_HW_VERSION_RMSK 0xffffffff
-#define HWIO_IPA_COMP_HW_VERSION_ATTR 0x1
-#define HWIO_IPA_COMP_HW_VERSION_IN in_dword_masked( \
-		HWIO_IPA_COMP_HW_VERSION_ADDR, \
-		HWIO_IPA_COMP_HW_VERSION_RMSK)
-#define HWIO_IPA_COMP_HW_VERSION_INM(m) in_dword_masked( \
-		HWIO_IPA_COMP_HW_VERSION_ADDR, \
-		m)
-#define HWIO_IPA_COMP_HW_VERSION_MAJOR_BMSK 0xf0000000
-#define HWIO_IPA_COMP_HW_VERSION_MAJOR_SHFT 0x1c
-#define HWIO_IPA_COMP_HW_VERSION_MINOR_BMSK 0xfff0000
-#define HWIO_IPA_COMP_HW_VERSION_MINOR_SHFT 0x10
-#define HWIO_IPA_COMP_HW_VERSION_STEP_BMSK 0xffff
-#define HWIO_IPA_COMP_HW_VERSION_STEP_SHFT 0x0
-#define HWIO_IPA_VERSION_ADDR (IPA_CFG_REG_BASE + 0x00000034)
-#define HWIO_IPA_VERSION_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00000034)
-#define HWIO_IPA_VERSION_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00000034)
-#define HWIO_IPA_ENABLED_PIPES_ADDR (IPA_CFG_REG_BASE + 0x00000038)
-#define HWIO_IPA_ENABLED_PIPES_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00000038)
-#define HWIO_IPA_ENABLED_PIPES_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00000038)
-#define HWIO_IPA_COMP_CFG_ADDR (IPA_CFG_REG_BASE + 0x0000003c)
-#define HWIO_IPA_COMP_CFG_PHYS (IPA_CFG_REG_BASE_PHYS + 0x0000003c)
-#define HWIO_IPA_COMP_CFG_OFFS (IPA_CFG_REG_BASE_OFFS + 0x0000003c)
-#define HWIO_IPA_COMP_CFG_RMSK 0x3fffee
-#define HWIO_IPA_COMP_CFG_ATTR 0x3
-#define HWIO_IPA_COMP_CFG_IN in_dword_masked(HWIO_IPA_COMP_CFG_ADDR, \
-					     HWIO_IPA_COMP_CFG_RMSK)
-#define HWIO_IPA_COMP_CFG_INM(m) in_dword_masked(HWIO_IPA_COMP_CFG_ADDR, m)
-#define HWIO_IPA_COMP_CFG_OUT(v) out_dword(HWIO_IPA_COMP_CFG_ADDR, v)
-#define HWIO_IPA_COMP_CFG_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_COMP_CFG_ADDR,	\
-		m, \
-		v, \
-		HWIO_IPA_COMP_CFG_IN)
-#define HWIO_IPA_COMP_CFG_IPA_FULL_FLUSH_WAIT_RSC_CLOSURE_EN_BMSK 0x200000
-#define HWIO_IPA_COMP_CFG_IPA_FULL_FLUSH_WAIT_RSC_CLOSURE_EN_SHFT 0x15
-#define HWIO_IPA_COMP_CFG_IPA_ATOMIC_FETCHER_ARB_LOCK_DIS_BMSK 0x1e0000
-#define HWIO_IPA_COMP_CFG_IPA_ATOMIC_FETCHER_ARB_LOCK_DIS_SHFT 0x11
-#define HWIO_IPA_COMP_CFG_IPA_QMB_SELECT_BY_ADDRESS_GLOBAL_EN_BMSK 0x10000
-#define HWIO_IPA_COMP_CFG_IPA_QMB_SELECT_BY_ADDRESS_GLOBAL_EN_SHFT 0x10
-#define HWIO_IPA_COMP_CFG_GSI_MULTI_AXI_MASTERS_DIS_BMSK 0x8000
-#define HWIO_IPA_COMP_CFG_GSI_MULTI_AXI_MASTERS_DIS_SHFT 0xf
-#define HWIO_IPA_COMP_CFG_GSI_SNOC_CNOC_LOOP_PROTECTION_DISABLE_BMSK \
-	0x4000
-#define HWIO_IPA_COMP_CFG_GSI_SNOC_CNOC_LOOP_PROTECTION_DISABLE_SHFT 0xe
-#define HWIO_IPA_COMP_CFG_GEN_QMB_0_SNOC_CNOC_LOOP_PROTECTION_DISABLE_BMSK \
-	0x2000
-#define HWIO_IPA_COMP_CFG_GEN_QMB_0_SNOC_CNOC_LOOP_PROTECTION_DISABLE_SHFT \
-	0xd
-#define HWIO_IPA_COMP_CFG_GEN_QMB_1_MULTI_INORDER_WR_DIS_BMSK 0x1000
-#define HWIO_IPA_COMP_CFG_GEN_QMB_1_MULTI_INORDER_WR_DIS_SHFT 0xc
-#define HWIO_IPA_COMP_CFG_GEN_QMB_0_MULTI_INORDER_WR_DIS_BMSK 0x800
-#define HWIO_IPA_COMP_CFG_GEN_QMB_0_MULTI_INORDER_WR_DIS_SHFT 0xb
-#define HWIO_IPA_COMP_CFG_GEN_QMB_1_MULTI_INORDER_RD_DIS_BMSK 0x400
-#define HWIO_IPA_COMP_CFG_GEN_QMB_1_MULTI_INORDER_RD_DIS_SHFT 0xa
-#define HWIO_IPA_COMP_CFG_GEN_QMB_0_MULTI_INORDER_RD_DIS_BMSK 0x200
-#define HWIO_IPA_COMP_CFG_GEN_QMB_0_MULTI_INORDER_RD_DIS_SHFT 0x9
-#define HWIO_IPA_COMP_CFG_GSI_MULTI_INORDER_WR_DIS_BMSK 0x100
-#define HWIO_IPA_COMP_CFG_GSI_MULTI_INORDER_WR_DIS_SHFT 0x8
-#define HWIO_IPA_COMP_CFG_GSI_MULTI_INORDER_RD_DIS_BMSK 0x80
-#define HWIO_IPA_COMP_CFG_GSI_MULTI_INORDER_RD_DIS_SHFT 0x7
-#define HWIO_IPA_COMP_CFG_IPA_QMB_SELECT_BY_ADDRESS_PROD_EN_BMSK 0x40
-#define HWIO_IPA_COMP_CFG_IPA_QMB_SELECT_BY_ADDRESS_PROD_EN_SHFT 0x6
-#define HWIO_IPA_COMP_CFG_IPA_QMB_SELECT_BY_ADDRESS_CONS_EN_BMSK 0x20
-#define HWIO_IPA_COMP_CFG_IPA_QMB_SELECT_BY_ADDRESS_CONS_EN_SHFT 0x5
-#define HWIO_IPA_COMP_CFG_GEN_QMB_1_SNOC_BYPASS_DIS_BMSK 0x8
-#define HWIO_IPA_COMP_CFG_GEN_QMB_1_SNOC_BYPASS_DIS_SHFT 0x3
-#define HWIO_IPA_COMP_CFG_GEN_QMB_0_SNOC_BYPASS_DIS_BMSK 0x4
-#define HWIO_IPA_COMP_CFG_GEN_QMB_0_SNOC_BYPASS_DIS_SHFT 0x2
-#define HWIO_IPA_COMP_CFG_GSI_SNOC_BYPASS_DIS_BMSK 0x2
-#define HWIO_IPA_COMP_CFG_GSI_SNOC_BYPASS_DIS_SHFT 0x1
-#define HWIO_IPA_CLKON_CFG_ADDR (IPA_CFG_REG_BASE + 0x00000044)
-#define HWIO_IPA_CLKON_CFG_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00000044)
-#define HWIO_IPA_CLKON_CFG_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00000044)
-#define HWIO_IPA_ROUTE_ADDR (IPA_CFG_REG_BASE + 0x00000048)
-#define HWIO_IPA_ROUTE_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00000048)
-#define HWIO_IPA_ROUTE_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00000048)
-#define HWIO_IPA_ROUTE_RMSK 0x13fffff
-#define HWIO_IPA_ROUTE_ATTR 0x3
-#define HWIO_IPA_ROUTE_IN in_dword_masked(HWIO_IPA_ROUTE_ADDR, \
-					  HWIO_IPA_ROUTE_RMSK)
-#define HWIO_IPA_ROUTE_INM(m) in_dword_masked(HWIO_IPA_ROUTE_ADDR, m)
-#define HWIO_IPA_ROUTE_OUT(v) out_dword(HWIO_IPA_ROUTE_ADDR, v)
-#define HWIO_IPA_ROUTE_OUTM(m, v) out_dword_masked_ns(HWIO_IPA_ROUTE_ADDR, \
-						      m, \
-						      v, \
-						      HWIO_IPA_ROUTE_IN)
-#define HWIO_IPA_ROUTE_ROUTE_DEF_RETAIN_HDR_BMSK 0x1000000
-#define HWIO_IPA_ROUTE_ROUTE_DEF_RETAIN_HDR_SHFT 0x18
-#define HWIO_IPA_ROUTE_ROUTE_FRAG_DEF_PIPE_BMSK 0x3e0000
-#define HWIO_IPA_ROUTE_ROUTE_FRAG_DEF_PIPE_SHFT 0x11
-#define HWIO_IPA_ROUTE_ROUTE_DEF_HDR_OFST_BMSK 0x1ff80
-#define HWIO_IPA_ROUTE_ROUTE_DEF_HDR_OFST_SHFT 0x7
-#define HWIO_IPA_ROUTE_ROUTE_DEF_HDR_TABLE_BMSK 0x40
-#define HWIO_IPA_ROUTE_ROUTE_DEF_HDR_TABLE_SHFT 0x6
-#define HWIO_IPA_ROUTE_ROUTE_DEF_PIPE_BMSK 0x3e
-#define HWIO_IPA_ROUTE_ROUTE_DEF_PIPE_SHFT 0x1
-#define HWIO_IPA_ROUTE_ROUTE_DIS_BMSK 0x1
-#define HWIO_IPA_ROUTE_ROUTE_DIS_SHFT 0x0
-#define HWIO_IPA_FILTER_ADDR (IPA_CFG_REG_BASE + 0x0000004c)
-#define HWIO_IPA_FILTER_PHYS (IPA_CFG_REG_BASE_PHYS + 0x0000004c)
-#define HWIO_IPA_FILTER_OFFS (IPA_CFG_REG_BASE_OFFS + 0x0000004c)
-#define HWIO_IPA_MASTER_PRIORITY_ADDR (IPA_CFG_REG_BASE + 0x00000050)
-#define HWIO_IPA_MASTER_PRIORITY_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00000050)
-#define HWIO_IPA_MASTER_PRIORITY_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00000050)
-#define HWIO_IPA_SHARED_MEM_SIZE_ADDR (IPA_CFG_REG_BASE + 0x00000054)
-#define HWIO_IPA_SHARED_MEM_SIZE_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00000054)
-#define HWIO_IPA_SHARED_MEM_SIZE_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00000054)
-#define HWIO_IPA_NAT_TIMER_ADDR (IPA_CFG_REG_BASE + 0x00000058)
-#define HWIO_IPA_NAT_TIMER_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00000058)
-#define HWIO_IPA_NAT_TIMER_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00000058)
-#define HWIO_IPA_TAG_TIMER_ADDR (IPA_CFG_REG_BASE + 0x00000060)
-#define HWIO_IPA_TAG_TIMER_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00000060)
-#define HWIO_IPA_TAG_TIMER_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00000060)
-#define HWIO_IPA_FRAG_RULES_CLR_ADDR (IPA_CFG_REG_BASE + 0x0000006c)
-#define HWIO_IPA_FRAG_RULES_CLR_PHYS (IPA_CFG_REG_BASE_PHYS + 0x0000006c)
-#define HWIO_IPA_FRAG_RULES_CLR_OFFS (IPA_CFG_REG_BASE_OFFS + 0x0000006c)
-#define HWIO_IPA_PROC_IPH_CFG_ADDR (IPA_CFG_REG_BASE + 0x00000070)
-#define HWIO_IPA_PROC_IPH_CFG_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00000070)
-#define HWIO_IPA_PROC_IPH_CFG_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00000070)
-#define HWIO_IPA_PROC_IPH_CFG_RMSK 0x1ff0ff7
-#define HWIO_IPA_PROC_IPH_CFG_ATTR 0x3
-#define HWIO_IPA_PROC_IPH_CFG_IN in_dword_masked( \
-		HWIO_IPA_PROC_IPH_CFG_ADDR, \
-		HWIO_IPA_PROC_IPH_CFG_RMSK)
-#define HWIO_IPA_PROC_IPH_CFG_INM(m) in_dword_masked( \
-		HWIO_IPA_PROC_IPH_CFG_ADDR, \
-		m)
-#define HWIO_IPA_PROC_IPH_CFG_OUT(v) out_dword(HWIO_IPA_PROC_IPH_CFG_ADDR, \
-					       v)
-#define HWIO_IPA_PROC_IPH_CFG_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_PROC_IPH_CFG_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_PROC_IPH_CFG_IN)
-#define HWIO_IPA_PROC_IPH_CFG_D_DCPH_MULTI_ENGINE_DISABLE_BMSK 0x1000000
-#define HWIO_IPA_PROC_IPH_CFG_D_DCPH_MULTI_ENGINE_DISABLE_SHFT 0x18
-#define HWIO_IPA_PROC_IPH_CFG_IPH_PKT_PARSER_PROTOCOL_STOP_VALUE_BMSK \
-	0xff0000
-#define HWIO_IPA_PROC_IPH_CFG_IPH_PKT_PARSER_PROTOCOL_STOP_VALUE_SHFT 0x10
-#define HWIO_IPA_PROC_IPH_CFG_IPH_PKT_PARSER_IHL_TO_2ND_FRAG_EN_BMSK 0x800
-#define HWIO_IPA_PROC_IPH_CFG_IPH_PKT_PARSER_IHL_TO_2ND_FRAG_EN_SHFT 0xb
-#define HWIO_IPA_PROC_IPH_CFG_IPH_PKT_PARSER_PROTOCOL_STOP_DEST_BMSK 0x400
-#define HWIO_IPA_PROC_IPH_CFG_IPH_PKT_PARSER_PROTOCOL_STOP_DEST_SHFT 0xa
-#define HWIO_IPA_PROC_IPH_CFG_IPH_PKT_PARSER_PROTOCOL_STOP_HOP_BMSK 0x200
-#define HWIO_IPA_PROC_IPH_CFG_IPH_PKT_PARSER_PROTOCOL_STOP_HOP_SHFT 0x9
-#define HWIO_IPA_PROC_IPH_CFG_IPH_PKT_PARSER_PROTOCOL_STOP_ENABLE_BMSK \
-	0x100
-#define HWIO_IPA_PROC_IPH_CFG_IPH_PKT_PARSER_PROTOCOL_STOP_ENABLE_SHFT 0x8
-#define HWIO_IPA_PROC_IPH_CFG_FTCH_DCPH_OVERLAP_ENABLE_BMSK 0x80
-#define HWIO_IPA_PROC_IPH_CFG_FTCH_DCPH_OVERLAP_ENABLE_SHFT 0x7
-#define HWIO_IPA_PROC_IPH_CFG_PIPESTAGE_OVERLAP_DISABLE_BMSK 0x40
-#define HWIO_IPA_PROC_IPH_CFG_PIPESTAGE_OVERLAP_DISABLE_SHFT 0x6
-#define HWIO_IPA_PROC_IPH_CFG_STATUS_FROM_IPH_FRST_ALWAYS_BMSK 0x10
-#define HWIO_IPA_PROC_IPH_CFG_STATUS_FROM_IPH_FRST_ALWAYS_SHFT 0x4
-#define HWIO_IPA_PROC_IPH_CFG_IPH_PIPELINING_DISABLE_BMSK 0x4
-#define HWIO_IPA_PROC_IPH_CFG_IPH_PIPELINING_DISABLE_SHFT 0x2
-#define HWIO_IPA_PROC_IPH_CFG_IPH_THRESHOLD_BMSK 0x3
-#define HWIO_IPA_PROC_IPH_CFG_IPH_THRESHOLD_SHFT 0x0
-#define HWIO_IPA_QSB_MAX_WRITES_ADDR (IPA_CFG_REG_BASE + 0x00000074)
-#define HWIO_IPA_QSB_MAX_WRITES_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00000074)
-#define HWIO_IPA_QSB_MAX_WRITES_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00000074)
-#define HWIO_IPA_QSB_MAX_READS_ADDR (IPA_CFG_REG_BASE + 0x00000078)
-#define HWIO_IPA_QSB_MAX_READS_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00000078)
-#define HWIO_IPA_QSB_MAX_READS_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00000078)
-#define HWIO_IPA_QSB_OUTSTANDING_COUNTER_ADDR (IPA_CFG_REG_BASE + \
-					       0x0000007c)
-#define HWIO_IPA_QSB_OUTSTANDING_COUNTER_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					       0x0000007c)
-#define HWIO_IPA_QSB_OUTSTANDING_COUNTER_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					       0x0000007c)
-#define HWIO_IPA_QSB_OUTSTANDING_BEATS_COUNTER_ADDR (IPA_CFG_REG_BASE +	\
-						     0x00000080)
-#define HWIO_IPA_QSB_OUTSTANDING_BEATS_COUNTER_PHYS (IPA_CFG_REG_BASE_PHYS \
-						     + 0x00000080)
-#define HWIO_IPA_QSB_OUTSTANDING_BEATS_COUNTER_OFFS (IPA_CFG_REG_BASE_OFFS \
-						     + 0x00000080)
-#define HWIO_IPA_QSB_READ_CFG_ADDR (IPA_CFG_REG_BASE + 0x00000084)
-#define HWIO_IPA_QSB_READ_CFG_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00000084)
-#define HWIO_IPA_QSB_READ_CFG_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00000084)
-#define HWIO_IPA_DPL_TIMER_LSB_ADDR (IPA_CFG_REG_BASE + 0x00000088)
-#define HWIO_IPA_DPL_TIMER_LSB_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00000088)
-#define HWIO_IPA_DPL_TIMER_LSB_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00000088)
-#define HWIO_IPA_DPL_TIMER_LSB_RMSK 0xffffffff
-#define HWIO_IPA_DPL_TIMER_LSB_ATTR 0x3
-#define HWIO_IPA_DPL_TIMER_LSB_IN in_dword_masked( \
-		HWIO_IPA_DPL_TIMER_LSB_ADDR, \
-		HWIO_IPA_DPL_TIMER_LSB_RMSK)
-#define HWIO_IPA_DPL_TIMER_LSB_INM(m) in_dword_masked( \
-		HWIO_IPA_DPL_TIMER_LSB_ADDR, \
-		m)
-#define HWIO_IPA_DPL_TIMER_LSB_OUT(v) out_dword( \
-		HWIO_IPA_DPL_TIMER_LSB_ADDR, \
-		v)
-#define HWIO_IPA_DPL_TIMER_LSB_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_DPL_TIMER_LSB_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_DPL_TIMER_LSB_IN)
-#define HWIO_IPA_DPL_TIMER_LSB_TOD_LSB_BMSK 0xffffffff
-#define HWIO_IPA_DPL_TIMER_LSB_TOD_LSB_SHFT 0x0
-#define HWIO_IPA_DPL_TIMER_MSB_ADDR (IPA_CFG_REG_BASE + 0x0000008c)
-#define HWIO_IPA_DPL_TIMER_MSB_PHYS (IPA_CFG_REG_BASE_PHYS + 0x0000008c)
-#define HWIO_IPA_DPL_TIMER_MSB_OFFS (IPA_CFG_REG_BASE_OFFS + 0x0000008c)
-#define HWIO_IPA_DPL_TIMER_MSB_RMSK 0x8000ffff
-#define HWIO_IPA_DPL_TIMER_MSB_ATTR 0x3
-#define HWIO_IPA_DPL_TIMER_MSB_IN in_dword_masked( \
-		HWIO_IPA_DPL_TIMER_MSB_ADDR, \
-		HWIO_IPA_DPL_TIMER_MSB_RMSK)
-#define HWIO_IPA_DPL_TIMER_MSB_INM(m) in_dword_masked( \
-		HWIO_IPA_DPL_TIMER_MSB_ADDR, \
-		m)
-#define HWIO_IPA_DPL_TIMER_MSB_OUT(v) out_dword( \
-		HWIO_IPA_DPL_TIMER_MSB_ADDR, \
-		v)
-#define HWIO_IPA_DPL_TIMER_MSB_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_DPL_TIMER_MSB_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_DPL_TIMER_MSB_IN)
-#define HWIO_IPA_DPL_TIMER_MSB_TIMER_EN_BMSK 0x80000000
-#define HWIO_IPA_DPL_TIMER_MSB_TIMER_EN_SHFT 0x1f
-#define HWIO_IPA_DPL_TIMER_MSB_TOD_MSB_BMSK 0xffff
-#define HWIO_IPA_DPL_TIMER_MSB_TOD_MSB_SHFT 0x0
-#define HWIO_IPA_STATE_TX_WRAPPER_ADDR (IPA_CFG_REG_BASE + 0x00000090)
-#define HWIO_IPA_STATE_TX_WRAPPER_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00000090)
-#define HWIO_IPA_STATE_TX_WRAPPER_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00000090)
-#define HWIO_IPA_STATE_TX_WRAPPER_RMSK 0x1e01ffff
-#define HWIO_IPA_STATE_TX_WRAPPER_ATTR 0x1
-#define HWIO_IPA_STATE_TX_WRAPPER_IN in_dword_masked( \
-		HWIO_IPA_STATE_TX_WRAPPER_ADDR,	\
-		HWIO_IPA_STATE_TX_WRAPPER_RMSK)
-#define HWIO_IPA_STATE_TX_WRAPPER_INM(m) in_dword_masked( \
-		HWIO_IPA_STATE_TX_WRAPPER_ADDR,	\
-		m)
-#define HWIO_IPA_STATE_TX_WRAPPER_COAL_SLAVE_OPEN_FRAME_BMSK 0x1e000000
-#define HWIO_IPA_STATE_TX_WRAPPER_COAL_SLAVE_OPEN_FRAME_SHFT 0x19
-#define HWIO_IPA_STATE_TX_WRAPPER_COAL_SLAVE_CTX_IDLE_BMSK 0x10000
-#define HWIO_IPA_STATE_TX_WRAPPER_COAL_SLAVE_CTX_IDLE_SHFT 0x10
-#define HWIO_IPA_STATE_TX_WRAPPER_COAL_SLAVE_IDLE_BMSK 0x8000
-#define HWIO_IPA_STATE_TX_WRAPPER_COAL_SLAVE_IDLE_SHFT 0xf
-#define HWIO_IPA_STATE_TX_WRAPPER_COAL_DIRECT_DMA_BMSK 0x6000
-#define HWIO_IPA_STATE_TX_WRAPPER_COAL_DIRECT_DMA_SHFT 0xd
-#define HWIO_IPA_STATE_TX_WRAPPER_NLO_DIRECT_DMA_BMSK 0x1800
-#define HWIO_IPA_STATE_TX_WRAPPER_NLO_DIRECT_DMA_SHFT 0xb
-#define HWIO_IPA_STATE_TX_WRAPPER_PKT_DROP_CNT_IDLE_BMSK 0x400
-#define HWIO_IPA_STATE_TX_WRAPPER_PKT_DROP_CNT_IDLE_SHFT 0xa
-#define HWIO_IPA_STATE_TX_WRAPPER_TRNSEQ_FORCE_VALID_BMSK 0x200
-#define HWIO_IPA_STATE_TX_WRAPPER_TRNSEQ_FORCE_VALID_SHFT 0x9
-#define HWIO_IPA_STATE_TX_WRAPPER_MBIM_DIRECT_DMA_BMSK 0x180
-#define HWIO_IPA_STATE_TX_WRAPPER_MBIM_DIRECT_DMA_SHFT 0x7
-#define HWIO_IPA_STATE_TX_WRAPPER_IPA_MBIM_PKT_FMS_IDLE_BMSK 0x40
-#define HWIO_IPA_STATE_TX_WRAPPER_IPA_MBIM_PKT_FMS_IDLE_SHFT 0x6
-#define HWIO_IPA_STATE_TX_WRAPPER_IPA_PROD_BRESP_TOGGLE_IDLE_BMSK 0x20
-#define HWIO_IPA_STATE_TX_WRAPPER_IPA_PROD_BRESP_TOGGLE_IDLE_SHFT 0x5
-#define HWIO_IPA_STATE_TX_WRAPPER_IPA_PROD_BRESP_EMPTY_BMSK 0x10
-#define HWIO_IPA_STATE_TX_WRAPPER_IPA_PROD_BRESP_EMPTY_SHFT 0x4
-#define HWIO_IPA_STATE_TX_WRAPPER_IPA_PROD_ACKMNGR_STATE_IDLE_BMSK 0x8
-#define HWIO_IPA_STATE_TX_WRAPPER_IPA_PROD_ACKMNGR_STATE_IDLE_SHFT 0x3
-#define HWIO_IPA_STATE_TX_WRAPPER_IPA_PROD_ACKMNGR_DB_EMPTY_BMSK 0x4
-#define HWIO_IPA_STATE_TX_WRAPPER_IPA_PROD_ACKMNGR_DB_EMPTY_SHFT 0x2
-#define HWIO_IPA_STATE_TX_WRAPPER_TX1_IDLE_BMSK 0x2
-#define HWIO_IPA_STATE_TX_WRAPPER_TX1_IDLE_SHFT 0x1
-#define HWIO_IPA_STATE_TX_WRAPPER_TX0_IDLE_BMSK 0x1
-#define HWIO_IPA_STATE_TX_WRAPPER_TX0_IDLE_SHFT 0x0
-#define HWIO_IPA_STATE_TX1_ADDR (IPA_CFG_REG_BASE + 0x00000094)
-#define HWIO_IPA_STATE_TX1_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00000094)
-#define HWIO_IPA_STATE_TX1_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00000094)
-#define HWIO_IPA_STATE_TX1_RMSK 0xffffffff
-#define HWIO_IPA_STATE_TX1_ATTR 0x1
-#define HWIO_IPA_STATE_TX1_IN in_dword_masked(HWIO_IPA_STATE_TX1_ADDR, \
-					      HWIO_IPA_STATE_TX1_RMSK)
-#define HWIO_IPA_STATE_TX1_INM(m) in_dword_masked(HWIO_IPA_STATE_TX1_ADDR, \
-						  m)
-#define HWIO_IPA_STATE_TX1_SUSPEND_REQ_EMPTY_BMSK 0x80000000
-#define HWIO_IPA_STATE_TX1_SUSPEND_REQ_EMPTY_SHFT 0x1f
-#define HWIO_IPA_STATE_TX1_LAST_CMD_PIPE_BMSK 0x7c000000
-#define HWIO_IPA_STATE_TX1_LAST_CMD_PIPE_SHFT 0x1a
-#define HWIO_IPA_STATE_TX1_CS_SNIF_IDLE_BMSK 0x2000000
-#define HWIO_IPA_STATE_TX1_CS_SNIF_IDLE_SHFT 0x19
-#define HWIO_IPA_STATE_TX1_SUSPEND_EMPTY_BMSK 0x1000000
-#define HWIO_IPA_STATE_TX1_SUSPEND_EMPTY_SHFT 0x18
-#define HWIO_IPA_STATE_TX1_RSRCREL_IDLE_BMSK 0x800000
-#define HWIO_IPA_STATE_TX1_RSRCREL_IDLE_SHFT 0x17
-#define HWIO_IPA_STATE_TX1_HOLB_MASK_IDLE_BMSK 0x400000
-#define HWIO_IPA_STATE_TX1_HOLB_MASK_IDLE_SHFT 0x16
-#define HWIO_IPA_STATE_TX1_HOLB_IDLE_BMSK 0x200000
-#define HWIO_IPA_STATE_TX1_HOLB_IDLE_SHFT 0x15
-#define HWIO_IPA_STATE_TX1_ALIGNER_EMPTY_BMSK 0x100000
-#define HWIO_IPA_STATE_TX1_ALIGNER_EMPTY_SHFT 0x14
-#define HWIO_IPA_STATE_TX1_PF_EMPTY_BMSK 0x80000
-#define HWIO_IPA_STATE_TX1_PF_EMPTY_SHFT 0x13
-#define HWIO_IPA_STATE_TX1_PF_IDLE_BMSK 0x40000
-#define HWIO_IPA_STATE_TX1_PF_IDLE_SHFT 0x12
-#define HWIO_IPA_STATE_TX1_DMAW_LAST_OUTSD_IDLE_BMSK 0x20000
-#define HWIO_IPA_STATE_TX1_DMAW_LAST_OUTSD_IDLE_SHFT 0x11
-#define HWIO_IPA_STATE_TX1_DMAW_IDLE_BMSK 0x10000
-#define HWIO_IPA_STATE_TX1_DMAW_IDLE_SHFT 0x10
-#define HWIO_IPA_STATE_TX1_AR_IDLE_BMSK 0x8000
-#define HWIO_IPA_STATE_TX1_AR_IDLE_SHFT 0xf
-#define HWIO_IPA_STATE_TX1_TX_CMD_BRESP_INJ_IDLE_BMSK 0x4000
-#define HWIO_IPA_STATE_TX1_TX_CMD_BRESP_INJ_IDLE_SHFT 0xe
-#define HWIO_IPA_STATE_TX1_TX_CMD_BRESP_ALOC_IDLE_BMSK 0x2000
-#define HWIO_IPA_STATE_TX1_TX_CMD_BRESP_ALOC_IDLE_SHFT 0xd
-#define HWIO_IPA_STATE_TX1_TX_CMD_SNIF_IDLE_BMSK 0x1000
-#define HWIO_IPA_STATE_TX1_TX_CMD_SNIF_IDLE_SHFT 0xc
-#define HWIO_IPA_STATE_TX1_TX_CMD_TRNSEQ_IDLE_BMSK 0x800
-#define HWIO_IPA_STATE_TX1_TX_CMD_TRNSEQ_IDLE_SHFT 0xb
-#define HWIO_IPA_STATE_TX1_TX_CMD_MAIN_IDLE_BMSK 0x400
-#define HWIO_IPA_STATE_TX1_TX_CMD_MAIN_IDLE_SHFT 0xa
-#define HWIO_IPA_STATE_TX1_PA_PUB_CNT_EMPTY_BMSK 0x200
-#define HWIO_IPA_STATE_TX1_PA_PUB_CNT_EMPTY_SHFT 0x9
-#define HWIO_IPA_STATE_TX1_PA_RST_IDLE_BMSK 0x100
-#define HWIO_IPA_STATE_TX1_PA_RST_IDLE_SHFT 0x8
-#define HWIO_IPA_STATE_TX1_PA_CTX_IDLE_BMSK 0x80
-#define HWIO_IPA_STATE_TX1_PA_CTX_IDLE_SHFT 0x7
-#define HWIO_IPA_STATE_TX1_PA_IDLE_BMSK 0x40
-#define HWIO_IPA_STATE_TX1_PA_IDLE_SHFT 0x6
-#define HWIO_IPA_STATE_TX1_ARBIT_TYPE_BMSK 0x38
-#define HWIO_IPA_STATE_TX1_ARBIT_TYPE_SHFT 0x3
-#define HWIO_IPA_STATE_TX1_FLOPPED_ARBIT_TYPE_BMSK 0x7
-#define HWIO_IPA_STATE_TX1_FLOPPED_ARBIT_TYPE_SHFT 0x0
-#define HWIO_IPA_STATE_FETCHER_ADDR (IPA_CFG_REG_BASE + 0x00000098)
-#define HWIO_IPA_STATE_FETCHER_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00000098)
-#define HWIO_IPA_STATE_FETCHER_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00000098)
-#define HWIO_IPA_STATE_FETCHER_RMSK 0xfffff
-#define HWIO_IPA_STATE_FETCHER_ATTR 0x1
-#define HWIO_IPA_STATE_FETCHER_IN in_dword_masked( \
-		HWIO_IPA_STATE_FETCHER_ADDR, \
-		HWIO_IPA_STATE_FETCHER_RMSK)
-#define HWIO_IPA_STATE_FETCHER_INM(m) in_dword_masked( \
-		HWIO_IPA_STATE_FETCHER_ADDR, \
-		m)
-#define HWIO_IPA_STATE_FETCHER_IPA_HPS_IMM_CMD_EXEC_STATE_IDLE_BMSK \
-	0x80000
-#define HWIO_IPA_STATE_FETCHER_IPA_HPS_IMM_CMD_EXEC_STATE_IDLE_SHFT 0x13
-#define HWIO_IPA_STATE_FETCHER_IPA_HPS_DMAR_SLOT_STATE_IDLE_BMSK 0x7f000
-#define HWIO_IPA_STATE_FETCHER_IPA_HPS_DMAR_SLOT_STATE_IDLE_SHFT 0xc
-#define HWIO_IPA_STATE_FETCHER_IPA_HPS_DMAR_STATE_IDLE_BMSK 0xfe0
-#define HWIO_IPA_STATE_FETCHER_IPA_HPS_DMAR_STATE_IDLE_SHFT 0x5
-#define HWIO_IPA_STATE_FETCHER_IPA_HPS_FTCH_CMPLT_STATE_IDLE_BMSK 0x10
-#define HWIO_IPA_STATE_FETCHER_IPA_HPS_FTCH_CMPLT_STATE_IDLE_SHFT 0x4
-#define HWIO_IPA_STATE_FETCHER_IPA_HPS_FTCH_IMM_STATE_IDLE_BMSK 0x8
-#define HWIO_IPA_STATE_FETCHER_IPA_HPS_FTCH_IMM_STATE_IDLE_SHFT 0x3
-#define HWIO_IPA_STATE_FETCHER_IPA_HPS_FTCH_PKT_STATE_IDLE_BMSK 0x4
-#define HWIO_IPA_STATE_FETCHER_IPA_HPS_FTCH_PKT_STATE_IDLE_SHFT 0x2
-#define HWIO_IPA_STATE_FETCHER_IPA_HPS_FTCH_ALLOC_STATE_IDLE_BMSK 0x2
-#define HWIO_IPA_STATE_FETCHER_IPA_HPS_FTCH_ALLOC_STATE_IDLE_SHFT 0x1
-#define HWIO_IPA_STATE_FETCHER_IPA_HPS_FTCH_STATE_IDLE_BMSK 0x1
-#define HWIO_IPA_STATE_FETCHER_IPA_HPS_FTCH_STATE_IDLE_SHFT 0x0
-#define HWIO_IPA_STATE_FETCHER_MASK_0_ADDR (IPA_CFG_REG_BASE + 0x0000009c)
-#define HWIO_IPA_STATE_FETCHER_MASK_0_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					    0x0000009c)
-#define HWIO_IPA_STATE_FETCHER_MASK_0_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					    0x0000009c)
-#define HWIO_IPA_STATE_FETCHER_MASK_0_RMSK 0xffffffff
-#define HWIO_IPA_STATE_FETCHER_MASK_0_ATTR 0x1
-#define HWIO_IPA_STATE_FETCHER_MASK_0_IN in_dword_masked( \
-		HWIO_IPA_STATE_FETCHER_MASK_0_ADDR, \
-		HWIO_IPA_STATE_FETCHER_MASK_0_RMSK)
-#define HWIO_IPA_STATE_FETCHER_MASK_0_INM(m) in_dword_masked( \
-		HWIO_IPA_STATE_FETCHER_MASK_0_ADDR, \
-		m)
-#define	\
-	HWIO_IPA_STATE_FETCHER_MASK_0_MASK_QUEUE_NO_RESOURCES_HPS_DMAR_BMSK \
-	0xff000000
-#define	\
-	HWIO_IPA_STATE_FETCHER_MASK_0_MASK_QUEUE_NO_RESOURCES_HPS_DMAR_SHFT \
-	0x18
-#define HWIO_IPA_STATE_FETCHER_MASK_0_MASK_QUEUE_NO_RESOURCES_CONTEXT_BMSK \
-	0xff0000
-#define HWIO_IPA_STATE_FETCHER_MASK_0_MASK_QUEUE_NO_RESOURCES_CONTEXT_SHFT \
-	0x10
-#define HWIO_IPA_STATE_FETCHER_MASK_0_MASK_QUEUE_IMM_EXEC_BMSK 0xff00
-#define HWIO_IPA_STATE_FETCHER_MASK_0_MASK_QUEUE_IMM_EXEC_SHFT 0x8
-#define HWIO_IPA_STATE_FETCHER_MASK_0_MASK_QUEUE_DMAR_USES_QUEUE_BMSK 0xff
-#define HWIO_IPA_STATE_FETCHER_MASK_0_MASK_QUEUE_DMAR_USES_QUEUE_SHFT 0x0
-#define HWIO_IPA_STATE_FETCHER_MASK_1_ADDR (IPA_CFG_REG_BASE + 0x000000cc)
-#define HWIO_IPA_STATE_FETCHER_MASK_1_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					    0x000000cc)
-#define HWIO_IPA_STATE_FETCHER_MASK_1_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					    0x000000cc)
-#define HWIO_IPA_STATE_FETCHER_MASK_1_RMSK 0xffffffff
-#define HWIO_IPA_STATE_FETCHER_MASK_1_ATTR 0x1
-#define HWIO_IPA_STATE_FETCHER_MASK_1_IN in_dword_masked( \
-		HWIO_IPA_STATE_FETCHER_MASK_1_ADDR, \
-		HWIO_IPA_STATE_FETCHER_MASK_1_RMSK)
-#define HWIO_IPA_STATE_FETCHER_MASK_1_INM(m) in_dword_masked( \
-		HWIO_IPA_STATE_FETCHER_MASK_1_ADDR, \
-		m)
-#define HWIO_IPA_STATE_FETCHER_MASK_1_MASK_QUEUE_NO_SPACE_DPL_FIFO_BMSK	\
-	0xff000000
-#define HWIO_IPA_STATE_FETCHER_MASK_1_MASK_QUEUE_NO_SPACE_DPL_FIFO_SHFT	\
-	0x18
-#define HWIO_IPA_STATE_FETCHER_MASK_1_MASK_QUEUE_STEP_MODE_BMSK 0xff0000
-#define HWIO_IPA_STATE_FETCHER_MASK_1_MASK_QUEUE_STEP_MODE_SHFT 0x10
-#define HWIO_IPA_STATE_FETCHER_MASK_1_MASK_QUEUE_ARB_LOCK_BMSK 0xff00
-#define HWIO_IPA_STATE_FETCHER_MASK_1_MASK_QUEUE_ARB_LOCK_SHFT 0x8
-#define	\
-	HWIO_IPA_STATE_FETCHER_MASK_1_MASK_QUEUE_NO_RESOURCES_ACK_ENTRY_BMSK \
-	0xff
-#define	\
-	HWIO_IPA_STATE_FETCHER_MASK_1_MASK_QUEUE_NO_RESOURCES_ACK_ENTRY_SHFT \
-	0x0
-#define HWIO_IPA_STATE_DPL_FIFO_ADDR (IPA_CFG_REG_BASE + 0x000000d0)
-#define HWIO_IPA_STATE_DPL_FIFO_PHYS (IPA_CFG_REG_BASE_PHYS + 0x000000d0)
-#define HWIO_IPA_STATE_DPL_FIFO_OFFS (IPA_CFG_REG_BASE_OFFS + 0x000000d0)
-#define HWIO_IPA_STATE_DPL_FIFO_RMSK 0x7
-#define HWIO_IPA_STATE_DPL_FIFO_ATTR 0x1
-#define HWIO_IPA_STATE_DPL_FIFO_IN in_dword_masked( \
-		HWIO_IPA_STATE_DPL_FIFO_ADDR, \
-		HWIO_IPA_STATE_DPL_FIFO_RMSK)
-#define HWIO_IPA_STATE_DPL_FIFO_INM(m) in_dword_masked(	\
-		HWIO_IPA_STATE_DPL_FIFO_ADDR, \
-		m)
-#define HWIO_IPA_STATE_DPL_FIFO_POP_FSM_STATE_BMSK 0x7
-#define HWIO_IPA_STATE_DPL_FIFO_POP_FSM_STATE_SHFT 0x0
-#define HWIO_IPA_STATE_COAL_MASTER_ADDR (IPA_CFG_REG_BASE + 0x000000d4)
-#define HWIO_IPA_STATE_COAL_MASTER_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					 0x000000d4)
-#define HWIO_IPA_STATE_COAL_MASTER_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					 0x000000d4)
-#define HWIO_IPA_STATE_COAL_MASTER_RMSK 0xffffffff
-#define HWIO_IPA_STATE_COAL_MASTER_ATTR 0x1
-#define HWIO_IPA_STATE_COAL_MASTER_IN in_dword_masked( \
-		HWIO_IPA_STATE_COAL_MASTER_ADDR, \
-		HWIO_IPA_STATE_COAL_MASTER_RMSK)
-#define HWIO_IPA_STATE_COAL_MASTER_INM(m) in_dword_masked( \
-		HWIO_IPA_STATE_COAL_MASTER_ADDR, \
-		m)
-#define HWIO_IPA_STATE_COAL_MASTER_VP_TIMER_EXPIRED_BMSK 0xf0000000
-#define HWIO_IPA_STATE_COAL_MASTER_VP_TIMER_EXPIRED_SHFT 0x1c
-#define HWIO_IPA_STATE_COAL_MASTER_LRU_VP_BMSK 0xf000000
-#define HWIO_IPA_STATE_COAL_MASTER_LRU_VP_SHFT 0x18
-#define HWIO_IPA_STATE_COAL_MASTER_INIT_VP_FSM_STATE_BMSK 0xf00000
-#define HWIO_IPA_STATE_COAL_MASTER_INIT_VP_FSM_STATE_SHFT 0x14
-#define HWIO_IPA_STATE_COAL_MASTER_CHECK_FIT_FSM_STATE_BMSK 0xf0000
-#define HWIO_IPA_STATE_COAL_MASTER_CHECK_FIT_FSM_STATE_SHFT 0x10
-#define HWIO_IPA_STATE_COAL_MASTER_HASH_CALC_FSM_STATE_BMSK 0xf000
-#define HWIO_IPA_STATE_COAL_MASTER_HASH_CALC_FSM_STATE_SHFT 0xc
-#define HWIO_IPA_STATE_COAL_MASTER_FIND_OPEN_FSM_STATE_BMSK 0xf00
-#define HWIO_IPA_STATE_COAL_MASTER_FIND_OPEN_FSM_STATE_SHFT 0x8
-#define HWIO_IPA_STATE_COAL_MASTER_MAIN_FSM_STATE_BMSK 0xf0
-#define HWIO_IPA_STATE_COAL_MASTER_MAIN_FSM_STATE_SHFT 0x4
-#define HWIO_IPA_STATE_COAL_MASTER_VP_VLD_BMSK 0xf
-#define HWIO_IPA_STATE_COAL_MASTER_VP_VLD_SHFT 0x0
-#define HWIO_IPA_STATE_DFETCHER_ADDR (IPA_CFG_REG_BASE + 0x000000a0)
-#define HWIO_IPA_STATE_DFETCHER_PHYS (IPA_CFG_REG_BASE_PHYS + 0x000000a0)
-#define HWIO_IPA_STATE_DFETCHER_OFFS (IPA_CFG_REG_BASE_OFFS + 0x000000a0)
-#define HWIO_IPA_STATE_DFETCHER_RMSK 0x3f3f3
-#define HWIO_IPA_STATE_DFETCHER_ATTR 0x1
-#define HWIO_IPA_STATE_DFETCHER_IN in_dword_masked( \
-		HWIO_IPA_STATE_DFETCHER_ADDR, \
-		HWIO_IPA_STATE_DFETCHER_RMSK)
-#define HWIO_IPA_STATE_DFETCHER_INM(m) in_dword_masked(	\
-		HWIO_IPA_STATE_DFETCHER_ADDR, \
-		m)
-#define HWIO_IPA_STATE_DFETCHER_IPA_DPS_DMAR_SLOT_STATE_IDLE_BMSK 0x3f000
-#define HWIO_IPA_STATE_DFETCHER_IPA_DPS_DMAR_SLOT_STATE_IDLE_SHFT 0xc
-#define HWIO_IPA_STATE_DFETCHER_IPA_DPS_DMAR_STATE_IDLE_BMSK 0x3f0
-#define HWIO_IPA_STATE_DFETCHER_IPA_DPS_DMAR_STATE_IDLE_SHFT 0x4
-#define HWIO_IPA_STATE_DFETCHER_IPA_DPS_FTCH_CMPLT_STATE_IDLE_BMSK 0x2
-#define HWIO_IPA_STATE_DFETCHER_IPA_DPS_FTCH_CMPLT_STATE_IDLE_SHFT 0x1
-#define HWIO_IPA_STATE_DFETCHER_IPA_DPS_FTCH_PKT_STATE_IDLE_BMSK 0x1
-#define HWIO_IPA_STATE_DFETCHER_IPA_DPS_FTCH_PKT_STATE_IDLE_SHFT 0x0
-#define HWIO_IPA_STATE_ACL_ADDR (IPA_CFG_REG_BASE + 0x000000a4)
-#define HWIO_IPA_STATE_ACL_PHYS (IPA_CFG_REG_BASE_PHYS + 0x000000a4)
-#define HWIO_IPA_STATE_ACL_OFFS (IPA_CFG_REG_BASE_OFFS + 0x000000a4)
-#define HWIO_IPA_STATE_ACL_RMSK 0xffcffff
-#define HWIO_IPA_STATE_ACL_ATTR 0x1
-#define HWIO_IPA_STATE_ACL_IN in_dword_masked(HWIO_IPA_STATE_ACL_ADDR, \
-					      HWIO_IPA_STATE_ACL_RMSK)
-#define HWIO_IPA_STATE_ACL_INM(m) in_dword_masked(HWIO_IPA_STATE_ACL_ADDR, \
-						  m)
-#define HWIO_IPA_STATE_ACL_IPA_HPS_COAL_MASTER_ACTIVE_BMSK 0x8000000
-#define HWIO_IPA_STATE_ACL_IPA_HPS_COAL_MASTER_ACTIVE_SHFT 0x1b
-#define HWIO_IPA_STATE_ACL_IPA_HPS_COAL_MASTER_EMPTY_BMSK 0x4000000
-#define HWIO_IPA_STATE_ACL_IPA_HPS_COAL_MASTER_EMPTY_SHFT 0x1a
-#define HWIO_IPA_STATE_ACL_IPA_DPS_D_DCPH_2ND_ACTIVE_BMSK 0x2000000
-#define HWIO_IPA_STATE_ACL_IPA_DPS_D_DCPH_2ND_ACTIVE_SHFT 0x19
-#define HWIO_IPA_STATE_ACL_IPA_DPS_D_DCPH_2ND_EMPTY_BMSK 0x1000000
-#define HWIO_IPA_STATE_ACL_IPA_DPS_D_DCPH_2ND_EMPTY_SHFT 0x18
-#define HWIO_IPA_STATE_ACL_IPA_DPS_SEQUENCER_IDLE_BMSK 0x800000
-#define HWIO_IPA_STATE_ACL_IPA_DPS_SEQUENCER_IDLE_SHFT 0x17
-#define HWIO_IPA_STATE_ACL_IPA_HPS_SEQUENCER_IDLE_BMSK 0x400000
-#define HWIO_IPA_STATE_ACL_IPA_HPS_SEQUENCER_IDLE_SHFT 0x16
-#define HWIO_IPA_STATE_ACL_IPA_DPS_D_DCPH_2_ACTIVE_BMSK 0x200000
-#define HWIO_IPA_STATE_ACL_IPA_DPS_D_DCPH_2_ACTIVE_SHFT 0x15
-#define HWIO_IPA_STATE_ACL_IPA_DPS_D_DCPH_2_EMPTY_BMSK 0x100000
-#define HWIO_IPA_STATE_ACL_IPA_DPS_D_DCPH_2_EMPTY_SHFT 0x14
-#define HWIO_IPA_STATE_ACL_IPA_DPS_DISPATCHER_ACTIVE_BMSK 0x80000
-#define HWIO_IPA_STATE_ACL_IPA_DPS_DISPATCHER_ACTIVE_SHFT 0x13
-#define HWIO_IPA_STATE_ACL_IPA_DPS_DISPATCHER_EMPTY_BMSK 0x40000
-#define HWIO_IPA_STATE_ACL_IPA_DPS_DISPATCHER_EMPTY_SHFT 0x12
-#define HWIO_IPA_STATE_ACL_IPA_DPS_D_DCPH_ACTIVE_BMSK 0x8000
-#define HWIO_IPA_STATE_ACL_IPA_DPS_D_DCPH_ACTIVE_SHFT 0xf
-#define HWIO_IPA_STATE_ACL_IPA_DPS_D_DCPH_EMPTY_BMSK 0x4000
-#define HWIO_IPA_STATE_ACL_IPA_DPS_D_DCPH_EMPTY_SHFT 0xe
-#define HWIO_IPA_STATE_ACL_IPA_HPS_ENQUEUER_ACTIVE_BMSK 0x2000
-#define HWIO_IPA_STATE_ACL_IPA_HPS_ENQUEUER_ACTIVE_SHFT 0xd
-#define HWIO_IPA_STATE_ACL_IPA_HPS_ENQUEUER_EMPTY_BMSK 0x1000
-#define HWIO_IPA_STATE_ACL_IPA_HPS_ENQUEUER_EMPTY_SHFT 0xc
-#define HWIO_IPA_STATE_ACL_IPA_HPS_UCP_ACTIVE_BMSK 0x800
-#define HWIO_IPA_STATE_ACL_IPA_HPS_UCP_ACTIVE_SHFT 0xb
-#define HWIO_IPA_STATE_ACL_IPA_HPS_UCP_EMPTY_BMSK 0x400
-#define HWIO_IPA_STATE_ACL_IPA_HPS_UCP_EMPTY_SHFT 0xa
-#define HWIO_IPA_STATE_ACL_IPA_HPS_HDRI_ACTIVE_BMSK 0x200
-#define HWIO_IPA_STATE_ACL_IPA_HPS_HDRI_ACTIVE_SHFT 0x9
-#define HWIO_IPA_STATE_ACL_IPA_HPS_HDRI_EMPTY_BMSK 0x100
-#define HWIO_IPA_STATE_ACL_IPA_HPS_HDRI_EMPTY_SHFT 0x8
-#define HWIO_IPA_STATE_ACL_IPA_HPS_ROUTER_ACTIVE_BMSK 0x80
-#define HWIO_IPA_STATE_ACL_IPA_HPS_ROUTER_ACTIVE_SHFT 0x7
-#define HWIO_IPA_STATE_ACL_IPA_HPS_ROUTER_EMPTY_BMSK 0x40
-#define HWIO_IPA_STATE_ACL_IPA_HPS_ROUTER_EMPTY_SHFT 0x6
-#define HWIO_IPA_STATE_ACL_IPA_HPS_FILTER_NAT_ACTIVE_BMSK 0x20
-#define HWIO_IPA_STATE_ACL_IPA_HPS_FILTER_NAT_ACTIVE_SHFT 0x5
-#define HWIO_IPA_STATE_ACL_IPA_HPS_FILTER_NAT_EMPTY_BMSK 0x10
-#define HWIO_IPA_STATE_ACL_IPA_HPS_FILTER_NAT_EMPTY_SHFT 0x4
-#define HWIO_IPA_STATE_ACL_IPA_HPS_PKT_PARSER_ACTIVE_BMSK 0x8
-#define HWIO_IPA_STATE_ACL_IPA_HPS_PKT_PARSER_ACTIVE_SHFT 0x3
-#define HWIO_IPA_STATE_ACL_IPA_HPS_PKT_PARSER_EMPTY_BMSK 0x4
-#define HWIO_IPA_STATE_ACL_IPA_HPS_PKT_PARSER_EMPTY_SHFT 0x2
-#define HWIO_IPA_STATE_ACL_IPA_HPS_H_DCPH_ACTIVE_BMSK 0x2
-#define HWIO_IPA_STATE_ACL_IPA_HPS_H_DCPH_ACTIVE_SHFT 0x1
-#define HWIO_IPA_STATE_ACL_IPA_HPS_H_DCPH_EMPTY_BMSK 0x1
-#define HWIO_IPA_STATE_ACL_IPA_HPS_H_DCPH_EMPTY_SHFT 0x0
-#define HWIO_IPA_STATE_GSI_TLV_ADDR (IPA_CFG_REG_BASE + 0x000000b8)
-#define HWIO_IPA_STATE_GSI_TLV_PHYS (IPA_CFG_REG_BASE_PHYS + 0x000000b8)
-#define HWIO_IPA_STATE_GSI_TLV_OFFS (IPA_CFG_REG_BASE_OFFS + 0x000000b8)
-#define HWIO_IPA_STATE_GSI_TLV_RMSK 0x1
-#define HWIO_IPA_STATE_GSI_TLV_ATTR 0x1
-#define HWIO_IPA_STATE_GSI_TLV_IN in_dword_masked( \
-		HWIO_IPA_STATE_GSI_TLV_ADDR, \
-		HWIO_IPA_STATE_GSI_TLV_RMSK)
-#define HWIO_IPA_STATE_GSI_TLV_INM(m) in_dword_masked( \
-		HWIO_IPA_STATE_GSI_TLV_ADDR, \
-		m)
-#define HWIO_IPA_STATE_GSI_TLV_IPA_GSI_TOGGLE_FSM_IDLE_BMSK 0x1
-#define HWIO_IPA_STATE_GSI_TLV_IPA_GSI_TOGGLE_FSM_IDLE_SHFT 0x0
-#define HWIO_IPA_STATE_GSI_AOS_ADDR (IPA_CFG_REG_BASE + 0x000000bc)
-#define HWIO_IPA_STATE_GSI_AOS_PHYS (IPA_CFG_REG_BASE_PHYS + 0x000000bc)
-#define HWIO_IPA_STATE_GSI_AOS_OFFS (IPA_CFG_REG_BASE_OFFS + 0x000000bc)
-#define HWIO_IPA_STATE_GSI_AOS_RMSK 0x1
-#define HWIO_IPA_STATE_GSI_AOS_ATTR 0x1
-#define HWIO_IPA_STATE_GSI_AOS_IN in_dword_masked( \
-		HWIO_IPA_STATE_GSI_AOS_ADDR, \
-		HWIO_IPA_STATE_GSI_AOS_RMSK)
-#define HWIO_IPA_STATE_GSI_AOS_INM(m) in_dword_masked( \
-		HWIO_IPA_STATE_GSI_AOS_ADDR, \
-		m)
-#define HWIO_IPA_STATE_GSI_AOS_IPA_GSI_AOS_FSM_IDLE_BMSK 0x1
-#define HWIO_IPA_STATE_GSI_AOS_IPA_GSI_AOS_FSM_IDLE_SHFT 0x0
-#define HWIO_IPA_STATE_GSI_IF_ADDR (IPA_CFG_REG_BASE + 0x000000c0)
-#define HWIO_IPA_STATE_GSI_IF_PHYS (IPA_CFG_REG_BASE_PHYS + 0x000000c0)
-#define HWIO_IPA_STATE_GSI_IF_OFFS (IPA_CFG_REG_BASE_OFFS + 0x000000c0)
-#define HWIO_IPA_STATE_GSI_IF_RMSK 0xff
-#define HWIO_IPA_STATE_GSI_IF_ATTR 0x1
-#define HWIO_IPA_STATE_GSI_IF_IN in_dword_masked( \
-		HWIO_IPA_STATE_GSI_IF_ADDR, \
-		HWIO_IPA_STATE_GSI_IF_RMSK)
-#define HWIO_IPA_STATE_GSI_IF_INM(m) in_dword_masked( \
-		HWIO_IPA_STATE_GSI_IF_ADDR, \
-		m)
-#define HWIO_IPA_STATE_GSI_IF_IPA_GSI_PROD_FSM_TX_1_BMSK 0xf0
-#define HWIO_IPA_STATE_GSI_IF_IPA_GSI_PROD_FSM_TX_1_SHFT 0x4
-#define HWIO_IPA_STATE_GSI_IF_IPA_GSI_PROD_FSM_TX_0_BMSK 0xf
-#define HWIO_IPA_STATE_GSI_IF_IPA_GSI_PROD_FSM_TX_0_SHFT 0x0
-#define HWIO_IPA_STATE_GSI_SKIP_ADDR (IPA_CFG_REG_BASE + 0x000000c4)
-#define HWIO_IPA_STATE_GSI_SKIP_PHYS (IPA_CFG_REG_BASE_PHYS + 0x000000c4)
-#define HWIO_IPA_STATE_GSI_SKIP_OFFS (IPA_CFG_REG_BASE_OFFS + 0x000000c4)
-#define HWIO_IPA_STATE_GSI_SKIP_RMSK 0x3
-#define HWIO_IPA_STATE_GSI_SKIP_ATTR 0x1
-#define HWIO_IPA_STATE_GSI_SKIP_IN in_dword_masked( \
-		HWIO_IPA_STATE_GSI_SKIP_ADDR, \
-		HWIO_IPA_STATE_GSI_SKIP_RMSK)
-#define HWIO_IPA_STATE_GSI_SKIP_INM(m) in_dword_masked(	\
-		HWIO_IPA_STATE_GSI_SKIP_ADDR, \
-		m)
-#define HWIO_IPA_STATE_GSI_SKIP_IPA_GSI_SKIP_FSM_BMSK 0x3
-#define HWIO_IPA_STATE_GSI_SKIP_IPA_GSI_SKIP_FSM_SHFT 0x0
-#define HWIO_IPA_STATE_GSI_IF_CONS_ADDR (IPA_CFG_REG_BASE + 0x000000c8)
-#define HWIO_IPA_STATE_GSI_IF_CONS_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					 0x000000c8)
-#define HWIO_IPA_STATE_GSI_IF_CONS_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					 0x000000c8)
-#define HWIO_IPA_STATE_GSI_IF_CONS_RMSK 0x7ffffff
-#define HWIO_IPA_STATE_GSI_IF_CONS_ATTR 0x1
-#define HWIO_IPA_STATE_GSI_IF_CONS_IN in_dword_masked( \
-		HWIO_IPA_STATE_GSI_IF_CONS_ADDR, \
-		HWIO_IPA_STATE_GSI_IF_CONS_RMSK)
-#define HWIO_IPA_STATE_GSI_IF_CONS_INM(m) in_dword_masked( \
-		HWIO_IPA_STATE_GSI_IF_CONS_ADDR, \
-		m)
-#define	\
-	HWIO_IPA_STATE_GSI_IF_CONS_IPA_STATE_GSI_IF_CONS_RX_REQ_NO_ZERO_BMSK \
-	0x7fe0000
-#define	\
-	HWIO_IPA_STATE_GSI_IF_CONS_IPA_STATE_GSI_IF_CONS_RX_REQ_NO_ZERO_SHFT \
-	0x11
-#define HWIO_IPA_STATE_GSI_IF_CONS_IPA_STATE_GSI_IF_CONS_RX_REQ_BMSK \
-	0x1ff80
-#define HWIO_IPA_STATE_GSI_IF_CONS_IPA_STATE_GSI_IF_CONS_RX_REQ_SHFT 0x7
-#define HWIO_IPA_STATE_GSI_IF_CONS_IPA_STATE_GSI_IF_CONS_CACHE_VLD_BMSK	\
-	0x7e
-#define HWIO_IPA_STATE_GSI_IF_CONS_IPA_STATE_GSI_IF_CONS_CACHE_VLD_SHFT	\
-	0x1
-#define HWIO_IPA_STATE_GSI_IF_CONS_IPA_STATE_GSI_IF_CONS_STATE_BMSK 0x1
-#define HWIO_IPA_STATE_GSI_IF_CONS_IPA_STATE_GSI_IF_CONS_STATE_SHFT 0x0
-#define HWIO_IPA_STATE_ADDR (IPA_CFG_REG_BASE + 0x000000a8)
-#define HWIO_IPA_STATE_PHYS (IPA_CFG_REG_BASE_PHYS + 0x000000a8)
-#define HWIO_IPA_STATE_OFFS (IPA_CFG_REG_BASE_OFFS + 0x000000a8)
-#define HWIO_IPA_STATE_RMSK 0xf7ffffff
-#define HWIO_IPA_STATE_ATTR 0x1
-#define HWIO_IPA_STATE_IN in_dword_masked(HWIO_IPA_STATE_ADDR, \
-					  HWIO_IPA_STATE_RMSK)
-#define HWIO_IPA_STATE_INM(m) in_dword_masked(HWIO_IPA_STATE_ADDR, m)
-#define HWIO_IPA_STATE_IPA_UC_RX_HND_CMDQ_EMPTY_BMSK 0x80000000
-#define HWIO_IPA_STATE_IPA_UC_RX_HND_CMDQ_EMPTY_SHFT 0x1f
-#define HWIO_IPA_STATE_IPA_DPS_TX_EMPTY_BMSK 0x40000000
-#define HWIO_IPA_STATE_IPA_DPS_TX_EMPTY_SHFT 0x1e
-#define HWIO_IPA_STATE_IPA_HPS_DPS_EMPTY_BMSK 0x20000000
-#define HWIO_IPA_STATE_IPA_HPS_DPS_EMPTY_SHFT 0x1d
-#define HWIO_IPA_STATE_IPA_RX_HPS_EMPTY_BMSK 0x10000000
-#define HWIO_IPA_STATE_IPA_RX_HPS_EMPTY_SHFT 0x1c
-#define HWIO_IPA_STATE_IPA_RX_SPLT_CMDQ_EMPTY_BMSK 0x7800000
-#define HWIO_IPA_STATE_IPA_RX_SPLT_CMDQ_EMPTY_SHFT 0x17
-#define HWIO_IPA_STATE_IPA_TX_COMMANDER_CMDQ_EMPTY_BMSK 0x400000
-#define HWIO_IPA_STATE_IPA_TX_COMMANDER_CMDQ_EMPTY_SHFT 0x16
-#define HWIO_IPA_STATE_IPA_RX_ACKQ_EMPTY_BMSK 0x200000
-#define HWIO_IPA_STATE_IPA_RX_ACKQ_EMPTY_SHFT 0x15
-#define HWIO_IPA_STATE_IPA_UC_ACKQ_EMPTY_BMSK 0x100000
-#define HWIO_IPA_STATE_IPA_UC_ACKQ_EMPTY_SHFT 0x14
-#define HWIO_IPA_STATE_IPA_TX_ACKQ_EMPTY_BMSK 0x80000
-#define HWIO_IPA_STATE_IPA_TX_ACKQ_EMPTY_SHFT 0x13
-#define HWIO_IPA_STATE_IPA_NTF_TX_EMPTY_BMSK 0x40000
-#define HWIO_IPA_STATE_IPA_NTF_TX_EMPTY_SHFT 0x12
-#define HWIO_IPA_STATE_IPA_FULL_IDLE_BMSK 0x20000
-#define HWIO_IPA_STATE_IPA_FULL_IDLE_SHFT 0x11
-#define HWIO_IPA_STATE_IPA_PROD_BRESP_IDLE_BMSK 0x10000
-#define HWIO_IPA_STATE_IPA_PROD_BRESP_IDLE_SHFT 0x10
-#define HWIO_IPA_STATE_IPA_PROD_ACKMNGR_STATE_IDLE_BMSK 0x8000
-#define HWIO_IPA_STATE_IPA_PROD_ACKMNGR_STATE_IDLE_SHFT 0xf
-#define HWIO_IPA_STATE_IPA_PROD_ACKMNGR_DB_EMPTY_BMSK 0x4000
-#define HWIO_IPA_STATE_IPA_PROD_ACKMNGR_DB_EMPTY_SHFT 0xe
-#define HWIO_IPA_STATE_IPA_TX_ACKQ_FULL_BMSK 0x2000
-#define HWIO_IPA_STATE_IPA_TX_ACKQ_FULL_SHFT 0xd
-#define HWIO_IPA_STATE_IPA_ACKMNGR_STATE_IDLE_BMSK 0x1000
-#define HWIO_IPA_STATE_IPA_ACKMNGR_STATE_IDLE_SHFT 0xc
-#define HWIO_IPA_STATE_IPA_ACKMNGR_DB_EMPTY_BMSK 0x800
-#define HWIO_IPA_STATE_IPA_ACKMNGR_DB_EMPTY_SHFT 0xb
-#define HWIO_IPA_STATE_IPA_RSRC_STATE_IDLE_BMSK 0x400
-#define HWIO_IPA_STATE_IPA_RSRC_STATE_IDLE_SHFT 0xa
-#define HWIO_IPA_STATE_IPA_RSRC_MNGR_DB_EMPTY_BMSK 0x200
-#define HWIO_IPA_STATE_IPA_RSRC_MNGR_DB_EMPTY_SHFT 0x9
-#define HWIO_IPA_STATE_MBIM_AGGR_IDLE_BMSK 0x100
-#define HWIO_IPA_STATE_MBIM_AGGR_IDLE_SHFT 0x8
-#define HWIO_IPA_STATE_AGGR_IDLE_BMSK 0x80
-#define HWIO_IPA_STATE_AGGR_IDLE_SHFT 0x7
-#define HWIO_IPA_STATE_IPA_NOC_IDLE_BMSK 0x40
-#define HWIO_IPA_STATE_IPA_NOC_IDLE_SHFT 0x6
-#define HWIO_IPA_STATE_IPA_STATUS_SNIFFER_IDLE_BMSK 0x20
-#define HWIO_IPA_STATE_IPA_STATUS_SNIFFER_IDLE_SHFT 0x5
-#define HWIO_IPA_STATE_BAM_GSI_IDLE_BMSK 0x10
-#define HWIO_IPA_STATE_BAM_GSI_IDLE_SHFT 0x4
-#define HWIO_IPA_STATE_DPL_FIFO_IDLE_BMSK 0x8
-#define HWIO_IPA_STATE_DPL_FIFO_IDLE_SHFT 0x3
-#define HWIO_IPA_STATE_TX_IDLE_BMSK 0x4
-#define HWIO_IPA_STATE_TX_IDLE_SHFT 0x2
-#define HWIO_IPA_STATE_RX_IDLE_BMSK 0x2
-#define HWIO_IPA_STATE_RX_IDLE_SHFT 0x1
-#define HWIO_IPA_STATE_RX_WAIT_BMSK 0x1
-#define HWIO_IPA_STATE_RX_WAIT_SHFT 0x0
-#define HWIO_IPA_STATE_RX_ACTIVE_ADDR (IPA_CFG_REG_BASE + 0x000000ac)
-#define HWIO_IPA_STATE_RX_ACTIVE_PHYS (IPA_CFG_REG_BASE_PHYS + 0x000000ac)
-#define HWIO_IPA_STATE_RX_ACTIVE_OFFS (IPA_CFG_REG_BASE_OFFS + 0x000000ac)
-#define HWIO_IPA_STATE_RX_ACTIVE_RMSK 0x1fff
-#define HWIO_IPA_STATE_RX_ACTIVE_ATTR 0x1
-#define HWIO_IPA_STATE_RX_ACTIVE_IN in_dword_masked( \
-		HWIO_IPA_STATE_RX_ACTIVE_ADDR, \
-		HWIO_IPA_STATE_RX_ACTIVE_RMSK)
-#define HWIO_IPA_STATE_RX_ACTIVE_INM(m) in_dword_masked( \
-		HWIO_IPA_STATE_RX_ACTIVE_ADDR, \
-		m)
-#define HWIO_IPA_STATE_RX_ACTIVE_ENDPOINTS_BMSK 0x1fff
-#define HWIO_IPA_STATE_RX_ACTIVE_ENDPOINTS_SHFT 0x0
-#define HWIO_IPA_STATE_TX0_ADDR (IPA_CFG_REG_BASE + 0x000000b0)
-#define HWIO_IPA_STATE_TX0_PHYS (IPA_CFG_REG_BASE_PHYS + 0x000000b0)
-#define HWIO_IPA_STATE_TX0_OFFS (IPA_CFG_REG_BASE_OFFS + 0x000000b0)
-#define HWIO_IPA_STATE_TX0_RMSK 0xfffffff
-#define HWIO_IPA_STATE_TX0_ATTR 0x1
-#define HWIO_IPA_STATE_TX0_IN in_dword_masked(HWIO_IPA_STATE_TX0_ADDR, \
-					      HWIO_IPA_STATE_TX0_RMSK)
-#define HWIO_IPA_STATE_TX0_INM(m) in_dword_masked(HWIO_IPA_STATE_TX0_ADDR, \
-						  m)
-#define HWIO_IPA_STATE_TX0_LAST_CMD_PIPE_BMSK 0xf800000
-#define HWIO_IPA_STATE_TX0_LAST_CMD_PIPE_SHFT 0x17
-#define HWIO_IPA_STATE_TX0_CS_SNIF_IDLE_BMSK 0x400000
-#define HWIO_IPA_STATE_TX0_CS_SNIF_IDLE_SHFT 0x16
-#define HWIO_IPA_STATE_TX0_SUSPEND_EMPTY_BMSK 0x200000
-#define HWIO_IPA_STATE_TX0_SUSPEND_EMPTY_SHFT 0x15
-#define HWIO_IPA_STATE_TX0_RSRCREL_IDLE_BMSK 0x100000
-#define HWIO_IPA_STATE_TX0_RSRCREL_IDLE_SHFT 0x14
-#define HWIO_IPA_STATE_TX0_HOLB_MASK_IDLE_BMSK 0x80000
-#define HWIO_IPA_STATE_TX0_HOLB_MASK_IDLE_SHFT 0x13
-#define HWIO_IPA_STATE_TX0_HOLB_IDLE_BMSK 0x40000
-#define HWIO_IPA_STATE_TX0_HOLB_IDLE_SHFT 0x12
-#define HWIO_IPA_STATE_TX0_ALIGNER_EMPTY_BMSK 0x20000
-#define HWIO_IPA_STATE_TX0_ALIGNER_EMPTY_SHFT 0x11
-#define HWIO_IPA_STATE_TX0_PF_EMPTY_BMSK 0x10000
-#define HWIO_IPA_STATE_TX0_PF_EMPTY_SHFT 0x10
-#define HWIO_IPA_STATE_TX0_PF_IDLE_BMSK 0x8000
-#define HWIO_IPA_STATE_TX0_PF_IDLE_SHFT 0xf
-#define HWIO_IPA_STATE_TX0_DMAW_LAST_OUTSD_IDLE_BMSK 0x4000
-#define HWIO_IPA_STATE_TX0_DMAW_LAST_OUTSD_IDLE_SHFT 0xe
-#define HWIO_IPA_STATE_TX0_DMAW_IDLE_BMSK 0x2000
-#define HWIO_IPA_STATE_TX0_DMAW_IDLE_SHFT 0xd
-#define HWIO_IPA_STATE_TX0_AR_IDLE_BMSK 0x1000
-#define HWIO_IPA_STATE_TX0_AR_IDLE_SHFT 0xc
-#define HWIO_IPA_STATE_TX0_TX_CMD_BRESP_INJ_IDLE_BMSK 0x800
-#define HWIO_IPA_STATE_TX0_TX_CMD_BRESP_INJ_IDLE_SHFT 0xb
-#define HWIO_IPA_STATE_TX0_TX_CMD_BRESP_ALOC_IDLE_BMSK 0x400
-#define HWIO_IPA_STATE_TX0_TX_CMD_BRESP_ALOC_IDLE_SHFT 0xa
-#define HWIO_IPA_STATE_TX0_TX_CMD_SNIF_IDLE_BMSK 0x200
-#define HWIO_IPA_STATE_TX0_TX_CMD_SNIF_IDLE_SHFT 0x9
-#define HWIO_IPA_STATE_TX0_TX_CMD_TRNSEQ_IDLE_BMSK 0x100
-#define HWIO_IPA_STATE_TX0_TX_CMD_TRNSEQ_IDLE_SHFT 0x8
-#define HWIO_IPA_STATE_TX0_TX_CMD_MAIN_IDLE_BMSK 0x80
-#define HWIO_IPA_STATE_TX0_TX_CMD_MAIN_IDLE_SHFT 0x7
-#define HWIO_IPA_STATE_TX0_PA_PUB_CNT_EMPTY_BMSK 0x40
-#define HWIO_IPA_STATE_TX0_PA_PUB_CNT_EMPTY_SHFT 0x6
-#define HWIO_IPA_STATE_TX0_PA_CTX_IDLE_BMSK 0x20
-#define HWIO_IPA_STATE_TX0_PA_CTX_IDLE_SHFT 0x5
-#define HWIO_IPA_STATE_TX0_PA_IDLE_BMSK 0x10
-#define HWIO_IPA_STATE_TX0_PA_IDLE_SHFT 0x4
-#define HWIO_IPA_STATE_TX0_NEXT_ARBIT_TYPE_BMSK 0xc
-#define HWIO_IPA_STATE_TX0_NEXT_ARBIT_TYPE_SHFT 0x2
-#define HWIO_IPA_STATE_TX0_LAST_ARBIT_TYPE_BMSK 0x3
-#define HWIO_IPA_STATE_TX0_LAST_ARBIT_TYPE_SHFT 0x0
-#define HWIO_IPA_STATE_AGGR_ACTIVE_ADDR (IPA_CFG_REG_BASE + 0x000000b4)
-#define HWIO_IPA_STATE_AGGR_ACTIVE_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					 0x000000b4)
-#define HWIO_IPA_STATE_AGGR_ACTIVE_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					 0x000000b4)
-#define HWIO_IPA_STATE_AGGR_ACTIVE_RMSK 0x7fffffff
-#define HWIO_IPA_STATE_AGGR_ACTIVE_ATTR 0x1
-#define HWIO_IPA_STATE_AGGR_ACTIVE_IN in_dword_masked( \
-		HWIO_IPA_STATE_AGGR_ACTIVE_ADDR, \
-		HWIO_IPA_STATE_AGGR_ACTIVE_RMSK)
-#define HWIO_IPA_STATE_AGGR_ACTIVE_INM(m) in_dword_masked( \
-		HWIO_IPA_STATE_AGGR_ACTIVE_ADDR, \
-		m)
-#define HWIO_IPA_STATE_AGGR_ACTIVE_ENDPOINTS_BMSK 0x7fffffff
-#define HWIO_IPA_STATE_AGGR_ACTIVE_ENDPOINTS_SHFT 0x0
-#define HWIO_IPA_GENERIC_RAM_ARBITER_PRIORITY_ADDR (IPA_CFG_REG_BASE + \
-						    0x000000d8)
-#define HWIO_IPA_GENERIC_RAM_ARBITER_PRIORITY_PHYS (IPA_CFG_REG_BASE_PHYS \
-						    + 0x000000d8)
-#define HWIO_IPA_GENERIC_RAM_ARBITER_PRIORITY_OFFS (IPA_CFG_REG_BASE_OFFS \
-						    + 0x000000d8)
-#define HWIO_IPA_STATE_NLO_AGGR_ADDR (IPA_CFG_REG_BASE + 0x000000dc)
-#define HWIO_IPA_STATE_NLO_AGGR_PHYS (IPA_CFG_REG_BASE_PHYS + 0x000000dc)
-#define HWIO_IPA_STATE_NLO_AGGR_OFFS (IPA_CFG_REG_BASE_OFFS + 0x000000dc)
-#define HWIO_IPA_STATE_NLO_AGGR_RMSK 0xffffffff
-#define HWIO_IPA_STATE_NLO_AGGR_ATTR 0x1
-#define HWIO_IPA_STATE_NLO_AGGR_IN in_dword_masked( \
-		HWIO_IPA_STATE_NLO_AGGR_ADDR, \
-		HWIO_IPA_STATE_NLO_AGGR_RMSK)
-#define HWIO_IPA_STATE_NLO_AGGR_INM(m) in_dword_masked(	\
-		HWIO_IPA_STATE_NLO_AGGR_ADDR, \
-		m)
-#define HWIO_IPA_STATE_NLO_AGGR_NLO_AGGR_STATE_BMSK 0xffffffff
-#define HWIO_IPA_STATE_NLO_AGGR_NLO_AGGR_STATE_SHFT 0x0
-#define HWIO_IPA_STATE_COAL_MASTER_1_ADDR (IPA_CFG_REG_BASE + 0x000000e0)
-#define HWIO_IPA_STATE_COAL_MASTER_1_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					   0x000000e0)
-#define HWIO_IPA_STATE_COAL_MASTER_1_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					   0x000000e0)
-#define HWIO_IPA_STATE_COAL_MASTER_1_RMSK 0x3fffffff
-#define HWIO_IPA_STATE_COAL_MASTER_1_ATTR 0x1
-#define HWIO_IPA_STATE_COAL_MASTER_1_IN in_dword_masked( \
-		HWIO_IPA_STATE_COAL_MASTER_1_ADDR, \
-		HWIO_IPA_STATE_COAL_MASTER_1_RMSK)
-#define HWIO_IPA_STATE_COAL_MASTER_1_INM(m) in_dword_masked( \
-		HWIO_IPA_STATE_COAL_MASTER_1_ADDR, \
-		m)
-#define HWIO_IPA_STATE_COAL_MASTER_1_ARBITER_STATE_BMSK 0x3c000000
-#define HWIO_IPA_STATE_COAL_MASTER_1_ARBITER_STATE_SHFT 0x1a
-#define HWIO_IPA_STATE_COAL_MASTER_1_CHECK_FIT_FSM_STATE_BMSK 0x3c00000
-#define HWIO_IPA_STATE_COAL_MASTER_1_CHECK_FIT_FSM_STATE_SHFT 0x16
-#define HWIO_IPA_STATE_COAL_MASTER_1_CHECK_FIT_RD_CTX_LINE_BMSK 0x3f0000
-#define HWIO_IPA_STATE_COAL_MASTER_1_CHECK_FIT_RD_CTX_LINE_SHFT 0x10
-#define HWIO_IPA_STATE_COAL_MASTER_1_INIT_VP_FSM_STATE_BMSK 0xf000
-#define HWIO_IPA_STATE_COAL_MASTER_1_INIT_VP_FSM_STATE_SHFT 0xc
-#define HWIO_IPA_STATE_COAL_MASTER_1_INIT_VP_RD_PKT_LINE_BMSK 0xfc0
-#define HWIO_IPA_STATE_COAL_MASTER_1_INIT_VP_RD_PKT_LINE_SHFT 0x6
-#define HWIO_IPA_STATE_COAL_MASTER_1_INIT_VP_WR_CTX_LINE_BMSK 0x3f
-#define HWIO_IPA_STATE_COAL_MASTER_1_INIT_VP_WR_CTX_LINE_SHFT 0x0
-#define HWIO_IPA_YELLOW_MARKER_BELOW_ADDR (IPA_CFG_REG_BASE + 0x00000110)
-#define HWIO_IPA_YELLOW_MARKER_BELOW_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					   0x00000110)
-#define HWIO_IPA_YELLOW_MARKER_BELOW_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					   0x00000110)
-#define HWIO_IPA_YELLOW_MARKER_BELOW_EN_ADDR (IPA_CFG_REG_BASE + \
-					      0x00000114)
-#define HWIO_IPA_YELLOW_MARKER_BELOW_EN_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					      0x00000114)
-#define HWIO_IPA_YELLOW_MARKER_BELOW_EN_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					      0x00000114)
-#define HWIO_IPA_YELLOW_MARKER_BELOW_CLR_ADDR (IPA_CFG_REG_BASE + \
-					       0x00000118)
-#define HWIO_IPA_YELLOW_MARKER_BELOW_CLR_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					       0x00000118)
-#define HWIO_IPA_YELLOW_MARKER_BELOW_CLR_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					       0x00000118)
-#define HWIO_IPA_RED_MARKER_BELOW_ADDR (IPA_CFG_REG_BASE + 0x0000011c)
-#define HWIO_IPA_RED_MARKER_BELOW_PHYS (IPA_CFG_REG_BASE_PHYS + 0x0000011c)
-#define HWIO_IPA_RED_MARKER_BELOW_OFFS (IPA_CFG_REG_BASE_OFFS + 0x0000011c)
-#define HWIO_IPA_RED_MARKER_BELOW_EN_ADDR (IPA_CFG_REG_BASE + 0x00000120)
-#define HWIO_IPA_RED_MARKER_BELOW_EN_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					   0x00000120)
-#define HWIO_IPA_RED_MARKER_BELOW_EN_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					   0x00000120)
-#define HWIO_IPA_RED_MARKER_BELOW_CLR_ADDR (IPA_CFG_REG_BASE + 0x00000124)
-#define HWIO_IPA_RED_MARKER_BELOW_CLR_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					    0x00000124)
-#define HWIO_IPA_RED_MARKER_BELOW_CLR_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					    0x00000124)
-#define HWIO_IPA_YELLOW_MARKER_SHADOW_ADDR (IPA_CFG_REG_BASE + 0x00000128)
-#define HWIO_IPA_YELLOW_MARKER_SHADOW_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					    0x00000128)
-#define HWIO_IPA_YELLOW_MARKER_SHADOW_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					    0x00000128)
-#define HWIO_IPA_RED_MARKER_SHADOW_ADDR (IPA_CFG_REG_BASE + 0x0000012c)
-#define HWIO_IPA_RED_MARKER_SHADOW_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					 0x0000012c)
-#define HWIO_IPA_RED_MARKER_SHADOW_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					 0x0000012c)
-#define HWIO_IPA_YELLOW_MARKER_ABOVE_ADDR (IPA_CFG_REG_BASE + 0x00000130)
-#define HWIO_IPA_YELLOW_MARKER_ABOVE_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					   0x00000130)
-#define HWIO_IPA_YELLOW_MARKER_ABOVE_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					   0x00000130)
-#define HWIO_IPA_YELLOW_MARKER_ABOVE_EN_ADDR (IPA_CFG_REG_BASE + \
-					      0x00000134)
-#define HWIO_IPA_YELLOW_MARKER_ABOVE_EN_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					      0x00000134)
-#define HWIO_IPA_YELLOW_MARKER_ABOVE_EN_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					      0x00000134)
-#define HWIO_IPA_YELLOW_MARKER_ABOVE_CLR_ADDR (IPA_CFG_REG_BASE + \
-					       0x00000138)
-#define HWIO_IPA_YELLOW_MARKER_ABOVE_CLR_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					       0x00000138)
-#define HWIO_IPA_YELLOW_MARKER_ABOVE_CLR_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					       0x00000138)
-#define HWIO_IPA_RED_MARKER_ABOVE_ADDR (IPA_CFG_REG_BASE + 0x0000013c)
-#define HWIO_IPA_RED_MARKER_ABOVE_PHYS (IPA_CFG_REG_BASE_PHYS + 0x0000013c)
-#define HWIO_IPA_RED_MARKER_ABOVE_OFFS (IPA_CFG_REG_BASE_OFFS + 0x0000013c)
-#define HWIO_IPA_RED_MARKER_ABOVE_EN_ADDR (IPA_CFG_REG_BASE + 0x00000140)
-#define HWIO_IPA_RED_MARKER_ABOVE_EN_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					   0x00000140)
-#define HWIO_IPA_RED_MARKER_ABOVE_EN_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					   0x00000140)
-#define HWIO_IPA_RED_MARKER_ABOVE_CLR_ADDR (IPA_CFG_REG_BASE + 0x00000144)
-#define HWIO_IPA_RED_MARKER_ABOVE_CLR_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					    0x00000144)
-#define HWIO_IPA_RED_MARKER_ABOVE_CLR_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					    0x00000144)
-#define HWIO_IPA_FILT_ROUT_HASH_EN_ADDR (IPA_CFG_REG_BASE + 0x00000148)
-#define HWIO_IPA_FILT_ROUT_HASH_EN_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					 0x00000148)
-#define HWIO_IPA_FILT_ROUT_HASH_EN_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					 0x00000148)
-#define HWIO_IPA_FILT_ROUT_HASH_EN_RMSK 0x1111
-#define HWIO_IPA_FILT_ROUT_HASH_EN_ATTR 0x3
-#define HWIO_IPA_FILT_ROUT_HASH_EN_IN in_dword_masked( \
-		HWIO_IPA_FILT_ROUT_HASH_EN_ADDR, \
-		HWIO_IPA_FILT_ROUT_HASH_EN_RMSK)
-#define HWIO_IPA_FILT_ROUT_HASH_EN_INM(m) in_dword_masked( \
-		HWIO_IPA_FILT_ROUT_HASH_EN_ADDR, \
-		m)
-#define HWIO_IPA_FILT_ROUT_HASH_EN_OUT(v) out_dword( \
-		HWIO_IPA_FILT_ROUT_HASH_EN_ADDR, \
-		v)
-#define HWIO_IPA_FILT_ROUT_HASH_EN_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_FILT_ROUT_HASH_EN_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_FILT_ROUT_HASH_EN_IN)
-#define HWIO_IPA_FILT_ROUT_HASH_EN_IPV4_FILTER_HASH_EN_BMSK 0x1000
-#define HWIO_IPA_FILT_ROUT_HASH_EN_IPV4_FILTER_HASH_EN_SHFT 0xc
-#define HWIO_IPA_FILT_ROUT_HASH_EN_IPV4_ROUTER_HASH_EN_BMSK 0x100
-#define HWIO_IPA_FILT_ROUT_HASH_EN_IPV4_ROUTER_HASH_EN_SHFT 0x8
-#define HWIO_IPA_FILT_ROUT_HASH_EN_IPV6_FILTER_HASH_EN_BMSK 0x10
-#define HWIO_IPA_FILT_ROUT_HASH_EN_IPV6_FILTER_HASH_EN_SHFT 0x4
-#define HWIO_IPA_FILT_ROUT_HASH_EN_IPV6_ROUTER_HASH_EN_BMSK 0x1
-#define HWIO_IPA_FILT_ROUT_HASH_EN_IPV6_ROUTER_HASH_EN_SHFT 0x0
-#define HWIO_IPA_FILT_ROUT_HASH_FLUSH_ADDR (IPA_CFG_REG_BASE + 0x0000014c)
-#define HWIO_IPA_FILT_ROUT_HASH_FLUSH_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					    0x0000014c)
-#define HWIO_IPA_FILT_ROUT_HASH_FLUSH_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					    0x0000014c)
-#define HWIO_IPA_FILT_ROUT_HASH_FLUSH_RMSK 0x1111
-#define HWIO_IPA_FILT_ROUT_HASH_FLUSH_ATTR 0x2
-#define HWIO_IPA_FILT_ROUT_HASH_FLUSH_OUT(v) out_dword(	\
-		HWIO_IPA_FILT_ROUT_HASH_FLUSH_ADDR, \
-		v)
-#define HWIO_IPA_FILT_ROUT_HASH_FLUSH_IPV4_FILTER_HASH_FLUSH_BMSK 0x1000
-#define HWIO_IPA_FILT_ROUT_HASH_FLUSH_IPV4_FILTER_HASH_FLUSH_SHFT 0xc
-#define HWIO_IPA_FILT_ROUT_HASH_FLUSH_IPV4_ROUTER_HASH_FLUSH_BMSK 0x100
-#define HWIO_IPA_FILT_ROUT_HASH_FLUSH_IPV4_ROUTER_HASH_FLUSH_SHFT 0x8
-#define HWIO_IPA_FILT_ROUT_HASH_FLUSH_IPV6_FILTER_HASH_FLUSH_BMSK 0x10
-#define HWIO_IPA_FILT_ROUT_HASH_FLUSH_IPV6_FILTER_HASH_FLUSH_SHFT 0x4
-#define HWIO_IPA_FILT_ROUT_HASH_FLUSH_IPV6_ROUTER_HASH_FLUSH_BMSK 0x1
-#define HWIO_IPA_FILT_ROUT_HASH_FLUSH_IPV6_ROUTER_HASH_FLUSH_SHFT 0x0
-#define HWIO_IPA_FILT_ROUT_CFG_ADDR (IPA_CFG_REG_BASE + 0x00000150)
-#define HWIO_IPA_FILT_ROUT_CFG_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00000150)
-#define HWIO_IPA_FILT_ROUT_CFG_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00000150)
-#define HWIO_IPA_IPV4_FILTER_INIT_VALUES_ADDR (IPA_CFG_REG_BASE + \
-					       0x00000160)
-#define HWIO_IPA_IPV4_FILTER_INIT_VALUES_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					       0x00000160)
-#define HWIO_IPA_IPV4_FILTER_INIT_VALUES_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					       0x00000160)
-#define HWIO_IPA_IPV4_FILTER_INIT_VALUES_RMSK 0xffffffff
-#define HWIO_IPA_IPV4_FILTER_INIT_VALUES_ATTR 0x1
-#define HWIO_IPA_IPV4_FILTER_INIT_VALUES_IN in_dword_masked( \
-		HWIO_IPA_IPV4_FILTER_INIT_VALUES_ADDR, \
-		HWIO_IPA_IPV4_FILTER_INIT_VALUES_RMSK)
-#define HWIO_IPA_IPV4_FILTER_INIT_VALUES_INM(m) in_dword_masked( \
-		HWIO_IPA_IPV4_FILTER_INIT_VALUES_ADDR, \
-		m)
-#define	\
-	HWIO_IPA_IPV4_FILTER_INIT_VALUES_IP_V4_FILTER_INIT_HASHED_ADDR_BMSK \
-	0xffff
-#define	\
-	HWIO_IPA_IPV4_FILTER_INIT_VALUES_IP_V4_FILTER_INIT_HASHED_ADDR_SHFT \
-	0x0
-#define HWIO_IPA_IPV6_FILTER_INIT_VALUES_ADDR (IPA_CFG_REG_BASE + \
-					       0x00000164)
-#define HWIO_IPA_IPV6_FILTER_INIT_VALUES_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					       0x00000164)
-#define HWIO_IPA_IPV6_FILTER_INIT_VALUES_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					       0x00000164)
-#define HWIO_IPA_IPV6_FILTER_INIT_VALUES_RMSK 0xffffffff
-#define HWIO_IPA_IPV6_FILTER_INIT_VALUES_ATTR 0x1
-#define HWIO_IPA_IPV6_FILTER_INIT_VALUES_IN in_dword_masked( \
-		HWIO_IPA_IPV6_FILTER_INIT_VALUES_ADDR, \
-		HWIO_IPA_IPV6_FILTER_INIT_VALUES_RMSK)
-#define HWIO_IPA_IPV6_FILTER_INIT_VALUES_INM(m) in_dword_masked( \
-		HWIO_IPA_IPV6_FILTER_INIT_VALUES_ADDR, \
-		m)
-#define	\
-	HWIO_IPA_IPV6_FILTER_INIT_VALUES_IP_V6_FILTER_INIT_HASHED_ADDR_BMSK \
-	0xffff
-#define	\
-	HWIO_IPA_IPV6_FILTER_INIT_VALUES_IP_V6_FILTER_INIT_HASHED_ADDR_SHFT \
-	0x0
-#define HWIO_IPA_IPV4_NAT_INIT_VALUES_0_ADDR (IPA_CFG_REG_BASE + \
-					      0x00000178)
-#define HWIO_IPA_IPV4_NAT_INIT_VALUES_0_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					      0x00000178)
-#define HWIO_IPA_IPV4_NAT_INIT_VALUES_0_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					      0x00000178)
-#define HWIO_IPA_IPV4_NAT_INIT_VALUES_0_MSB_ADDR (IPA_CFG_REG_BASE + \
-						  0x0000017c)
-#define HWIO_IPA_IPV4_NAT_INIT_VALUES_0_MSB_PHYS (IPA_CFG_REG_BASE_PHYS + \
-						  0x0000017c)
-#define HWIO_IPA_IPV4_NAT_INIT_VALUES_0_MSB_OFFS (IPA_CFG_REG_BASE_OFFS + \
-						  0x0000017c)
-#define HWIO_IPA_IPV4_NAT_INIT_VALUES_1_ADDR (IPA_CFG_REG_BASE + \
-					      0x00000180)
-#define HWIO_IPA_IPV4_NAT_INIT_VALUES_1_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					      0x00000180)
-#define HWIO_IPA_IPV4_NAT_INIT_VALUES_1_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					      0x00000180)
-#define HWIO_IPA_IPV4_NAT_INIT_VALUES_1_MSB_ADDR (IPA_CFG_REG_BASE + \
-						  0x00000184)
-#define HWIO_IPA_IPV4_NAT_INIT_VALUES_1_MSB_PHYS (IPA_CFG_REG_BASE_PHYS + \
-						  0x00000184)
-#define HWIO_IPA_IPV4_NAT_INIT_VALUES_1_MSB_OFFS (IPA_CFG_REG_BASE_OFFS + \
-						  0x00000184)
-#define HWIO_IPA_IPV4_NAT_INIT_VALUES_2_ADDR (IPA_CFG_REG_BASE + \
-					      0x00000188)
-#define HWIO_IPA_IPV4_NAT_INIT_VALUES_2_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					      0x00000188)
-#define HWIO_IPA_IPV4_NAT_INIT_VALUES_2_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					      0x00000188)
-#define HWIO_IPA_IPV4_NAT_INIT_VALUES_2_MSB_ADDR (IPA_CFG_REG_BASE + \
-						  0x0000018c)
-#define HWIO_IPA_IPV4_NAT_INIT_VALUES_2_MSB_PHYS (IPA_CFG_REG_BASE_PHYS + \
-						  0x0000018c)
-#define HWIO_IPA_IPV4_NAT_INIT_VALUES_2_MSB_OFFS (IPA_CFG_REG_BASE_OFFS + \
-						  0x0000018c)
-#define HWIO_IPA_IPV4_NAT_INIT_VALUES_3_ADDR (IPA_CFG_REG_BASE + \
-					      0x00000190)
-#define HWIO_IPA_IPV4_NAT_INIT_VALUES_3_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					      0x00000190)
-#define HWIO_IPA_IPV4_NAT_INIT_VALUES_3_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					      0x00000190)
-#define HWIO_IPA_IPV4_NAT_INIT_VALUES_3_MSB_ADDR (IPA_CFG_REG_BASE + \
-						  0x00000194)
-#define HWIO_IPA_IPV4_NAT_INIT_VALUES_3_MSB_PHYS (IPA_CFG_REG_BASE_PHYS + \
-						  0x00000194)
-#define HWIO_IPA_IPV4_NAT_INIT_VALUES_3_MSB_OFFS (IPA_CFG_REG_BASE_OFFS + \
-						  0x00000194)
-#define HWIO_IPA_IPV4_NAT_INIT_VALUES_4_ADDR (IPA_CFG_REG_BASE + \
-					      0x00000198)
-#define HWIO_IPA_IPV4_NAT_INIT_VALUES_4_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					      0x00000198)
-#define HWIO_IPA_IPV4_NAT_INIT_VALUES_4_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					      0x00000198)
-#define HWIO_IPA_IPV4_NAT_INIT_VALUES_5_ADDR (IPA_CFG_REG_BASE + \
-					      0x0000019c)
-#define HWIO_IPA_IPV4_NAT_INIT_VALUES_5_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					      0x0000019c)
-#define HWIO_IPA_IPV4_NAT_INIT_VALUES_5_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					      0x0000019c)
-#define HWIO_IPA_IPV4_ROUTE_INIT_VALUES_ADDR (IPA_CFG_REG_BASE + \
-					      0x000001a0)
-#define HWIO_IPA_IPV4_ROUTE_INIT_VALUES_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					      0x000001a0)
-#define HWIO_IPA_IPV4_ROUTE_INIT_VALUES_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					      0x000001a0)
-#define HWIO_IPA_IPV4_ROUTE_INIT_VALUES_RMSK 0xffffffff
-#define HWIO_IPA_IPV4_ROUTE_INIT_VALUES_ATTR 0x1
-#define HWIO_IPA_IPV4_ROUTE_INIT_VALUES_IN in_dword_masked( \
-		HWIO_IPA_IPV4_ROUTE_INIT_VALUES_ADDR, \
-		HWIO_IPA_IPV4_ROUTE_INIT_VALUES_RMSK)
-#define HWIO_IPA_IPV4_ROUTE_INIT_VALUES_INM(m) in_dword_masked(	\
-		HWIO_IPA_IPV4_ROUTE_INIT_VALUES_ADDR, \
-		m)
-#define	\
-	HWIO_IPA_IPV4_ROUTE_INIT_VALUES_IP_V4_ROUTE_INIT_NON_HASHED_ADDR_BMSK \
-	0xffff0000
-#define	\
-	HWIO_IPA_IPV4_ROUTE_INIT_VALUES_IP_V4_ROUTE_INIT_NON_HASHED_ADDR_SHFT \
-	0x10
-#define HWIO_IPA_IPV4_ROUTE_INIT_VALUES_IP_V4_ROUTE_INIT_HASHED_ADDR_BMSK \
-	0xffff
-#define HWIO_IPA_IPV4_ROUTE_INIT_VALUES_IP_V4_ROUTE_INIT_HASHED_ADDR_SHFT \
-	0x0
-#define HWIO_IPA_IPV6_ROUTE_INIT_VALUES_ADDR (IPA_CFG_REG_BASE + \
-					      0x000001a4)
-#define HWIO_IPA_IPV6_ROUTE_INIT_VALUES_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					      0x000001a4)
-#define HWIO_IPA_IPV6_ROUTE_INIT_VALUES_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					      0x000001a4)
-#define HWIO_IPA_IPV6_ROUTE_INIT_VALUES_RMSK 0xffffffff
-#define HWIO_IPA_IPV6_ROUTE_INIT_VALUES_ATTR 0x1
-#define HWIO_IPA_IPV6_ROUTE_INIT_VALUES_IN in_dword_masked( \
-		HWIO_IPA_IPV6_ROUTE_INIT_VALUES_ADDR, \
-		HWIO_IPA_IPV6_ROUTE_INIT_VALUES_RMSK)
-#define HWIO_IPA_IPV6_ROUTE_INIT_VALUES_INM(m) in_dword_masked(	\
-		HWIO_IPA_IPV6_ROUTE_INIT_VALUES_ADDR, \
-		m)
-#define	\
-	HWIO_IPA_IPV6_ROUTE_INIT_VALUES_IP_V6_ROUTE_INIT_NON_HASHED_ADDR_BMSK \
-	0xffff0000
-#define	\
-	HWIO_IPA_IPV6_ROUTE_INIT_VALUES_IP_V6_ROUTE_INIT_NON_HASHED_ADDR_SHFT \
-	0x10
-#define HWIO_IPA_IPV6_ROUTE_INIT_VALUES_IP_V6_ROUTE_INIT_HASHED_ADDR_BMSK \
-	0xffff
-#define HWIO_IPA_IPV6_ROUTE_INIT_VALUES_IP_V6_ROUTE_INIT_HASHED_ADDR_SHFT \
-	0x0
-#define HWIO_IPA_IPV6_CONN_TRACK_INIT_VALUES_0_ADDR (IPA_CFG_REG_BASE +	\
-						     0x000001a8)
-#define HWIO_IPA_IPV6_CONN_TRACK_INIT_VALUES_0_PHYS (IPA_CFG_REG_BASE_PHYS \
-						     + 0x000001a8)
-#define HWIO_IPA_IPV6_CONN_TRACK_INIT_VALUES_0_OFFS (IPA_CFG_REG_BASE_OFFS \
-						     + 0x000001a8)
-#define HWIO_IPA_IPV6_CONN_TRACK_INIT_VALUES_0_MSB_ADDR (IPA_CFG_REG_BASE \
-							 + 0x000001ac)
-#define HWIO_IPA_IPV6_CONN_TRACK_INIT_VALUES_0_MSB_PHYS ( \
-		IPA_CFG_REG_BASE_PHYS + 0x000001ac)
-#define HWIO_IPA_IPV6_CONN_TRACK_INIT_VALUES_0_MSB_OFFS ( \
-		IPA_CFG_REG_BASE_OFFS + 0x000001ac)
-#define HWIO_IPA_IPV6_CONN_TRACK_INIT_VALUES_1_ADDR (IPA_CFG_REG_BASE +	\
-						     0x000001b0)
-#define HWIO_IPA_IPV6_CONN_TRACK_INIT_VALUES_1_PHYS (IPA_CFG_REG_BASE_PHYS \
-						     + 0x000001b0)
-#define HWIO_IPA_IPV6_CONN_TRACK_INIT_VALUES_1_OFFS (IPA_CFG_REG_BASE_OFFS \
-						     + 0x000001b0)
-#define HWIO_IPA_IPV6_CONN_TRACK_INIT_VALUES_1_MSB_ADDR (IPA_CFG_REG_BASE \
-							 + 0x000001b4)
-#define HWIO_IPA_IPV6_CONN_TRACK_INIT_VALUES_1_MSB_PHYS ( \
-		IPA_CFG_REG_BASE_PHYS + 0x000001b4)
-#define HWIO_IPA_IPV6_CONN_TRACK_INIT_VALUES_1_MSB_OFFS ( \
-		IPA_CFG_REG_BASE_OFFS + 0x000001b4)
-#define HWIO_IPA_IPV6_CONN_TRACK_INIT_VALUES_2_ADDR (IPA_CFG_REG_BASE +	\
-						     0x000001b8)
-#define HWIO_IPA_IPV6_CONN_TRACK_INIT_VALUES_2_PHYS (IPA_CFG_REG_BASE_PHYS \
-						     + 0x000001b8)
-#define HWIO_IPA_IPV6_CONN_TRACK_INIT_VALUES_2_OFFS (IPA_CFG_REG_BASE_OFFS \
-						     + 0x000001b8)
-#define HWIO_IPA_HDR_INIT_LOCAL_VALUES_ADDR (IPA_CFG_REG_BASE + 0x000001c0)
-#define HWIO_IPA_HDR_INIT_LOCAL_VALUES_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					     0x000001c0)
-#define HWIO_IPA_HDR_INIT_LOCAL_VALUES_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					     0x000001c0)
-#define HWIO_IPA_HDR_INIT_SYSTEM_VALUES_ADDR (IPA_CFG_REG_BASE + \
-					      0x000001c4)
-#define HWIO_IPA_HDR_INIT_SYSTEM_VALUES_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					      0x000001c4)
-#define HWIO_IPA_HDR_INIT_SYSTEM_VALUES_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					      0x000001c4)
-#define HWIO_IPA_HDR_INIT_SYSTEM_VALUES_MSB_ADDR (IPA_CFG_REG_BASE + \
-						  0x000001c8)
-#define HWIO_IPA_HDR_INIT_SYSTEM_VALUES_MSB_PHYS (IPA_CFG_REG_BASE_PHYS + \
-						  0x000001c8)
-#define HWIO_IPA_HDR_INIT_SYSTEM_VALUES_MSB_OFFS (IPA_CFG_REG_BASE_OFFS + \
-						  0x000001c8)
-#define HWIO_IPA_IMM_CMD_ACCESS_PIPE_VALUES_ADDR (IPA_CFG_REG_BASE + \
-						  0x000001cc)
-#define HWIO_IPA_IMM_CMD_ACCESS_PIPE_VALUES_PHYS (IPA_CFG_REG_BASE_PHYS + \
-						  0x000001cc)
-#define HWIO_IPA_IMM_CMD_ACCESS_PIPE_VALUES_OFFS (IPA_CFG_REG_BASE_OFFS + \
-						  0x000001cc)
-#define HWIO_IPA_FRAG_VALUES_ADDR (IPA_CFG_REG_BASE + 0x000001d8)
-#define HWIO_IPA_FRAG_VALUES_PHYS (IPA_CFG_REG_BASE_PHYS + 0x000001d8)
-#define HWIO_IPA_FRAG_VALUES_OFFS (IPA_CFG_REG_BASE_OFFS + 0x000001d8)
-#define HWIO_IPA_BAM_ACTIVATED_PORTS_ADDR (IPA_CFG_REG_BASE + 0x000001dc)
-#define HWIO_IPA_BAM_ACTIVATED_PORTS_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					   0x000001dc)
-#define HWIO_IPA_BAM_ACTIVATED_PORTS_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					   0x000001dc)
-#define HWIO_IPA_BAM_ACTIVATED_PORTS_RMSK 0x7fffffff
-#define HWIO_IPA_BAM_ACTIVATED_PORTS_ATTR 0x1
-#define HWIO_IPA_BAM_ACTIVATED_PORTS_IN in_dword_masked( \
-		HWIO_IPA_BAM_ACTIVATED_PORTS_ADDR, \
-		HWIO_IPA_BAM_ACTIVATED_PORTS_RMSK)
-#define HWIO_IPA_BAM_ACTIVATED_PORTS_INM(m) in_dword_masked( \
-		HWIO_IPA_BAM_ACTIVATED_PORTS_ADDR, \
-		m)
-#define HWIO_IPA_BAM_ACTIVATED_PORTS_ENDPOINTS_BMSK 0x7fffffff
-#define HWIO_IPA_BAM_ACTIVATED_PORTS_ENDPOINTS_SHFT 0x0
-#define HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_ADDR (IPA_CFG_REG_BASE + \
-					       0x000001e0)
-#define HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					       0x000001e0)
-#define HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					       0x000001e0)
-#define HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_RMSK 0xffffffff
-#define HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_ATTR 0x3
-#define HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_IN in_dword_masked( \
-		HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_ADDR, \
-		HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_RMSK)
-#define HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_INM(m) in_dword_masked( \
-		HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_ADDR, \
-		m)
-#define HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_OUT(v) out_dword( \
-		HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_ADDR, \
-		v)
-#define HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_IN)
-#define HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_ADDR_BMSK 0xfffffff8
-#define HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_ADDR_SHFT 0x3
-#define HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_ZERO_BMSK 0x7
-#define HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_ZERO_SHFT 0x0
-#define HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_MSB_ADDR (IPA_CFG_REG_BASE + \
-						   0x000001e4)
-#define HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_MSB_PHYS (IPA_CFG_REG_BASE_PHYS + \
-						   0x000001e4)
-#define HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_MSB_OFFS (IPA_CFG_REG_BASE_OFFS + \
-						   0x000001e4)
-#define HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_MSB_RMSK 0xffffffff
-#define HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_MSB_ATTR 0x3
-#define HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_MSB_IN in_dword_masked( \
-		HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_MSB_ADDR, \
-		HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_MSB_RMSK)
-#define HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_MSB_INM(m) in_dword_masked( \
-		HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_MSB_ADDR, \
-		m)
-#define HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_MSB_OUT(v) out_dword( \
-		HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_MSB_ADDR, \
-		v)
-#define HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_MSB_OUTM(m, \
-						  v) out_dword_masked_ns( \
-		HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_MSB_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_MSB_IN)
-#define HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_MSB_ADDR_BMSK 0xffffffff
-#define HWIO_IPA_SYS_PKT_PROC_CNTXT_BASE_MSB_ADDR_SHFT 0x0
-#define HWIO_IPA_LOCAL_PKT_PROC_CNTXT_BASE_ADDR (IPA_CFG_REG_BASE + \
-						 0x000001e8)
-#define HWIO_IPA_LOCAL_PKT_PROC_CNTXT_BASE_PHYS (IPA_CFG_REG_BASE_PHYS + \
-						 0x000001e8)
-#define HWIO_IPA_LOCAL_PKT_PROC_CNTXT_BASE_OFFS (IPA_CFG_REG_BASE_OFFS + \
-						 0x000001e8)
-#define HWIO_IPA_LOCAL_PKT_PROC_CNTXT_BASE_RMSK 0x3ffff
-#define HWIO_IPA_LOCAL_PKT_PROC_CNTXT_BASE_ATTR 0x3
-#define HWIO_IPA_LOCAL_PKT_PROC_CNTXT_BASE_IN in_dword_masked( \
-		HWIO_IPA_LOCAL_PKT_PROC_CNTXT_BASE_ADDR, \
-		HWIO_IPA_LOCAL_PKT_PROC_CNTXT_BASE_RMSK)
-#define HWIO_IPA_LOCAL_PKT_PROC_CNTXT_BASE_INM(m) in_dword_masked( \
-		HWIO_IPA_LOCAL_PKT_PROC_CNTXT_BASE_ADDR, \
-		m)
-#define HWIO_IPA_LOCAL_PKT_PROC_CNTXT_BASE_OUT(v) out_dword( \
-		HWIO_IPA_LOCAL_PKT_PROC_CNTXT_BASE_ADDR, \
-		v)
-#define HWIO_IPA_LOCAL_PKT_PROC_CNTXT_BASE_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_LOCAL_PKT_PROC_CNTXT_BASE_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_LOCAL_PKT_PROC_CNTXT_BASE_IN)
-#define HWIO_IPA_LOCAL_PKT_PROC_CNTXT_BASE_ADDR_BMSK 0x3fff8
-#define HWIO_IPA_LOCAL_PKT_PROC_CNTXT_BASE_ADDR_SHFT 0x3
-#define HWIO_IPA_LOCAL_PKT_PROC_CNTXT_BASE_ZERO_BMSK 0x7
-#define HWIO_IPA_LOCAL_PKT_PROC_CNTXT_BASE_ZERO_SHFT 0x0
-#define HWIO_IPA_AGGR_FORCE_CLOSE_ADDR (IPA_CFG_REG_BASE + 0x000001ec)
-#define HWIO_IPA_AGGR_FORCE_CLOSE_PHYS (IPA_CFG_REG_BASE_PHYS + 0x000001ec)
-#define HWIO_IPA_AGGR_FORCE_CLOSE_OFFS (IPA_CFG_REG_BASE_OFFS + 0x000001ec)
-#define HWIO_IPA_SCND_FRAG_VALUES_ADDR (IPA_CFG_REG_BASE + 0x000001f4)
-#define HWIO_IPA_SCND_FRAG_VALUES_PHYS (IPA_CFG_REG_BASE_PHYS + 0x000001f4)
-#define HWIO_IPA_SCND_FRAG_VALUES_OFFS (IPA_CFG_REG_BASE_OFFS + 0x000001f4)
-#define HWIO_IPA_TX_CFG_ADDR (IPA_CFG_REG_BASE + 0x000001fc)
-#define HWIO_IPA_TX_CFG_PHYS (IPA_CFG_REG_BASE_PHYS + 0x000001fc)
-#define HWIO_IPA_TX_CFG_OFFS (IPA_CFG_REG_BASE_OFFS + 0x000001fc)
-#define HWIO_IPA_NAT_UC_EXTERNAL_CFG_ADDR (IPA_CFG_REG_BASE + 0x00000200)
-#define HWIO_IPA_NAT_UC_EXTERNAL_CFG_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					   0x00000200)
-#define HWIO_IPA_NAT_UC_EXTERNAL_CFG_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					   0x00000200)
-#define HWIO_IPA_NAT_UC_LOCAL_CFG_ADDR (IPA_CFG_REG_BASE + 0x00000204)
-#define HWIO_IPA_NAT_UC_LOCAL_CFG_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00000204)
-#define HWIO_IPA_NAT_UC_LOCAL_CFG_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00000204)
-#define HWIO_IPA_NAT_UC_SHARED_CFG_ADDR (IPA_CFG_REG_BASE + 0x00000208)
-#define HWIO_IPA_NAT_UC_SHARED_CFG_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					 0x00000208)
-#define HWIO_IPA_NAT_UC_SHARED_CFG_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					 0x00000208)
-#define HWIO_IPA_RAM_INTLV_CFG_ADDR (IPA_CFG_REG_BASE + 0x0000020c)
-#define HWIO_IPA_RAM_INTLV_CFG_PHYS (IPA_CFG_REG_BASE_PHYS + 0x0000020c)
-#define HWIO_IPA_RAM_INTLV_CFG_OFFS (IPA_CFG_REG_BASE_OFFS + 0x0000020c)
-#define HWIO_IPA_FLAVOR_0_ADDR (IPA_CFG_REG_BASE + 0x00000210)
-#define HWIO_IPA_FLAVOR_0_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00000210)
-#define HWIO_IPA_FLAVOR_0_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00000210)
-#define HWIO_IPA_FLAVOR_1_ADDR (IPA_CFG_REG_BASE + 0x00000214)
-#define HWIO_IPA_FLAVOR_1_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00000214)
-#define HWIO_IPA_FLAVOR_1_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00000214)
-#define HWIO_IPA_FLAVOR_2_ADDR (IPA_CFG_REG_BASE + 0x00000218)
-#define HWIO_IPA_FLAVOR_2_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00000218)
-#define HWIO_IPA_FLAVOR_2_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00000218)
-#define HWIO_IPA_FLAVOR_3_ADDR (IPA_CFG_REG_BASE + 0x0000021c)
-#define HWIO_IPA_FLAVOR_3_PHYS (IPA_CFG_REG_BASE_PHYS + 0x0000021c)
-#define HWIO_IPA_FLAVOR_3_OFFS (IPA_CFG_REG_BASE_OFFS + 0x0000021c)
-#define HWIO_IPA_FLAVOR_4_ADDR (IPA_CFG_REG_BASE + 0x00000220)
-#define HWIO_IPA_FLAVOR_4_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00000220)
-#define HWIO_IPA_FLAVOR_4_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00000220)
-#define HWIO_IPA_FLAVOR_5_ADDR (IPA_CFG_REG_BASE + 0x00000224)
-#define HWIO_IPA_FLAVOR_5_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00000224)
-#define HWIO_IPA_FLAVOR_5_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00000224)
-#define HWIO_IPA_FLAVOR_6_ADDR (IPA_CFG_REG_BASE + 0x00000228)
-#define HWIO_IPA_FLAVOR_6_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00000228)
-#define HWIO_IPA_FLAVOR_6_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00000228)
-#define HWIO_IPA_FLAVOR_7_ADDR (IPA_CFG_REG_BASE + 0x0000022c)
-#define HWIO_IPA_FLAVOR_7_PHYS (IPA_CFG_REG_BASE_PHYS + 0x0000022c)
-#define HWIO_IPA_FLAVOR_7_OFFS (IPA_CFG_REG_BASE_OFFS + 0x0000022c)
-#define HWIO_IPA_CONN_TRACK_UC_EXTERNAL_CFG_ADDR (IPA_CFG_REG_BASE + \
-						  0x00000230)
-#define HWIO_IPA_CONN_TRACK_UC_EXTERNAL_CFG_PHYS (IPA_CFG_REG_BASE_PHYS + \
-						  0x00000230)
-#define HWIO_IPA_CONN_TRACK_UC_EXTERNAL_CFG_OFFS (IPA_CFG_REG_BASE_OFFS + \
-						  0x00000230)
-#define HWIO_IPA_CONN_TRACK_UC_LOCAL_CFG_ADDR (IPA_CFG_REG_BASE + \
-					       0x00000234)
-#define HWIO_IPA_CONN_TRACK_UC_LOCAL_CFG_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					       0x00000234)
-#define HWIO_IPA_CONN_TRACK_UC_LOCAL_CFG_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					       0x00000234)
-#define HWIO_IPA_CONN_TRACK_UC_SHARED_CFG_ADDR (IPA_CFG_REG_BASE + \
-						0x00000238)
-#define HWIO_IPA_CONN_TRACK_UC_SHARED_CFG_PHYS (IPA_CFG_REG_BASE_PHYS +	\
-						0x00000238)
-#define HWIO_IPA_CONN_TRACK_UC_SHARED_CFG_OFFS (IPA_CFG_REG_BASE_OFFS +	\
-						0x00000238)
-#define HWIO_IPA_IDLE_INDICATION_CFG_ADDR (IPA_CFG_REG_BASE + 0x00000240)
-#define HWIO_IPA_IDLE_INDICATION_CFG_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					   0x00000240)
-#define HWIO_IPA_IDLE_INDICATION_CFG_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					   0x00000240)
-#define HWIO_IPA_QTIME_TIMESTAMP_CFG_ADDR (IPA_CFG_REG_BASE + 0x0000024c)
-#define HWIO_IPA_QTIME_TIMESTAMP_CFG_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					   0x0000024c)
-#define HWIO_IPA_QTIME_TIMESTAMP_CFG_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					   0x0000024c)
-#define HWIO_IPA_TIMERS_XO_CLK_DIV_CFG_ADDR (IPA_CFG_REG_BASE + 0x00000250)
-#define HWIO_IPA_TIMERS_XO_CLK_DIV_CFG_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					     0x00000250)
-#define HWIO_IPA_TIMERS_XO_CLK_DIV_CFG_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					     0x00000250)
-#define HWIO_IPA_TIMERS_PULSE_GRAN_CFG_ADDR (IPA_CFG_REG_BASE + 0x00000254)
-#define HWIO_IPA_TIMERS_PULSE_GRAN_CFG_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					     0x00000254)
-#define HWIO_IPA_TIMERS_PULSE_GRAN_CFG_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					     0x00000254)
-#define HWIO_IPA_QTIME_SMP_ADDR (IPA_CFG_REG_BASE + 0x00000260)
-#define HWIO_IPA_QTIME_SMP_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00000260)
-#define HWIO_IPA_QTIME_SMP_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00000260)
-#define HWIO_IPA_QTIME_LSB_ADDR (IPA_CFG_REG_BASE + 0x00000264)
-#define HWIO_IPA_QTIME_LSB_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00000264)
-#define HWIO_IPA_QTIME_LSB_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00000264)
-#define HWIO_IPA_QTIME_MSB_ADDR (IPA_CFG_REG_BASE + 0x00000268)
-#define HWIO_IPA_QTIME_MSB_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00000268)
-#define HWIO_IPA_QTIME_MSB_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00000268)
-#define HWIO_IPA_SRC_RSRC_AMOUNT_REDUCE_EN_ADDR (IPA_CFG_REG_BASE + \
-						 0x00000334)
-#define HWIO_IPA_SRC_RSRC_AMOUNT_REDUCE_EN_PHYS (IPA_CFG_REG_BASE_PHYS + \
-						 0x00000334)
-#define HWIO_IPA_SRC_RSRC_AMOUNT_REDUCE_EN_OFFS (IPA_CFG_REG_BASE_OFFS + \
-						 0x00000334)
-#define HWIO_IPA_SRC_RSRC_AMOUNT_REDUCE_VALUES_0_ADDR (IPA_CFG_REG_BASE + \
-						       0x00000338)
-#define HWIO_IPA_SRC_RSRC_AMOUNT_REDUCE_VALUES_0_PHYS (	\
-		IPA_CFG_REG_BASE_PHYS + 0x00000338)
-#define HWIO_IPA_SRC_RSRC_AMOUNT_REDUCE_VALUES_0_OFFS (	\
-		IPA_CFG_REG_BASE_OFFS + 0x00000338)
-#define HWIO_IPA_SRC_RSRC_AMOUNT_REDUCE_VALUES_1_ADDR (IPA_CFG_REG_BASE + \
-						       0x0000033c)
-#define HWIO_IPA_SRC_RSRC_AMOUNT_REDUCE_VALUES_1_PHYS (	\
-		IPA_CFG_REG_BASE_PHYS + 0x0000033c)
-#define HWIO_IPA_SRC_RSRC_AMOUNT_REDUCE_VALUES_1_OFFS (	\
-		IPA_CFG_REG_BASE_OFFS + 0x0000033c)
-#define HWIO_IPA_DST_RSRC_AMOUNT_REDUCE_EN_ADDR (IPA_CFG_REG_BASE + \
-						 0x00000340)
-#define HWIO_IPA_DST_RSRC_AMOUNT_REDUCE_EN_PHYS (IPA_CFG_REG_BASE_PHYS + \
-						 0x00000340)
-#define HWIO_IPA_DST_RSRC_AMOUNT_REDUCE_EN_OFFS (IPA_CFG_REG_BASE_OFFS + \
-						 0x00000340)
-#define HWIO_IPA_DST_RSRC_AMOUNT_REDUCE_VALUES_0_ADDR (IPA_CFG_REG_BASE + \
-						       0x00000344)
-#define HWIO_IPA_DST_RSRC_AMOUNT_REDUCE_VALUES_0_PHYS (	\
-		IPA_CFG_REG_BASE_PHYS + 0x00000344)
-#define HWIO_IPA_DST_RSRC_AMOUNT_REDUCE_VALUES_0_OFFS (	\
-		IPA_CFG_REG_BASE_OFFS + 0x00000344)
-#define HWIO_IPA_HPS_DPS_CMDQ_RED_IRQ_MASK_ENABLE_ADDR (IPA_CFG_REG_BASE + \
-							0x00000348)
-#define HWIO_IPA_HPS_DPS_CMDQ_RED_IRQ_MASK_ENABLE_PHYS ( \
-		IPA_CFG_REG_BASE_PHYS + 0x00000348)
-#define HWIO_IPA_HPS_DPS_CMDQ_RED_IRQ_MASK_ENABLE_OFFS ( \
-		IPA_CFG_REG_BASE_OFFS + 0x00000348)
-#define HWIO_IPA_SRC_RSRC_GRP_01_RSRC_TYPE_n_ADDR(n) (IPA_CFG_REG_BASE + \
-						      0x00000400 + 0x20 * \
-						      (n))
-#define HWIO_IPA_SRC_RSRC_GRP_01_RSRC_TYPE_n_PHYS(n) ( \
-		IPA_CFG_REG_BASE_PHYS + 0x00000400 + 0x20 * (n))
-#define HWIO_IPA_SRC_RSRC_GRP_01_RSRC_TYPE_n_OFFS(n) ( \
-		IPA_CFG_REG_BASE_OFFS + 0x00000400 + 0x20 * (n))
-#define HWIO_IPA_SRC_RSRC_GRP_01_RSRC_TYPE_n_RMSK 0x3f3f3f3f
-#define HWIO_IPA_SRC_RSRC_GRP_01_RSRC_TYPE_n_MAXn 4
-#define HWIO_IPA_SRC_RSRC_GRP_01_RSRC_TYPE_n_ATTR 0x3
-#define HWIO_IPA_SRC_RSRC_GRP_01_RSRC_TYPE_n_INI(n) in_dword_masked( \
-		HWIO_IPA_SRC_RSRC_GRP_01_RSRC_TYPE_n_ADDR(n), \
-		HWIO_IPA_SRC_RSRC_GRP_01_RSRC_TYPE_n_RMSK)
-#define HWIO_IPA_SRC_RSRC_GRP_01_RSRC_TYPE_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_SRC_RSRC_GRP_01_RSRC_TYPE_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_SRC_RSRC_GRP_01_RSRC_TYPE_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_SRC_RSRC_GRP_01_RSRC_TYPE_n_ADDR(n), \
-		val)
-#define HWIO_IPA_SRC_RSRC_GRP_01_RSRC_TYPE_n_OUTMI(n, mask, \
-						   val)	\
-	out_dword_masked_ns(HWIO_IPA_SRC_RSRC_GRP_01_RSRC_TYPE_n_ADDR( \
-				    n),	\
-			    mask, val, \
-			    HWIO_IPA_SRC_RSRC_GRP_01_RSRC_TYPE_n_INI(n))
-#define HWIO_IPA_SRC_RSRC_GRP_01_RSRC_TYPE_n_SRC_RSRC_GRP_1_MAX_LIMIT_BMSK \
-	0x3f000000
-#define HWIO_IPA_SRC_RSRC_GRP_01_RSRC_TYPE_n_SRC_RSRC_GRP_1_MAX_LIMIT_SHFT \
-	0x18
-#define HWIO_IPA_SRC_RSRC_GRP_01_RSRC_TYPE_n_SRC_RSRC_GRP_1_MIN_LIMIT_BMSK \
-	0x3f0000
-#define HWIO_IPA_SRC_RSRC_GRP_01_RSRC_TYPE_n_SRC_RSRC_GRP_1_MIN_LIMIT_SHFT \
-	0x10
-#define HWIO_IPA_SRC_RSRC_GRP_01_RSRC_TYPE_n_SRC_RSRC_GRP_0_MAX_LIMIT_BMSK \
-	0x3f00
-#define HWIO_IPA_SRC_RSRC_GRP_01_RSRC_TYPE_n_SRC_RSRC_GRP_0_MAX_LIMIT_SHFT \
-	0x8
-#define HWIO_IPA_SRC_RSRC_GRP_01_RSRC_TYPE_n_SRC_RSRC_GRP_0_MIN_LIMIT_BMSK \
-	0x3f
-#define HWIO_IPA_SRC_RSRC_GRP_01_RSRC_TYPE_n_SRC_RSRC_GRP_0_MIN_LIMIT_SHFT \
-	0x0
-#define HWIO_IPA_SRC_RSRC_GRP_23_RSRC_TYPE_n_ADDR(n) (IPA_CFG_REG_BASE + \
-						      0x00000404 + 0x20 * \
-						      (n))
-#define HWIO_IPA_SRC_RSRC_GRP_23_RSRC_TYPE_n_PHYS(n) ( \
-		IPA_CFG_REG_BASE_PHYS + 0x00000404 + 0x20 * (n))
-#define HWIO_IPA_SRC_RSRC_GRP_23_RSRC_TYPE_n_OFFS(n) ( \
-		IPA_CFG_REG_BASE_OFFS + 0x00000404 + 0x20 * (n))
-#define HWIO_IPA_SRC_RSRC_GRP_23_RSRC_TYPE_n_RMSK 0x3f3f3f3f
-#define HWIO_IPA_SRC_RSRC_GRP_23_RSRC_TYPE_n_MAXn 4
-#define HWIO_IPA_SRC_RSRC_GRP_23_RSRC_TYPE_n_ATTR 0x3
-#define HWIO_IPA_SRC_RSRC_GRP_23_RSRC_TYPE_n_INI(n) in_dword_masked( \
-		HWIO_IPA_SRC_RSRC_GRP_23_RSRC_TYPE_n_ADDR(n), \
-		HWIO_IPA_SRC_RSRC_GRP_23_RSRC_TYPE_n_RMSK)
-#define HWIO_IPA_SRC_RSRC_GRP_23_RSRC_TYPE_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_SRC_RSRC_GRP_23_RSRC_TYPE_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_SRC_RSRC_GRP_23_RSRC_TYPE_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_SRC_RSRC_GRP_23_RSRC_TYPE_n_ADDR(n), \
-		val)
-#define HWIO_IPA_SRC_RSRC_GRP_23_RSRC_TYPE_n_OUTMI(n, mask, \
-						   val)	\
-	out_dword_masked_ns(HWIO_IPA_SRC_RSRC_GRP_23_RSRC_TYPE_n_ADDR( \
-				    n),	\
-			    mask, val, \
-			    HWIO_IPA_SRC_RSRC_GRP_23_RSRC_TYPE_n_INI(n))
-#define HWIO_IPA_SRC_RSRC_GRP_23_RSRC_TYPE_n_SRC_RSRC_GRP_3_MAX_LIMIT_BMSK \
-	0x3f000000
-#define HWIO_IPA_SRC_RSRC_GRP_23_RSRC_TYPE_n_SRC_RSRC_GRP_3_MAX_LIMIT_SHFT \
-	0x18
-#define HWIO_IPA_SRC_RSRC_GRP_23_RSRC_TYPE_n_SRC_RSRC_GRP_3_MIN_LIMIT_BMSK \
-	0x3f0000
-#define HWIO_IPA_SRC_RSRC_GRP_23_RSRC_TYPE_n_SRC_RSRC_GRP_3_MIN_LIMIT_SHFT \
-	0x10
-#define HWIO_IPA_SRC_RSRC_GRP_23_RSRC_TYPE_n_SRC_RSRC_GRP_2_MAX_LIMIT_BMSK \
-	0x3f00
-#define HWIO_IPA_SRC_RSRC_GRP_23_RSRC_TYPE_n_SRC_RSRC_GRP_2_MAX_LIMIT_SHFT \
-	0x8
-#define HWIO_IPA_SRC_RSRC_GRP_23_RSRC_TYPE_n_SRC_RSRC_GRP_2_MIN_LIMIT_BMSK \
-	0x3f
-#define HWIO_IPA_SRC_RSRC_GRP_23_RSRC_TYPE_n_SRC_RSRC_GRP_2_MIN_LIMIT_SHFT \
-	0x0
-#define HWIO_IPA_SRC_RSRC_GRP_45_RSRC_TYPE_n_ADDR(n) (IPA_CFG_REG_BASE + \
-						      0x00000408 + 0x20 * \
-						      (n))
-#define HWIO_IPA_SRC_RSRC_GRP_45_RSRC_TYPE_n_PHYS(n) ( \
-		IPA_CFG_REG_BASE_PHYS + 0x00000408 + 0x20 * (n))
-#define HWIO_IPA_SRC_RSRC_GRP_45_RSRC_TYPE_n_OFFS(n) ( \
-		IPA_CFG_REG_BASE_OFFS + 0x00000408 + 0x20 * (n))
-#define HWIO_IPA_SRC_RSRC_GRP_45_RSRC_TYPE_n_RMSK 0x3f3f
-#define HWIO_IPA_SRC_RSRC_GRP_45_RSRC_TYPE_n_MAXn 4
-#define HWIO_IPA_SRC_RSRC_GRP_45_RSRC_TYPE_n_ATTR 0x3
-#define HWIO_IPA_SRC_RSRC_GRP_45_RSRC_TYPE_n_INI(n) in_dword_masked( \
-		HWIO_IPA_SRC_RSRC_GRP_45_RSRC_TYPE_n_ADDR(n), \
-		HWIO_IPA_SRC_RSRC_GRP_45_RSRC_TYPE_n_RMSK)
-#define HWIO_IPA_SRC_RSRC_GRP_45_RSRC_TYPE_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_SRC_RSRC_GRP_45_RSRC_TYPE_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_SRC_RSRC_GRP_45_RSRC_TYPE_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_SRC_RSRC_GRP_45_RSRC_TYPE_n_ADDR(n), \
-		val)
-#define HWIO_IPA_SRC_RSRC_GRP_45_RSRC_TYPE_n_OUTMI(n, mask, \
-						   val)	\
-	out_dword_masked_ns(HWIO_IPA_SRC_RSRC_GRP_45_RSRC_TYPE_n_ADDR( \
-				    n),	\
-			    mask, val, \
-			    HWIO_IPA_SRC_RSRC_GRP_45_RSRC_TYPE_n_INI(n))
-#define HWIO_IPA_SRC_RSRC_GRP_45_RSRC_TYPE_n_SRC_RSRC_GRP_4_MAX_LIMIT_BMSK \
-	0x3f00
-#define HWIO_IPA_SRC_RSRC_GRP_45_RSRC_TYPE_n_SRC_RSRC_GRP_4_MAX_LIMIT_SHFT \
-	0x8
-#define HWIO_IPA_SRC_RSRC_GRP_45_RSRC_TYPE_n_SRC_RSRC_GRP_4_MIN_LIMIT_BMSK \
-	0x3f
-#define HWIO_IPA_SRC_RSRC_GRP_45_RSRC_TYPE_n_SRC_RSRC_GRP_4_MIN_LIMIT_SHFT \
-	0x0
-#define HWIO_IPA_SRC_RSRC_GRP_0123_RSRC_TYPE_CNT_n_ADDR(n) ( \
-		IPA_CFG_REG_BASE + 0x00000410 + 0x20 * (n))
-#define HWIO_IPA_SRC_RSRC_GRP_0123_RSRC_TYPE_CNT_n_PHYS(n) ( \
-		IPA_CFG_REG_BASE_PHYS + 0x00000410 + 0x20 * (n))
-#define HWIO_IPA_SRC_RSRC_GRP_0123_RSRC_TYPE_CNT_n_OFFS(n) ( \
-		IPA_CFG_REG_BASE_OFFS + 0x00000410 + 0x20 * (n))
-#define HWIO_IPA_SRC_RSRC_GRP_0123_RSRC_TYPE_CNT_n_RMSK 0x3f3f3f3f
-#define HWIO_IPA_SRC_RSRC_GRP_0123_RSRC_TYPE_CNT_n_MAXn 4
-#define HWIO_IPA_SRC_RSRC_GRP_0123_RSRC_TYPE_CNT_n_ATTR 0x1
-#define HWIO_IPA_SRC_RSRC_GRP_0123_RSRC_TYPE_CNT_n_INI(n) in_dword_masked( \
-		HWIO_IPA_SRC_RSRC_GRP_0123_RSRC_TYPE_CNT_n_ADDR(n), \
-		HWIO_IPA_SRC_RSRC_GRP_0123_RSRC_TYPE_CNT_n_RMSK)
-#define HWIO_IPA_SRC_RSRC_GRP_0123_RSRC_TYPE_CNT_n_INMI(n, \
-							mask) \
-	in_dword_masked( \
-		HWIO_IPA_SRC_RSRC_GRP_0123_RSRC_TYPE_CNT_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_SRC_RSRC_GRP_0123_RSRC_TYPE_CNT_n_SRC_RSRC_GRP_3_CNT_BMSK \
-	0x3f000000
-#define HWIO_IPA_SRC_RSRC_GRP_0123_RSRC_TYPE_CNT_n_SRC_RSRC_GRP_3_CNT_SHFT \
-	0x18
-#define HWIO_IPA_SRC_RSRC_GRP_0123_RSRC_TYPE_CNT_n_SRC_RSRC_GRP_2_CNT_BMSK \
-	0x3f0000
-#define HWIO_IPA_SRC_RSRC_GRP_0123_RSRC_TYPE_CNT_n_SRC_RSRC_GRP_2_CNT_SHFT \
-	0x10
-#define HWIO_IPA_SRC_RSRC_GRP_0123_RSRC_TYPE_CNT_n_SRC_RSRC_GRP_1_CNT_BMSK \
-	0x3f00
-#define HWIO_IPA_SRC_RSRC_GRP_0123_RSRC_TYPE_CNT_n_SRC_RSRC_GRP_1_CNT_SHFT \
-	0x8
-#define HWIO_IPA_SRC_RSRC_GRP_0123_RSRC_TYPE_CNT_n_SRC_RSRC_GRP_0_CNT_BMSK \
-	0x3f
-#define HWIO_IPA_SRC_RSRC_GRP_0123_RSRC_TYPE_CNT_n_SRC_RSRC_GRP_0_CNT_SHFT \
-	0x0
-#define HWIO_IPA_SRC_RSRC_GRP_4567_RSRC_TYPE_CNT_n_ADDR(n) ( \
-		IPA_CFG_REG_BASE + 0x00000414 + 0x20 * (n))
-#define HWIO_IPA_SRC_RSRC_GRP_4567_RSRC_TYPE_CNT_n_PHYS(n) ( \
-		IPA_CFG_REG_BASE_PHYS + 0x00000414 + 0x20 * (n))
-#define HWIO_IPA_SRC_RSRC_GRP_4567_RSRC_TYPE_CNT_n_OFFS(n) ( \
-		IPA_CFG_REG_BASE_OFFS + 0x00000414 + 0x20 * (n))
-#define HWIO_IPA_SRC_RSRC_GRP_4567_RSRC_TYPE_CNT_n_RMSK 0x3f
-#define HWIO_IPA_SRC_RSRC_GRP_4567_RSRC_TYPE_CNT_n_MAXn 4
-#define HWIO_IPA_SRC_RSRC_GRP_4567_RSRC_TYPE_CNT_n_ATTR 0x1
-#define HWIO_IPA_SRC_RSRC_GRP_4567_RSRC_TYPE_CNT_n_INI(n) in_dword_masked( \
-		HWIO_IPA_SRC_RSRC_GRP_4567_RSRC_TYPE_CNT_n_ADDR(n), \
-		HWIO_IPA_SRC_RSRC_GRP_4567_RSRC_TYPE_CNT_n_RMSK)
-#define HWIO_IPA_SRC_RSRC_GRP_4567_RSRC_TYPE_CNT_n_INMI(n, \
-							mask) \
-	in_dword_masked( \
-		HWIO_IPA_SRC_RSRC_GRP_4567_RSRC_TYPE_CNT_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_SRC_RSRC_GRP_4567_RSRC_TYPE_CNT_n_SRC_RSRC_GRP_4_CNT_BMSK \
-	0x3f
-#define HWIO_IPA_SRC_RSRC_GRP_4567_RSRC_TYPE_CNT_n_SRC_RSRC_GRP_4_CNT_SHFT \
-	0x0
-#define HWIO_IPA_SRC_RSRC_TYPE_AMOUNT_n_ADDR(n) (IPA_CFG_REG_BASE + \
-						 0x00000418 + 0x20 * (n))
-#define HWIO_IPA_SRC_RSRC_TYPE_AMOUNT_n_PHYS(n) (IPA_CFG_REG_BASE_PHYS + \
-						 0x00000418 + 0x20 * (n))
-#define HWIO_IPA_SRC_RSRC_TYPE_AMOUNT_n_OFFS(n) (IPA_CFG_REG_BASE_OFFS + \
-						 0x00000418 + 0x20 * (n))
-#define HWIO_IPA_DST_RSRC_GRP_01_RSRC_TYPE_n_ADDR(n) (IPA_CFG_REG_BASE + \
-						      0x00000500 + 0x20 * \
-						      (n))
-#define HWIO_IPA_DST_RSRC_GRP_01_RSRC_TYPE_n_PHYS(n) ( \
-		IPA_CFG_REG_BASE_PHYS + 0x00000500 + 0x20 * (n))
-#define HWIO_IPA_DST_RSRC_GRP_01_RSRC_TYPE_n_OFFS(n) ( \
-		IPA_CFG_REG_BASE_OFFS + 0x00000500 + 0x20 * (n))
-#define HWIO_IPA_DST_RSRC_GRP_01_RSRC_TYPE_n_RMSK 0x3f3f3f3f
-#define HWIO_IPA_DST_RSRC_GRP_01_RSRC_TYPE_n_MAXn 1
-#define HWIO_IPA_DST_RSRC_GRP_01_RSRC_TYPE_n_ATTR 0x3
-#define HWIO_IPA_DST_RSRC_GRP_01_RSRC_TYPE_n_INI(n) in_dword_masked( \
-		HWIO_IPA_DST_RSRC_GRP_01_RSRC_TYPE_n_ADDR(n), \
-		HWIO_IPA_DST_RSRC_GRP_01_RSRC_TYPE_n_RMSK)
-#define HWIO_IPA_DST_RSRC_GRP_01_RSRC_TYPE_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_DST_RSRC_GRP_01_RSRC_TYPE_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_DST_RSRC_GRP_01_RSRC_TYPE_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_DST_RSRC_GRP_01_RSRC_TYPE_n_ADDR(n), \
-		val)
-#define HWIO_IPA_DST_RSRC_GRP_01_RSRC_TYPE_n_OUTMI(n, mask, \
-						   val)	\
-	out_dword_masked_ns(HWIO_IPA_DST_RSRC_GRP_01_RSRC_TYPE_n_ADDR( \
-				    n),	\
-			    mask, val, \
-			    HWIO_IPA_DST_RSRC_GRP_01_RSRC_TYPE_n_INI(n))
-#define HWIO_IPA_DST_RSRC_GRP_01_RSRC_TYPE_n_DST_RSRC_GRP_1_MAX_LIMIT_BMSK \
-	0x3f000000
-#define HWIO_IPA_DST_RSRC_GRP_01_RSRC_TYPE_n_DST_RSRC_GRP_1_MAX_LIMIT_SHFT \
-	0x18
-#define HWIO_IPA_DST_RSRC_GRP_01_RSRC_TYPE_n_DST_RSRC_GRP_1_MIN_LIMIT_BMSK \
-	0x3f0000
-#define HWIO_IPA_DST_RSRC_GRP_01_RSRC_TYPE_n_DST_RSRC_GRP_1_MIN_LIMIT_SHFT \
-	0x10
-#define HWIO_IPA_DST_RSRC_GRP_01_RSRC_TYPE_n_DST_RSRC_GRP_0_MAX_LIMIT_BMSK \
-	0x3f00
-#define HWIO_IPA_DST_RSRC_GRP_01_RSRC_TYPE_n_DST_RSRC_GRP_0_MAX_LIMIT_SHFT \
-	0x8
-#define HWIO_IPA_DST_RSRC_GRP_01_RSRC_TYPE_n_DST_RSRC_GRP_0_MIN_LIMIT_BMSK \
-	0x3f
-#define HWIO_IPA_DST_RSRC_GRP_01_RSRC_TYPE_n_DST_RSRC_GRP_0_MIN_LIMIT_SHFT \
-	0x0
-#define HWIO_IPA_DST_RSRC_GRP_23_RSRC_TYPE_n_ADDR(n) (IPA_CFG_REG_BASE + \
-						      0x00000504 + 0x20 * \
-						      (n))
-#define HWIO_IPA_DST_RSRC_GRP_23_RSRC_TYPE_n_PHYS(n) ( \
-		IPA_CFG_REG_BASE_PHYS + 0x00000504 + 0x20 * (n))
-#define HWIO_IPA_DST_RSRC_GRP_23_RSRC_TYPE_n_OFFS(n) ( \
-		IPA_CFG_REG_BASE_OFFS + 0x00000504 + 0x20 * (n))
-#define HWIO_IPA_DST_RSRC_GRP_23_RSRC_TYPE_n_RMSK 0x3f3f3f3f
-#define HWIO_IPA_DST_RSRC_GRP_23_RSRC_TYPE_n_MAXn 1
-#define HWIO_IPA_DST_RSRC_GRP_23_RSRC_TYPE_n_ATTR 0x3
-#define HWIO_IPA_DST_RSRC_GRP_23_RSRC_TYPE_n_INI(n) in_dword_masked( \
-		HWIO_IPA_DST_RSRC_GRP_23_RSRC_TYPE_n_ADDR(n), \
-		HWIO_IPA_DST_RSRC_GRP_23_RSRC_TYPE_n_RMSK)
-#define HWIO_IPA_DST_RSRC_GRP_23_RSRC_TYPE_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_DST_RSRC_GRP_23_RSRC_TYPE_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_DST_RSRC_GRP_23_RSRC_TYPE_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_DST_RSRC_GRP_23_RSRC_TYPE_n_ADDR(n), \
-		val)
-#define HWIO_IPA_DST_RSRC_GRP_23_RSRC_TYPE_n_OUTMI(n, mask, \
-						   val)	\
-	out_dword_masked_ns(HWIO_IPA_DST_RSRC_GRP_23_RSRC_TYPE_n_ADDR( \
-				    n),	\
-			    mask, val, \
-			    HWIO_IPA_DST_RSRC_GRP_23_RSRC_TYPE_n_INI(n))
-#define HWIO_IPA_DST_RSRC_GRP_23_RSRC_TYPE_n_DST_RSRC_GRP_3_MAX_LIMIT_BMSK \
-	0x3f000000
-#define HWIO_IPA_DST_RSRC_GRP_23_RSRC_TYPE_n_DST_RSRC_GRP_3_MAX_LIMIT_SHFT \
-	0x18
-#define HWIO_IPA_DST_RSRC_GRP_23_RSRC_TYPE_n_DST_RSRC_GRP_3_MIN_LIMIT_BMSK \
-	0x3f0000
-#define HWIO_IPA_DST_RSRC_GRP_23_RSRC_TYPE_n_DST_RSRC_GRP_3_MIN_LIMIT_SHFT \
-	0x10
-#define HWIO_IPA_DST_RSRC_GRP_23_RSRC_TYPE_n_DST_RSRC_GRP_2_MAX_LIMIT_BMSK \
-	0x3f00
-#define HWIO_IPA_DST_RSRC_GRP_23_RSRC_TYPE_n_DST_RSRC_GRP_2_MAX_LIMIT_SHFT \
-	0x8
-#define HWIO_IPA_DST_RSRC_GRP_23_RSRC_TYPE_n_DST_RSRC_GRP_2_MIN_LIMIT_BMSK \
-	0x3f
-#define HWIO_IPA_DST_RSRC_GRP_23_RSRC_TYPE_n_DST_RSRC_GRP_2_MIN_LIMIT_SHFT \
-	0x0
-#define HWIO_IPA_DST_RSRC_GRP_45_RSRC_TYPE_n_ADDR(n) (IPA_CFG_REG_BASE + \
-						      0x00000508 + 0x20 * \
-						      (n))
-#define HWIO_IPA_DST_RSRC_GRP_45_RSRC_TYPE_n_PHYS(n) ( \
-		IPA_CFG_REG_BASE_PHYS + 0x00000508 + 0x20 * (n))
-#define HWIO_IPA_DST_RSRC_GRP_45_RSRC_TYPE_n_OFFS(n) ( \
-		IPA_CFG_REG_BASE_OFFS + 0x00000508 + 0x20 * (n))
-#define HWIO_IPA_DST_RSRC_GRP_45_RSRC_TYPE_n_RMSK 0x3f3f
-#define HWIO_IPA_DST_RSRC_GRP_45_RSRC_TYPE_n_MAXn 1
-#define HWIO_IPA_DST_RSRC_GRP_45_RSRC_TYPE_n_ATTR 0x3
-#define HWIO_IPA_DST_RSRC_GRP_45_RSRC_TYPE_n_INI(n) in_dword_masked( \
-		HWIO_IPA_DST_RSRC_GRP_45_RSRC_TYPE_n_ADDR(n), \
-		HWIO_IPA_DST_RSRC_GRP_45_RSRC_TYPE_n_RMSK)
-#define HWIO_IPA_DST_RSRC_GRP_45_RSRC_TYPE_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_DST_RSRC_GRP_45_RSRC_TYPE_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_DST_RSRC_GRP_45_RSRC_TYPE_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_DST_RSRC_GRP_45_RSRC_TYPE_n_ADDR(n), \
-		val)
-#define HWIO_IPA_DST_RSRC_GRP_45_RSRC_TYPE_n_OUTMI(n, mask, \
-						   val)	\
-	out_dword_masked_ns(HWIO_IPA_DST_RSRC_GRP_45_RSRC_TYPE_n_ADDR( \
-				    n),	\
-			    mask, val, \
-			    HWIO_IPA_DST_RSRC_GRP_45_RSRC_TYPE_n_INI(n))
-#define HWIO_IPA_DST_RSRC_GRP_45_RSRC_TYPE_n_DST_RSRC_GRP_4_MAX_LIMIT_BMSK \
-	0x3f00
-#define HWIO_IPA_DST_RSRC_GRP_45_RSRC_TYPE_n_DST_RSRC_GRP_4_MAX_LIMIT_SHFT \
-	0x8
-#define HWIO_IPA_DST_RSRC_GRP_45_RSRC_TYPE_n_DST_RSRC_GRP_4_MIN_LIMIT_BMSK \
-	0x3f
-#define HWIO_IPA_DST_RSRC_GRP_45_RSRC_TYPE_n_DST_RSRC_GRP_4_MIN_LIMIT_SHFT \
-	0x0
-#define HWIO_IPA_DST_RSRC_GRP_0123_RSRC_TYPE_CNT_n_ADDR(n) ( \
-		IPA_CFG_REG_BASE + 0x00000510 + 0x20 * (n))
-#define HWIO_IPA_DST_RSRC_GRP_0123_RSRC_TYPE_CNT_n_PHYS(n) ( \
-		IPA_CFG_REG_BASE_PHYS + 0x00000510 + 0x20 * (n))
-#define HWIO_IPA_DST_RSRC_GRP_0123_RSRC_TYPE_CNT_n_OFFS(n) ( \
-		IPA_CFG_REG_BASE_OFFS + 0x00000510 + 0x20 * (n))
-#define HWIO_IPA_DST_RSRC_GRP_0123_RSRC_TYPE_CNT_n_RMSK 0x3f3f3f3f
-#define HWIO_IPA_DST_RSRC_GRP_0123_RSRC_TYPE_CNT_n_MAXn 1
-#define HWIO_IPA_DST_RSRC_GRP_0123_RSRC_TYPE_CNT_n_ATTR 0x1
-#define HWIO_IPA_DST_RSRC_GRP_0123_RSRC_TYPE_CNT_n_INI(n) in_dword_masked( \
-		HWIO_IPA_DST_RSRC_GRP_0123_RSRC_TYPE_CNT_n_ADDR(n), \
-		HWIO_IPA_DST_RSRC_GRP_0123_RSRC_TYPE_CNT_n_RMSK)
-#define HWIO_IPA_DST_RSRC_GRP_0123_RSRC_TYPE_CNT_n_INMI(n, \
-							mask) \
-	in_dword_masked( \
-		HWIO_IPA_DST_RSRC_GRP_0123_RSRC_TYPE_CNT_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_DST_RSRC_GRP_0123_RSRC_TYPE_CNT_n_DST_RSRC_GRP_3_CNT_BMSK \
-	0x3f000000
-#define HWIO_IPA_DST_RSRC_GRP_0123_RSRC_TYPE_CNT_n_DST_RSRC_GRP_3_CNT_SHFT \
-	0x18
-#define HWIO_IPA_DST_RSRC_GRP_0123_RSRC_TYPE_CNT_n_DST_RSRC_GRP_2_CNT_BMSK \
-	0x3f0000
-#define HWIO_IPA_DST_RSRC_GRP_0123_RSRC_TYPE_CNT_n_DST_RSRC_GRP_2_CNT_SHFT \
-	0x10
-#define HWIO_IPA_DST_RSRC_GRP_0123_RSRC_TYPE_CNT_n_DST_RSRC_GRP_1_CNT_BMSK \
-	0x3f00
-#define HWIO_IPA_DST_RSRC_GRP_0123_RSRC_TYPE_CNT_n_DST_RSRC_GRP_1_CNT_SHFT \
-	0x8
-#define HWIO_IPA_DST_RSRC_GRP_0123_RSRC_TYPE_CNT_n_DST_RSRC_GRP_0_CNT_BMSK \
-	0x3f
-#define HWIO_IPA_DST_RSRC_GRP_0123_RSRC_TYPE_CNT_n_DST_RSRC_GRP_0_CNT_SHFT \
-	0x0
-#define HWIO_IPA_DST_RSRC_GRP_4567_RSRC_TYPE_CNT_n_ADDR(n) ( \
-		IPA_CFG_REG_BASE + 0x00000514 + 0x20 * (n))
-#define HWIO_IPA_DST_RSRC_GRP_4567_RSRC_TYPE_CNT_n_PHYS(n) ( \
-		IPA_CFG_REG_BASE_PHYS + 0x00000514 + 0x20 * (n))
-#define HWIO_IPA_DST_RSRC_GRP_4567_RSRC_TYPE_CNT_n_OFFS(n) ( \
-		IPA_CFG_REG_BASE_OFFS + 0x00000514 + 0x20 * (n))
-#define HWIO_IPA_DST_RSRC_GRP_4567_RSRC_TYPE_CNT_n_RMSK 0xff
-#define HWIO_IPA_DST_RSRC_GRP_4567_RSRC_TYPE_CNT_n_MAXn 1
-#define HWIO_IPA_DST_RSRC_GRP_4567_RSRC_TYPE_CNT_n_ATTR 0x1
-#define HWIO_IPA_DST_RSRC_GRP_4567_RSRC_TYPE_CNT_n_INI(n) in_dword_masked( \
-		HWIO_IPA_DST_RSRC_GRP_4567_RSRC_TYPE_CNT_n_ADDR(n), \
-		HWIO_IPA_DST_RSRC_GRP_4567_RSRC_TYPE_CNT_n_RMSK)
-#define HWIO_IPA_DST_RSRC_GRP_4567_RSRC_TYPE_CNT_n_INMI(n, \
-							mask) \
-	in_dword_masked( \
-		HWIO_IPA_DST_RSRC_GRP_4567_RSRC_TYPE_CNT_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_DST_RSRC_GRP_4567_RSRC_TYPE_CNT_n_DST_RSRC_GRP_4_CNT_BMSK \
-	0xff
-#define HWIO_IPA_DST_RSRC_GRP_4567_RSRC_TYPE_CNT_n_DST_RSRC_GRP_4_CNT_SHFT \
-	0x0
-#define HWIO_IPA_DST_RSRC_TYPE_AMOUNT_n_ADDR(n) (IPA_CFG_REG_BASE + \
-						 0x00000518 + 0x20 * (n))
-#define HWIO_IPA_DST_RSRC_TYPE_AMOUNT_n_PHYS(n) (IPA_CFG_REG_BASE_PHYS + \
-						 0x00000518 + 0x20 * (n))
-#define HWIO_IPA_DST_RSRC_TYPE_AMOUNT_n_OFFS(n) (IPA_CFG_REG_BASE_OFFS + \
-						 0x00000518 + 0x20 * (n))
-#define HWIO_IPA_RSRC_GRP_CFG_ADDR (IPA_CFG_REG_BASE + 0x000005a0)
-#define HWIO_IPA_RSRC_GRP_CFG_PHYS (IPA_CFG_REG_BASE_PHYS + 0x000005a0)
-#define HWIO_IPA_RSRC_GRP_CFG_OFFS (IPA_CFG_REG_BASE_OFFS + 0x000005a0)
-#define HWIO_IPA_RSRC_GRP_CFG_RMSK 0x3f11f171
-#define HWIO_IPA_RSRC_GRP_CFG_ATTR 0x3
-#define HWIO_IPA_RSRC_GRP_CFG_IN in_dword_masked( \
-		HWIO_IPA_RSRC_GRP_CFG_ADDR, \
-		HWIO_IPA_RSRC_GRP_CFG_RMSK)
-#define HWIO_IPA_RSRC_GRP_CFG_INM(m) in_dword_masked( \
-		HWIO_IPA_RSRC_GRP_CFG_ADDR, \
-		m)
-#define HWIO_IPA_RSRC_GRP_CFG_OUT(v) out_dword(HWIO_IPA_RSRC_GRP_CFG_ADDR, \
-					       v)
-#define HWIO_IPA_RSRC_GRP_CFG_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_RSRC_GRP_CFG_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_RSRC_GRP_CFG_IN)
-#define HWIO_IPA_RSRC_GRP_CFG_DST_GRP_SPECIAL_INDEX_BMSK 0x3f000000
-#define HWIO_IPA_RSRC_GRP_CFG_DST_GRP_SPECIAL_INDEX_SHFT 0x18
-#define HWIO_IPA_RSRC_GRP_CFG_DST_GRP_SPECIAL_VALID_BMSK 0x100000
-#define HWIO_IPA_RSRC_GRP_CFG_DST_GRP_SPECIAL_VALID_SHFT 0x14
-#define HWIO_IPA_RSRC_GRP_CFG_DST_PIPE_SPECIAL_INDEX_BMSK 0x1f000
-#define HWIO_IPA_RSRC_GRP_CFG_DST_PIPE_SPECIAL_INDEX_SHFT 0xc
-#define HWIO_IPA_RSRC_GRP_CFG_DST_PIPE_SPECIAL_VALID_BMSK 0x100
-#define HWIO_IPA_RSRC_GRP_CFG_DST_PIPE_SPECIAL_VALID_SHFT 0x8
-#define HWIO_IPA_RSRC_GRP_CFG_SRC_GRP_SPECIAL_INDEX_BMSK 0x70
-#define HWIO_IPA_RSRC_GRP_CFG_SRC_GRP_SPECIAL_INDEX_SHFT 0x4
-#define HWIO_IPA_RSRC_GRP_CFG_SRC_GRP_SPECIAL_VALID_BMSK 0x1
-#define HWIO_IPA_RSRC_GRP_CFG_SRC_GRP_SPECIAL_VALID_SHFT 0x0
-#define HWIO_IPA_PIPELINE_DISABLE_ADDR (IPA_CFG_REG_BASE + 0x000005a8)
-#define HWIO_IPA_PIPELINE_DISABLE_PHYS (IPA_CFG_REG_BASE_PHYS + 0x000005a8)
-#define HWIO_IPA_PIPELINE_DISABLE_OFFS (IPA_CFG_REG_BASE_OFFS + 0x000005a8)
-#define HWIO_IPA_PIPELINE_DISABLE_RMSK 0x8
-#define HWIO_IPA_PIPELINE_DISABLE_ATTR 0x3
-#define HWIO_IPA_PIPELINE_DISABLE_IN in_dword_masked( \
-		HWIO_IPA_PIPELINE_DISABLE_ADDR,	\
-		HWIO_IPA_PIPELINE_DISABLE_RMSK)
-#define HWIO_IPA_PIPELINE_DISABLE_INM(m) in_dword_masked( \
-		HWIO_IPA_PIPELINE_DISABLE_ADDR,	\
-		m)
-#define HWIO_IPA_PIPELINE_DISABLE_OUT(v) out_dword( \
-		HWIO_IPA_PIPELINE_DISABLE_ADDR,	\
-		v)
-#define HWIO_IPA_PIPELINE_DISABLE_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_PIPELINE_DISABLE_ADDR,	\
-		m, \
-		v, \
-		HWIO_IPA_PIPELINE_DISABLE_IN)
-#define HWIO_IPA_PIPELINE_DISABLE_RX_CMDQ_SPLITTER_DIS_BMSK 0x8
-#define HWIO_IPA_PIPELINE_DISABLE_RX_CMDQ_SPLITTER_DIS_SHFT 0x3
-#define HWIO_IPA_AXI_CFG_ADDR (IPA_CFG_REG_BASE + 0x000005ac)
-#define HWIO_IPA_AXI_CFG_PHYS (IPA_CFG_REG_BASE_PHYS + 0x000005ac)
-#define HWIO_IPA_AXI_CFG_OFFS (IPA_CFG_REG_BASE_OFFS + 0x000005ac)
-#define HWIO_IPA_STAT_QUOTA_BASE_n_ADDR(n) (IPA_CFG_REG_BASE + \
-					    0x00000700 + 0x4 * (n))
-#define HWIO_IPA_STAT_QUOTA_BASE_n_PHYS(n) (IPA_CFG_REG_BASE_PHYS + \
-					    0x00000700 + 0x4 * (n))
-#define HWIO_IPA_STAT_QUOTA_BASE_n_OFFS(n) (IPA_CFG_REG_BASE_OFFS + \
-					    0x00000700 + 0x4 * (n))
-#define HWIO_IPA_STAT_QUOTA_MASK_n_ADDR(n) (IPA_CFG_REG_BASE + \
-					    0x00000708 + 0x4 * (n))
-#define HWIO_IPA_STAT_QUOTA_MASK_n_PHYS(n) (IPA_CFG_REG_BASE_PHYS + \
-					    0x00000708 + 0x4 * (n))
-#define HWIO_IPA_STAT_QUOTA_MASK_n_OFFS(n) (IPA_CFG_REG_BASE_OFFS + \
-					    0x00000708 + 0x4 * (n))
-#define HWIO_IPA_STAT_TETHERING_BASE_n_ADDR(n) (IPA_CFG_REG_BASE + \
-						0x00000710 + 0x4 * (n))
-#define HWIO_IPA_STAT_TETHERING_BASE_n_PHYS(n) (IPA_CFG_REG_BASE_PHYS +	\
-						0x00000710 + 0x4 * (n))
-#define HWIO_IPA_STAT_TETHERING_BASE_n_OFFS(n) (IPA_CFG_REG_BASE_OFFS +	\
-						0x00000710 + 0x4 * (n))
-#define HWIO_IPA_STAT_TETHERING_MASK_n_ADDR(n) (IPA_CFG_REG_BASE + \
-						0x00000718 + 0x4 * (n))
-#define HWIO_IPA_STAT_TETHERING_MASK_n_PHYS(n) (IPA_CFG_REG_BASE_PHYS +	\
-						0x00000718 + 0x4 * (n))
-#define HWIO_IPA_STAT_TETHERING_MASK_n_OFFS(n) (IPA_CFG_REG_BASE_OFFS +	\
-						0x00000718 + 0x4 * (n))
-#define HWIO_IPA_STAT_FILTER_IPV4_BASE_ADDR (IPA_CFG_REG_BASE + 0x00000720)
-#define HWIO_IPA_STAT_FILTER_IPV4_BASE_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					     0x00000720)
-#define HWIO_IPA_STAT_FILTER_IPV4_BASE_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					     0x00000720)
-#define HWIO_IPA_STAT_FILTER_IPV6_BASE_ADDR (IPA_CFG_REG_BASE + 0x00000724)
-#define HWIO_IPA_STAT_FILTER_IPV6_BASE_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					     0x00000724)
-#define HWIO_IPA_STAT_FILTER_IPV6_BASE_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					     0x00000724)
-#define HWIO_IPA_STAT_ROUTER_IPV4_BASE_ADDR (IPA_CFG_REG_BASE + 0x00000728)
-#define HWIO_IPA_STAT_ROUTER_IPV4_BASE_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					     0x00000728)
-#define HWIO_IPA_STAT_ROUTER_IPV4_BASE_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					     0x00000728)
-#define HWIO_IPA_STAT_ROUTER_IPV6_BASE_ADDR (IPA_CFG_REG_BASE + 0x0000072c)
-#define HWIO_IPA_STAT_ROUTER_IPV6_BASE_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					     0x0000072c)
-#define HWIO_IPA_STAT_ROUTER_IPV6_BASE_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					     0x0000072c)
-#define HWIO_IPA_STAT_DROP_CNT_BASE_n_ADDR(n) (IPA_CFG_REG_BASE + \
-					       0x00000750 + 0x4 * (n))
-#define HWIO_IPA_STAT_DROP_CNT_BASE_n_PHYS(n) (IPA_CFG_REG_BASE_PHYS + \
-					       0x00000750 + 0x4 * (n))
-#define HWIO_IPA_STAT_DROP_CNT_BASE_n_OFFS(n) (IPA_CFG_REG_BASE_OFFS + \
-					       0x00000750 + 0x4 * (n))
-#define HWIO_IPA_STAT_DROP_CNT_MASK_n_ADDR(n) (IPA_CFG_REG_BASE + \
-					       0x00000758 + 0x4 * (n))
-#define HWIO_IPA_STAT_DROP_CNT_MASK_n_PHYS(n) (IPA_CFG_REG_BASE_PHYS + \
-					       0x00000758 + 0x4 * (n))
-#define HWIO_IPA_STAT_DROP_CNT_MASK_n_OFFS(n) (IPA_CFG_REG_BASE_OFFS + \
-					       0x00000758 + 0x4 * (n))
-#define HWIO_IPA_ENDP_INIT_CTRL_n_ADDR(n) (IPA_CFG_REG_BASE + 0x00000800 + \
-					   0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_CTRL_n_PHYS(n) (IPA_CFG_REG_BASE_PHYS + \
-					   0x00000800 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_CTRL_n_OFFS(n) (IPA_CFG_REG_BASE_OFFS + \
-					   0x00000800 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_CTRL_n_RMSK 0x3
-#define HWIO_IPA_ENDP_INIT_CTRL_n_MAXn 30
-#define HWIO_IPA_ENDP_INIT_CTRL_n_ATTR 0x3
-#define HWIO_IPA_ENDP_INIT_CTRL_n_INI(n) in_dword_masked( \
-		HWIO_IPA_ENDP_INIT_CTRL_n_ADDR(n), \
-		HWIO_IPA_ENDP_INIT_CTRL_n_RMSK)
-#define HWIO_IPA_ENDP_INIT_CTRL_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_ENDP_INIT_CTRL_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_ENDP_INIT_CTRL_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_ENDP_INIT_CTRL_n_ADDR(n), \
-		val)
-#define HWIO_IPA_ENDP_INIT_CTRL_n_OUTMI(n, mask, val) out_dword_masked_ns( \
-		HWIO_IPA_ENDP_INIT_CTRL_n_ADDR(n), \
-		mask, \
-		val, \
-		HWIO_IPA_ENDP_INIT_CTRL_n_INI(n))
-#define HWIO_IPA_ENDP_INIT_CTRL_n_ENDP_DELAY_BMSK 0x2
-#define HWIO_IPA_ENDP_INIT_CTRL_n_ENDP_DELAY_SHFT 0x1
-#define HWIO_IPA_ENDP_INIT_CTRL_n_ENDP_SUSPEND_BMSK 0x1
-#define HWIO_IPA_ENDP_INIT_CTRL_n_ENDP_SUSPEND_SHFT 0x0
-#define HWIO_IPA_ENDP_INIT_CTRL_SCND_n_ADDR(n) (IPA_CFG_REG_BASE + \
-						0x00000804 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_CTRL_SCND_n_PHYS(n) (IPA_CFG_REG_BASE_PHYS +	\
-						0x00000804 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_CTRL_SCND_n_OFFS(n) (IPA_CFG_REG_BASE_OFFS +	\
-						0x00000804 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_CTRL_SCND_n_RMSK 0x2
-#define HWIO_IPA_ENDP_INIT_CTRL_SCND_n_MAXn 30
-#define HWIO_IPA_ENDP_INIT_CTRL_SCND_n_ATTR 0x3
-#define HWIO_IPA_ENDP_INIT_CTRL_SCND_n_INI(n) in_dword_masked( \
-		HWIO_IPA_ENDP_INIT_CTRL_SCND_n_ADDR(n),	\
-		HWIO_IPA_ENDP_INIT_CTRL_SCND_n_RMSK)
-#define HWIO_IPA_ENDP_INIT_CTRL_SCND_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_ENDP_INIT_CTRL_SCND_n_ADDR(n),	\
-		mask)
-#define HWIO_IPA_ENDP_INIT_CTRL_SCND_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_ENDP_INIT_CTRL_SCND_n_ADDR(n),	\
-		val)
-#define HWIO_IPA_ENDP_INIT_CTRL_SCND_n_OUTMI(n, mask, \
-					     val) out_dword_masked_ns( \
-		HWIO_IPA_ENDP_INIT_CTRL_SCND_n_ADDR( \
-			n), \
-		mask, \
-		val, \
-		HWIO_IPA_ENDP_INIT_CTRL_SCND_n_INI(n))
-#define HWIO_IPA_ENDP_INIT_CTRL_SCND_n_ENDP_DELAY_BMSK 0x2
-#define HWIO_IPA_ENDP_INIT_CTRL_SCND_n_ENDP_DELAY_SHFT 0x1
-#define HWIO_IPA_ENDP_INIT_CFG_n_ADDR(n) (IPA_CFG_REG_BASE + 0x00000808 + \
-					  0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_CFG_n_PHYS(n) (IPA_CFG_REG_BASE_PHYS + \
-					  0x00000808 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_CFG_n_OFFS(n) (IPA_CFG_REG_BASE_OFFS + \
-					  0x00000808 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_CFG_n_RMSK 0x17f
-#define HWIO_IPA_ENDP_INIT_CFG_n_MAXn 30
-#define HWIO_IPA_ENDP_INIT_CFG_n_ATTR 0x3
-#define HWIO_IPA_ENDP_INIT_CFG_n_INI(n) in_dword_masked( \
-		HWIO_IPA_ENDP_INIT_CFG_n_ADDR(n), \
-		HWIO_IPA_ENDP_INIT_CFG_n_RMSK)
-#define HWIO_IPA_ENDP_INIT_CFG_n_INMI(n, mask) in_dword_masked(	\
-		HWIO_IPA_ENDP_INIT_CFG_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_ENDP_INIT_CFG_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_ENDP_INIT_CFG_n_ADDR(n), \
-		val)
-#define HWIO_IPA_ENDP_INIT_CFG_n_OUTMI(n, mask, val) out_dword_masked_ns( \
-		HWIO_IPA_ENDP_INIT_CFG_n_ADDR(n), \
-		mask, \
-		val, \
-		HWIO_IPA_ENDP_INIT_CFG_n_INI(n))
-#define HWIO_IPA_ENDP_INIT_CFG_n_GEN_QMB_MASTER_SEL_BMSK 0x100
-#define HWIO_IPA_ENDP_INIT_CFG_n_GEN_QMB_MASTER_SEL_SHFT 0x8
-#define HWIO_IPA_ENDP_INIT_CFG_n_CS_METADATA_HDR_OFFSET_BMSK 0x78
-#define HWIO_IPA_ENDP_INIT_CFG_n_CS_METADATA_HDR_OFFSET_SHFT 0x3
-#define HWIO_IPA_ENDP_INIT_CFG_n_CS_OFFLOAD_EN_BMSK 0x6
-#define HWIO_IPA_ENDP_INIT_CFG_n_CS_OFFLOAD_EN_SHFT 0x1
-#define HWIO_IPA_ENDP_INIT_CFG_n_FRAG_OFFLOAD_EN_BMSK 0x1
-#define HWIO_IPA_ENDP_INIT_CFG_n_FRAG_OFFLOAD_EN_SHFT 0x0
-#define HWIO_IPA_ENDP_INIT_NAT_n_ADDR(n) (IPA_CFG_REG_BASE + 0x0000080c + \
-					  0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_NAT_n_PHYS(n) (IPA_CFG_REG_BASE_PHYS + \
-					  0x0000080c + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_NAT_n_OFFS(n) (IPA_CFG_REG_BASE_OFFS + \
-					  0x0000080c + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_NAT_n_RMSK 0x3
-#define HWIO_IPA_ENDP_INIT_NAT_n_MAXn 12
-#define HWIO_IPA_ENDP_INIT_NAT_n_ATTR 0x3
-#define HWIO_IPA_ENDP_INIT_NAT_n_INI(n) in_dword_masked( \
-		HWIO_IPA_ENDP_INIT_NAT_n_ADDR(n), \
-		HWIO_IPA_ENDP_INIT_NAT_n_RMSK)
-#define HWIO_IPA_ENDP_INIT_NAT_n_INMI(n, mask) in_dword_masked(	\
-		HWIO_IPA_ENDP_INIT_NAT_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_ENDP_INIT_NAT_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_ENDP_INIT_NAT_n_ADDR(n), \
-		val)
-#define HWIO_IPA_ENDP_INIT_NAT_n_OUTMI(n, mask, val) out_dword_masked_ns( \
-		HWIO_IPA_ENDP_INIT_NAT_n_ADDR(n), \
-		mask, \
-		val, \
-		HWIO_IPA_ENDP_INIT_NAT_n_INI(n))
-#define HWIO_IPA_ENDP_INIT_NAT_n_NAT_EN_BMSK 0x3
-#define HWIO_IPA_ENDP_INIT_NAT_n_NAT_EN_SHFT 0x0
-#define HWIO_IPA_ENDP_INIT_HDR_n_ADDR(n) (IPA_CFG_REG_BASE + 0x00000810 + \
-					  0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_HDR_n_PHYS(n) (IPA_CFG_REG_BASE_PHYS + \
-					  0x00000810 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_HDR_n_OFFS(n) (IPA_CFG_REG_BASE_OFFS + \
-					  0x00000810 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_HDR_n_RMSK 0xffffffff
-#define HWIO_IPA_ENDP_INIT_HDR_n_MAXn 30
-#define HWIO_IPA_ENDP_INIT_HDR_n_ATTR 0x3
-#define HWIO_IPA_ENDP_INIT_HDR_n_INI(n) in_dword_masked( \
-		HWIO_IPA_ENDP_INIT_HDR_n_ADDR(n), \
-		HWIO_IPA_ENDP_INIT_HDR_n_RMSK)
-#define HWIO_IPA_ENDP_INIT_HDR_n_INMI(n, mask) in_dword_masked(	\
-		HWIO_IPA_ENDP_INIT_HDR_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_ENDP_INIT_HDR_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_ENDP_INIT_HDR_n_ADDR(n), \
-		val)
-#define HWIO_IPA_ENDP_INIT_HDR_n_OUTMI(n, mask, val) out_dword_masked_ns( \
-		HWIO_IPA_ENDP_INIT_HDR_n_ADDR(n), \
-		mask, \
-		val, \
-		HWIO_IPA_ENDP_INIT_HDR_n_INI(n))
-#define HWIO_IPA_ENDP_INIT_HDR_n_HDR_OFST_METADATA_MSB_BMSK 0xc0000000
-#define HWIO_IPA_ENDP_INIT_HDR_n_HDR_OFST_METADATA_MSB_SHFT 0x1e
-#define HWIO_IPA_ENDP_INIT_HDR_n_HDR_LEN_MSB_BMSK 0x30000000
-#define HWIO_IPA_ENDP_INIT_HDR_n_HDR_LEN_MSB_SHFT 0x1c
-#define HWIO_IPA_ENDP_INIT_HDR_n_HDR_LEN_INC_DEAGG_HDR_BMSK 0x8000000
-#define HWIO_IPA_ENDP_INIT_HDR_n_HDR_LEN_INC_DEAGG_HDR_SHFT 0x1b
-#define HWIO_IPA_ENDP_INIT_HDR_n_HDR_A5_MUX_BMSK 0x4000000
-#define HWIO_IPA_ENDP_INIT_HDR_n_HDR_A5_MUX_SHFT 0x1a
-#define HWIO_IPA_ENDP_INIT_HDR_n_HDR_OFST_PKT_SIZE_BMSK 0x3f00000
-#define HWIO_IPA_ENDP_INIT_HDR_n_HDR_OFST_PKT_SIZE_SHFT 0x14
-#define HWIO_IPA_ENDP_INIT_HDR_n_HDR_OFST_PKT_SIZE_VALID_BMSK 0x80000
-#define HWIO_IPA_ENDP_INIT_HDR_n_HDR_OFST_PKT_SIZE_VALID_SHFT 0x13
-#define HWIO_IPA_ENDP_INIT_HDR_n_HDR_ADDITIONAL_CONST_LEN_BMSK 0x7e000
-#define HWIO_IPA_ENDP_INIT_HDR_n_HDR_ADDITIONAL_CONST_LEN_SHFT 0xd
-#define HWIO_IPA_ENDP_INIT_HDR_n_HDR_OFST_METADATA_BMSK 0x1f80
-#define HWIO_IPA_ENDP_INIT_HDR_n_HDR_OFST_METADATA_SHFT 0x7
-#define HWIO_IPA_ENDP_INIT_HDR_n_HDR_OFST_METADATA_VALID_BMSK 0x40
-#define HWIO_IPA_ENDP_INIT_HDR_n_HDR_OFST_METADATA_VALID_SHFT 0x6
-#define HWIO_IPA_ENDP_INIT_HDR_n_HDR_LEN_BMSK 0x3f
-#define HWIO_IPA_ENDP_INIT_HDR_n_HDR_LEN_SHFT 0x0
-#define HWIO_IPA_ENDP_INIT_HDR_EXT_n_ADDR(n) (IPA_CFG_REG_BASE + \
-					      0x00000814 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_HDR_EXT_n_PHYS(n) (IPA_CFG_REG_BASE_PHYS + \
-					      0x00000814 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_HDR_EXT_n_OFFS(n) (IPA_CFG_REG_BASE_OFFS + \
-					      0x00000814 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_HDR_EXT_n_RMSK 0x3f3fff
-#define HWIO_IPA_ENDP_INIT_HDR_EXT_n_MAXn 30
-#define HWIO_IPA_ENDP_INIT_HDR_EXT_n_ATTR 0x3
-#define HWIO_IPA_ENDP_INIT_HDR_EXT_n_INI(n) in_dword_masked( \
-		HWIO_IPA_ENDP_INIT_HDR_EXT_n_ADDR(n), \
-		HWIO_IPA_ENDP_INIT_HDR_EXT_n_RMSK)
-#define HWIO_IPA_ENDP_INIT_HDR_EXT_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_ENDP_INIT_HDR_EXT_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_ENDP_INIT_HDR_EXT_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_ENDP_INIT_HDR_EXT_n_ADDR(n), \
-		val)
-#define HWIO_IPA_ENDP_INIT_HDR_EXT_n_OUTMI(n, mask, \
-					   val) out_dword_masked_ns( \
-		HWIO_IPA_ENDP_INIT_HDR_EXT_n_ADDR( \
-			n), \
-		mask, \
-		val, \
-		HWIO_IPA_ENDP_INIT_HDR_EXT_n_INI(n))
-#define HWIO_IPA_ENDP_INIT_HDR_EXT_n_HDR_ADDITIONAL_CONST_LEN_MSB_BMSK \
-	0x300000
-#define HWIO_IPA_ENDP_INIT_HDR_EXT_n_HDR_ADDITIONAL_CONST_LEN_MSB_SHFT \
-	0x14
-#define HWIO_IPA_ENDP_INIT_HDR_EXT_n_HDR_OFST_PKT_SIZE_MSB_BMSK 0xc0000
-#define HWIO_IPA_ENDP_INIT_HDR_EXT_n_HDR_OFST_PKT_SIZE_MSB_SHFT 0x12
-#define HWIO_IPA_ENDP_INIT_HDR_EXT_n_HDR_TOTAL_LEN_OR_PAD_OFFSET_MSB_BMSK \
-	0x30000
-#define HWIO_IPA_ENDP_INIT_HDR_EXT_n_HDR_TOTAL_LEN_OR_PAD_OFFSET_MSB_SHFT \
-	0x10
-#define HWIO_IPA_ENDP_INIT_HDR_EXT_n_HDR_PAD_TO_ALIGNMENT_BMSK 0x3c00
-#define HWIO_IPA_ENDP_INIT_HDR_EXT_n_HDR_PAD_TO_ALIGNMENT_SHFT 0xa
-#define HWIO_IPA_ENDP_INIT_HDR_EXT_n_HDR_TOTAL_LEN_OR_PAD_OFFSET_BMSK \
-	0x3f0
-#define HWIO_IPA_ENDP_INIT_HDR_EXT_n_HDR_TOTAL_LEN_OR_PAD_OFFSET_SHFT 0x4
-#define HWIO_IPA_ENDP_INIT_HDR_EXT_n_HDR_PAYLOAD_LEN_INC_PADDING_BMSK 0x8
-#define HWIO_IPA_ENDP_INIT_HDR_EXT_n_HDR_PAYLOAD_LEN_INC_PADDING_SHFT 0x3
-#define HWIO_IPA_ENDP_INIT_HDR_EXT_n_HDR_TOTAL_LEN_OR_PAD_BMSK 0x4
-#define HWIO_IPA_ENDP_INIT_HDR_EXT_n_HDR_TOTAL_LEN_OR_PAD_SHFT 0x2
-#define HWIO_IPA_ENDP_INIT_HDR_EXT_n_HDR_TOTAL_LEN_OR_PAD_VALID_BMSK 0x2
-#define HWIO_IPA_ENDP_INIT_HDR_EXT_n_HDR_TOTAL_LEN_OR_PAD_VALID_SHFT 0x1
-#define HWIO_IPA_ENDP_INIT_HDR_METADATA_MASK_n_ADDR(n) (IPA_CFG_REG_BASE + \
-							0x00000818 + \
-							0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_HDR_METADATA_MASK_n_PHYS(n) ( \
-		IPA_CFG_REG_BASE_PHYS + 0x00000818 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_HDR_METADATA_MASK_n_OFFS(n) ( \
-		IPA_CFG_REG_BASE_OFFS + 0x00000818 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_HDR_METADATA_MASK_n_RMSK 0xffffffff
-#define HWIO_IPA_ENDP_INIT_HDR_METADATA_MASK_n_MAXn 30
-#define HWIO_IPA_ENDP_INIT_HDR_METADATA_MASK_n_ATTR 0x3
-#define HWIO_IPA_ENDP_INIT_HDR_METADATA_MASK_n_INI(n) in_dword_masked( \
-		HWIO_IPA_ENDP_INIT_HDR_METADATA_MASK_n_ADDR(n),	\
-		HWIO_IPA_ENDP_INIT_HDR_METADATA_MASK_n_RMSK)
-#define HWIO_IPA_ENDP_INIT_HDR_METADATA_MASK_n_INMI(n, \
-						    mask) in_dword_masked( \
-		HWIO_IPA_ENDP_INIT_HDR_METADATA_MASK_n_ADDR( \
-			n), \
-		mask)
-#define HWIO_IPA_ENDP_INIT_HDR_METADATA_MASK_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_ENDP_INIT_HDR_METADATA_MASK_n_ADDR(n),	\
-		val)
-#define HWIO_IPA_ENDP_INIT_HDR_METADATA_MASK_n_OUTMI(n, mask, \
-						     val) \
-	out_dword_masked_ns( \
-		HWIO_IPA_ENDP_INIT_HDR_METADATA_MASK_n_ADDR(n),	\
-		mask, \
-		val, \
-		HWIO_IPA_ENDP_INIT_HDR_METADATA_MASK_n_INI(n))
-#define HWIO_IPA_ENDP_INIT_HDR_METADATA_MASK_n_METADATA_MASK_BMSK \
-	0xffffffff
-#define HWIO_IPA_ENDP_INIT_HDR_METADATA_MASK_n_METADATA_MASK_SHFT 0x0
-#define HWIO_IPA_ENDP_INIT_HDR_METADATA_n_ADDR(n) (IPA_CFG_REG_BASE + \
-						   0x0000081c + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_HDR_METADATA_n_PHYS(n) (IPA_CFG_REG_BASE_PHYS + \
-						   0x0000081c + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_HDR_METADATA_n_OFFS(n) (IPA_CFG_REG_BASE_OFFS + \
-						   0x0000081c + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_HDR_METADATA_n_RMSK 0xffffffff
-#define HWIO_IPA_ENDP_INIT_HDR_METADATA_n_MAXn 12
-#define HWIO_IPA_ENDP_INIT_HDR_METADATA_n_ATTR 0x3
-#define HWIO_IPA_ENDP_INIT_HDR_METADATA_n_INI(n) in_dword_masked( \
-		HWIO_IPA_ENDP_INIT_HDR_METADATA_n_ADDR(n), \
-		HWIO_IPA_ENDP_INIT_HDR_METADATA_n_RMSK)
-#define HWIO_IPA_ENDP_INIT_HDR_METADATA_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_ENDP_INIT_HDR_METADATA_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_ENDP_INIT_HDR_METADATA_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_ENDP_INIT_HDR_METADATA_n_ADDR(n), \
-		val)
-#define HWIO_IPA_ENDP_INIT_HDR_METADATA_n_OUTMI(n, mask, \
-						val) out_dword_masked_ns( \
-		HWIO_IPA_ENDP_INIT_HDR_METADATA_n_ADDR(	\
-			n), \
-		mask, \
-		val, \
-		HWIO_IPA_ENDP_INIT_HDR_METADATA_n_INI(n))
-#define HWIO_IPA_ENDP_INIT_HDR_METADATA_n_METADATA_BMSK 0xffffffff
-#define HWIO_IPA_ENDP_INIT_HDR_METADATA_n_METADATA_SHFT 0x0
-#define HWIO_IPA_ENDP_INIT_MODE_n_ADDR(n) (IPA_CFG_REG_BASE + 0x00000820 + \
-					   0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_MODE_n_PHYS(n) (IPA_CFG_REG_BASE_PHYS + \
-					   0x00000820 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_MODE_n_OFFS(n) (IPA_CFG_REG_BASE_OFFS + \
-					   0x00000820 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_MODE_n_RMSK 0x3ffff1ff
-#define HWIO_IPA_ENDP_INIT_MODE_n_MAXn 12
-#define HWIO_IPA_ENDP_INIT_MODE_n_ATTR 0x3
-#define HWIO_IPA_ENDP_INIT_MODE_n_INI(n) in_dword_masked( \
-		HWIO_IPA_ENDP_INIT_MODE_n_ADDR(n), \
-		HWIO_IPA_ENDP_INIT_MODE_n_RMSK)
-#define HWIO_IPA_ENDP_INIT_MODE_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_ENDP_INIT_MODE_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_ENDP_INIT_MODE_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_ENDP_INIT_MODE_n_ADDR(n), \
-		val)
-#define HWIO_IPA_ENDP_INIT_MODE_n_OUTMI(n, mask, val) out_dword_masked_ns( \
-		HWIO_IPA_ENDP_INIT_MODE_n_ADDR(n), \
-		mask, \
-		val, \
-		HWIO_IPA_ENDP_INIT_MODE_n_INI(n))
-#define HWIO_IPA_ENDP_INIT_MODE_n_PAD_EN_BMSK 0x20000000
-#define HWIO_IPA_ENDP_INIT_MODE_n_PAD_EN_SHFT 0x1d
-#define HWIO_IPA_ENDP_INIT_MODE_n_PIPE_REPLICATE_EN_BMSK 0x10000000
-#define HWIO_IPA_ENDP_INIT_MODE_n_PIPE_REPLICATE_EN_SHFT 0x1c
-#define HWIO_IPA_ENDP_INIT_MODE_n_BYTE_THRESHOLD_BMSK 0xffff000
-#define HWIO_IPA_ENDP_INIT_MODE_n_BYTE_THRESHOLD_SHFT 0xc
-#define HWIO_IPA_ENDP_INIT_MODE_n_DEST_PIPE_INDEX_BMSK 0x1f0
-#define HWIO_IPA_ENDP_INIT_MODE_n_DEST_PIPE_INDEX_SHFT 0x4
-#define HWIO_IPA_ENDP_INIT_MODE_n_DCPH_ENABLE_BMSK 0x8
-#define HWIO_IPA_ENDP_INIT_MODE_n_DCPH_ENABLE_SHFT 0x3
-#define HWIO_IPA_ENDP_INIT_MODE_n_MODE_BMSK 0x7
-#define HWIO_IPA_ENDP_INIT_MODE_n_MODE_SHFT 0x0
-#define HWIO_IPA_ENDP_INIT_AGGR_n_ADDR(n) (IPA_CFG_REG_BASE + 0x00000824 + \
-					   0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_AGGR_n_PHYS(n) (IPA_CFG_REG_BASE_PHYS + \
-					   0x00000824 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_AGGR_n_OFFS(n) (IPA_CFG_REG_BASE_OFFS + \
-					   0x00000824 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_AGGR_n_RMSK 0xdfff7ff
-#define HWIO_IPA_ENDP_INIT_AGGR_n_MAXn 30
-#define HWIO_IPA_ENDP_INIT_AGGR_n_ATTR 0x3
-#define HWIO_IPA_ENDP_INIT_AGGR_n_INI(n) in_dword_masked( \
-		HWIO_IPA_ENDP_INIT_AGGR_n_ADDR(n), \
-		HWIO_IPA_ENDP_INIT_AGGR_n_RMSK)
-#define HWIO_IPA_ENDP_INIT_AGGR_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_ENDP_INIT_AGGR_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_ENDP_INIT_AGGR_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_ENDP_INIT_AGGR_n_ADDR(n), \
-		val)
-#define HWIO_IPA_ENDP_INIT_AGGR_n_OUTMI(n, mask, val) out_dword_masked_ns( \
-		HWIO_IPA_ENDP_INIT_AGGR_n_ADDR(n), \
-		mask, \
-		val, \
-		HWIO_IPA_ENDP_INIT_AGGR_n_INI(n))
-#define HWIO_IPA_ENDP_INIT_AGGR_n_AGGR_GRAN_SEL_BMSK 0x8000000
-#define HWIO_IPA_ENDP_INIT_AGGR_n_AGGR_GRAN_SEL_SHFT 0x1b
-#define HWIO_IPA_ENDP_INIT_AGGR_n_AGGR_HARD_BYTE_LIMIT_ENABLE_BMSK \
-	0x4000000
-#define HWIO_IPA_ENDP_INIT_AGGR_n_AGGR_HARD_BYTE_LIMIT_ENABLE_SHFT 0x1a
-#define HWIO_IPA_ENDP_INIT_AGGR_n_AGGR_FORCE_CLOSE_BMSK 0x1000000
-#define HWIO_IPA_ENDP_INIT_AGGR_n_AGGR_FORCE_CLOSE_SHFT 0x18
-#define HWIO_IPA_ENDP_INIT_AGGR_n_AGGR_SW_EOF_ACTIVE_BMSK 0x800000
-#define HWIO_IPA_ENDP_INIT_AGGR_n_AGGR_SW_EOF_ACTIVE_SHFT 0x17
-#define HWIO_IPA_ENDP_INIT_AGGR_n_AGGR_PKT_LIMIT_BMSK 0x7e0000
-#define HWIO_IPA_ENDP_INIT_AGGR_n_AGGR_PKT_LIMIT_SHFT 0x11
-#define HWIO_IPA_ENDP_INIT_AGGR_n_AGGR_TIME_LIMIT_BMSK 0x1f000
-#define HWIO_IPA_ENDP_INIT_AGGR_n_AGGR_TIME_LIMIT_SHFT 0xc
-#define HWIO_IPA_ENDP_INIT_AGGR_n_AGGR_BYTE_LIMIT_BMSK 0x7e0
-#define HWIO_IPA_ENDP_INIT_AGGR_n_AGGR_BYTE_LIMIT_SHFT 0x5
-#define HWIO_IPA_ENDP_INIT_AGGR_n_AGGR_TYPE_BMSK 0x1c
-#define HWIO_IPA_ENDP_INIT_AGGR_n_AGGR_TYPE_SHFT 0x2
-#define HWIO_IPA_ENDP_INIT_AGGR_n_AGGR_EN_BMSK 0x3
-#define HWIO_IPA_ENDP_INIT_AGGR_n_AGGR_EN_SHFT 0x0
-#define HWIO_IPA_ENDP_INIT_HOL_BLOCK_EN_n_ADDR(n) (IPA_CFG_REG_BASE + \
-						   0x0000082c + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_HOL_BLOCK_EN_n_PHYS(n) (IPA_CFG_REG_BASE_PHYS + \
-						   0x0000082c + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_HOL_BLOCK_EN_n_OFFS(n) (IPA_CFG_REG_BASE_OFFS + \
-						   0x0000082c + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_HOL_BLOCK_EN_n_RMSK 0x1
-#define HWIO_IPA_ENDP_INIT_HOL_BLOCK_EN_n_MAXn 30
-#define HWIO_IPA_ENDP_INIT_HOL_BLOCK_EN_n_ATTR 0x3
-#define HWIO_IPA_ENDP_INIT_HOL_BLOCK_EN_n_INI(n) in_dword_masked( \
-		HWIO_IPA_ENDP_INIT_HOL_BLOCK_EN_n_ADDR(n), \
-		HWIO_IPA_ENDP_INIT_HOL_BLOCK_EN_n_RMSK)
-#define HWIO_IPA_ENDP_INIT_HOL_BLOCK_EN_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_ENDP_INIT_HOL_BLOCK_EN_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_ENDP_INIT_HOL_BLOCK_EN_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_ENDP_INIT_HOL_BLOCK_EN_n_ADDR(n), \
-		val)
-#define HWIO_IPA_ENDP_INIT_HOL_BLOCK_EN_n_OUTMI(n, mask, \
-						val) out_dword_masked_ns( \
-		HWIO_IPA_ENDP_INIT_HOL_BLOCK_EN_n_ADDR(	\
-			n), \
-		mask, \
-		val, \
-		HWIO_IPA_ENDP_INIT_HOL_BLOCK_EN_n_INI(n))
-#define HWIO_IPA_ENDP_INIT_HOL_BLOCK_EN_n_EN_BMSK 0x1
-#define HWIO_IPA_ENDP_INIT_HOL_BLOCK_EN_n_EN_SHFT 0x0
-#define HWIO_IPA_ENDP_INIT_HOL_BLOCK_TIMER_n_ADDR(n) (IPA_CFG_REG_BASE + \
-						      0x00000830 + 0x70 * \
-						      (n))
-#define HWIO_IPA_ENDP_INIT_HOL_BLOCK_TIMER_n_PHYS(n) ( \
-		IPA_CFG_REG_BASE_PHYS + 0x00000830 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_HOL_BLOCK_TIMER_n_OFFS(n) ( \
-		IPA_CFG_REG_BASE_OFFS + 0x00000830 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_HOL_BLOCK_TIMER_n_RMSK 0x11f
-#define HWIO_IPA_ENDP_INIT_HOL_BLOCK_TIMER_n_MAXn 30
-#define HWIO_IPA_ENDP_INIT_HOL_BLOCK_TIMER_n_ATTR 0x3
-#define HWIO_IPA_ENDP_INIT_HOL_BLOCK_TIMER_n_INI(n) in_dword_masked( \
-		HWIO_IPA_ENDP_INIT_HOL_BLOCK_TIMER_n_ADDR(n), \
-		HWIO_IPA_ENDP_INIT_HOL_BLOCK_TIMER_n_RMSK)
-#define HWIO_IPA_ENDP_INIT_HOL_BLOCK_TIMER_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_ENDP_INIT_HOL_BLOCK_TIMER_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_ENDP_INIT_HOL_BLOCK_TIMER_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_ENDP_INIT_HOL_BLOCK_TIMER_n_ADDR(n), \
-		val)
-#define HWIO_IPA_ENDP_INIT_HOL_BLOCK_TIMER_n_OUTMI(n, mask, \
-						   val)	\
-	out_dword_masked_ns(HWIO_IPA_ENDP_INIT_HOL_BLOCK_TIMER_n_ADDR( \
-				    n),	\
-			    mask, val, \
-			    HWIO_IPA_ENDP_INIT_HOL_BLOCK_TIMER_n_INI(n))
-#define HWIO_IPA_ENDP_INIT_HOL_BLOCK_TIMER_n_GRAN_SEL_BMSK 0x100
-#define HWIO_IPA_ENDP_INIT_HOL_BLOCK_TIMER_n_GRAN_SEL_SHFT 0x8
-#define HWIO_IPA_ENDP_INIT_HOL_BLOCK_TIMER_n_TIME_LIMIT_BMSK 0x1f
-#define HWIO_IPA_ENDP_INIT_HOL_BLOCK_TIMER_n_TIME_LIMIT_SHFT 0x0
-#define HWIO_IPA_ENDP_INIT_DEAGGR_n_ADDR(n) (IPA_CFG_REG_BASE +	\
-					     0x00000834 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_DEAGGR_n_PHYS(n) (IPA_CFG_REG_BASE_PHYS + \
-					     0x00000834 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_DEAGGR_n_OFFS(n) (IPA_CFG_REG_BASE_OFFS + \
-					     0x00000834 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_DEAGGR_n_RMSK 0xffff7fff
-#define HWIO_IPA_ENDP_INIT_DEAGGR_n_MAXn 12
-#define HWIO_IPA_ENDP_INIT_DEAGGR_n_ATTR 0x3
-#define HWIO_IPA_ENDP_INIT_DEAGGR_n_INI(n) in_dword_masked( \
-		HWIO_IPA_ENDP_INIT_DEAGGR_n_ADDR(n), \
-		HWIO_IPA_ENDP_INIT_DEAGGR_n_RMSK)
-#define HWIO_IPA_ENDP_INIT_DEAGGR_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_ENDP_INIT_DEAGGR_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_ENDP_INIT_DEAGGR_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_ENDP_INIT_DEAGGR_n_ADDR(n), \
-		val)
-#define HWIO_IPA_ENDP_INIT_DEAGGR_n_OUTMI(n, mask, \
-					  val) out_dword_masked_ns( \
-		HWIO_IPA_ENDP_INIT_DEAGGR_n_ADDR( \
-			n), \
-		mask, \
-		val, \
-		HWIO_IPA_ENDP_INIT_DEAGGR_n_INI(n))
-#define HWIO_IPA_ENDP_INIT_DEAGGR_n_MAX_PACKET_LEN_BMSK 0xffff0000
-#define HWIO_IPA_ENDP_INIT_DEAGGR_n_MAX_PACKET_LEN_SHFT 0x10
-#define HWIO_IPA_ENDP_INIT_DEAGGR_n_IGNORE_MIN_PKT_ERR_BMSK 0x4000
-#define HWIO_IPA_ENDP_INIT_DEAGGR_n_IGNORE_MIN_PKT_ERR_SHFT 0xe
-#define HWIO_IPA_ENDP_INIT_DEAGGR_n_PACKET_OFFSET_LOCATION_BMSK 0x3f00
-#define HWIO_IPA_ENDP_INIT_DEAGGR_n_PACKET_OFFSET_LOCATION_SHFT 0x8
-#define HWIO_IPA_ENDP_INIT_DEAGGR_n_PACKET_OFFSET_VALID_BMSK 0x80
-#define HWIO_IPA_ENDP_INIT_DEAGGR_n_PACKET_OFFSET_VALID_SHFT 0x7
-#define HWIO_IPA_ENDP_INIT_DEAGGR_n_SYSPIPE_ERR_DETECTION_BMSK 0x40
-#define HWIO_IPA_ENDP_INIT_DEAGGR_n_SYSPIPE_ERR_DETECTION_SHFT 0x6
-#define HWIO_IPA_ENDP_INIT_DEAGGR_n_DEAGGR_HDR_LEN_BMSK 0x3f
-#define HWIO_IPA_ENDP_INIT_DEAGGR_n_DEAGGR_HDR_LEN_SHFT 0x0
-#define HWIO_IPA_ENDP_INIT_RSRC_GRP_n_ADDR(n) (IPA_CFG_REG_BASE + \
-					       0x00000838 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_RSRC_GRP_n_PHYS(n) (IPA_CFG_REG_BASE_PHYS + \
-					       0x00000838 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_RSRC_GRP_n_OFFS(n) (IPA_CFG_REG_BASE_OFFS + \
-					       0x00000838 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_RSRC_GRP_n_RMSK 0x7
-#define HWIO_IPA_ENDP_INIT_RSRC_GRP_n_MAXn 30
-#define HWIO_IPA_ENDP_INIT_RSRC_GRP_n_ATTR 0x3
-#define HWIO_IPA_ENDP_INIT_RSRC_GRP_n_INI(n) in_dword_masked( \
-		HWIO_IPA_ENDP_INIT_RSRC_GRP_n_ADDR(n), \
-		HWIO_IPA_ENDP_INIT_RSRC_GRP_n_RMSK)
-#define HWIO_IPA_ENDP_INIT_RSRC_GRP_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_ENDP_INIT_RSRC_GRP_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_ENDP_INIT_RSRC_GRP_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_ENDP_INIT_RSRC_GRP_n_ADDR(n), \
-		val)
-#define HWIO_IPA_ENDP_INIT_RSRC_GRP_n_OUTMI(n, mask, \
-					    val) out_dword_masked_ns( \
-		HWIO_IPA_ENDP_INIT_RSRC_GRP_n_ADDR( \
-			n), \
-		mask, \
-		val, \
-		HWIO_IPA_ENDP_INIT_RSRC_GRP_n_INI(n))
-#define HWIO_IPA_ENDP_INIT_RSRC_GRP_n_RSRC_GRP_BMSK 0x7
-#define HWIO_IPA_ENDP_INIT_RSRC_GRP_n_RSRC_GRP_SHFT 0x0
-#define HWIO_IPA_ENDP_INIT_SEQ_n_ADDR(n) (IPA_CFG_REG_BASE + 0x0000083c + \
-					  0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_SEQ_n_PHYS(n) (IPA_CFG_REG_BASE_PHYS + \
-					  0x0000083c + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_SEQ_n_OFFS(n) (IPA_CFG_REG_BASE_OFFS + \
-					  0x0000083c + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_SEQ_n_RMSK 0xffff
-#define HWIO_IPA_ENDP_INIT_SEQ_n_MAXn 12
-#define HWIO_IPA_ENDP_INIT_SEQ_n_ATTR 0x3
-#define HWIO_IPA_ENDP_INIT_SEQ_n_INI(n) in_dword_masked( \
-		HWIO_IPA_ENDP_INIT_SEQ_n_ADDR(n), \
-		HWIO_IPA_ENDP_INIT_SEQ_n_RMSK)
-#define HWIO_IPA_ENDP_INIT_SEQ_n_INMI(n, mask) in_dword_masked(	\
-		HWIO_IPA_ENDP_INIT_SEQ_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_ENDP_INIT_SEQ_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_ENDP_INIT_SEQ_n_ADDR(n), \
-		val)
-#define HWIO_IPA_ENDP_INIT_SEQ_n_OUTMI(n, mask, val) out_dword_masked_ns( \
-		HWIO_IPA_ENDP_INIT_SEQ_n_ADDR(n), \
-		mask, \
-		val, \
-		HWIO_IPA_ENDP_INIT_SEQ_n_INI(n))
-#define HWIO_IPA_ENDP_INIT_SEQ_n_DPS_REP_SEQ_TYPE_BMSK 0xf000
-#define HWIO_IPA_ENDP_INIT_SEQ_n_DPS_REP_SEQ_TYPE_SHFT 0xc
-#define HWIO_IPA_ENDP_INIT_SEQ_n_HPS_REP_SEQ_TYPE_BMSK 0xf00
-#define HWIO_IPA_ENDP_INIT_SEQ_n_HPS_REP_SEQ_TYPE_SHFT 0x8
-#define HWIO_IPA_ENDP_INIT_SEQ_n_DPS_SEQ_TYPE_BMSK 0xf0
-#define HWIO_IPA_ENDP_INIT_SEQ_n_DPS_SEQ_TYPE_SHFT 0x4
-#define HWIO_IPA_ENDP_INIT_SEQ_n_HPS_SEQ_TYPE_BMSK 0xf
-#define HWIO_IPA_ENDP_INIT_SEQ_n_HPS_SEQ_TYPE_SHFT 0x0
-#define HWIO_IPA_ENDP_STATUS_n_ADDR(n) (IPA_CFG_REG_BASE + 0x00000840 +	\
-					0x70 * (n))
-#define HWIO_IPA_ENDP_STATUS_n_PHYS(n) (IPA_CFG_REG_BASE_PHYS +	\
-					0x00000840 + 0x70 * (n))
-#define HWIO_IPA_ENDP_STATUS_n_OFFS(n) (IPA_CFG_REG_BASE_OFFS +	\
-					0x00000840 + 0x70 * (n))
-#define HWIO_IPA_ENDP_STATUS_n_RMSK 0x23f
-#define HWIO_IPA_ENDP_STATUS_n_MAXn 30
-#define HWIO_IPA_ENDP_STATUS_n_ATTR 0x3
-#define HWIO_IPA_ENDP_STATUS_n_INI(n) in_dword_masked( \
-		HWIO_IPA_ENDP_STATUS_n_ADDR(n),	\
-		HWIO_IPA_ENDP_STATUS_n_RMSK)
-#define HWIO_IPA_ENDP_STATUS_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_ENDP_STATUS_n_ADDR(n),	\
-		mask)
-#define HWIO_IPA_ENDP_STATUS_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_ENDP_STATUS_n_ADDR(n),	\
-		val)
-#define HWIO_IPA_ENDP_STATUS_n_OUTMI(n, mask, val) out_dword_masked_ns(	\
-		HWIO_IPA_ENDP_STATUS_n_ADDR(n),	\
-		mask, \
-		val, \
-		HWIO_IPA_ENDP_STATUS_n_INI(n))
-#define HWIO_IPA_ENDP_STATUS_n_STATUS_ENDP_BMSK 0x3e
-#define HWIO_IPA_ENDP_STATUS_n_STATUS_ENDP_SHFT 0x1
-#define HWIO_IPA_ENDP_STATUS_n_STATUS_EN_BMSK 0x1
-#define HWIO_IPA_ENDP_STATUS_n_STATUS_EN_SHFT 0x0
-#define HWIO_IPA_ENDP_SRC_ID_WRITE_n_ADDR(n) (IPA_CFG_REG_BASE + \
-					      0x00000848 + 0x70 * (n))
-#define HWIO_IPA_ENDP_SRC_ID_WRITE_n_PHYS(n) (IPA_CFG_REG_BASE_PHYS + \
-					      0x00000848 + 0x70 * (n))
-#define HWIO_IPA_ENDP_SRC_ID_WRITE_n_OFFS(n) (IPA_CFG_REG_BASE_OFFS + \
-					      0x00000848 + 0x70 * (n))
-#define HWIO_IPA_ENDP_SRC_ID_READ_n_ADDR(n) (IPA_CFG_REG_BASE +	\
-					     0x0000084c + 0x70 * (n))
-#define HWIO_IPA_ENDP_SRC_ID_READ_n_PHYS(n) (IPA_CFG_REG_BASE_PHYS + \
-					     0x0000084c + 0x70 * (n))
-#define HWIO_IPA_ENDP_SRC_ID_READ_n_OFFS(n) (IPA_CFG_REG_BASE_OFFS + \
-					     0x0000084c + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_CONN_TRACK_n_ADDR(n) (IPA_CFG_REG_BASE + \
-						 0x00000850 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_CONN_TRACK_n_PHYS(n) (IPA_CFG_REG_BASE_PHYS + \
-						 0x00000850 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_CONN_TRACK_n_OFFS(n) (IPA_CFG_REG_BASE_OFFS + \
-						 0x00000850 + 0x70 * (n))
-#define HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_ADDR(n) (IPA_CFG_REG_BASE + \
-						       0x0000085c + 0x70 * \
-						       (n))
-#define HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_PHYS(n) (	\
-		IPA_CFG_REG_BASE_PHYS + 0x0000085c + 0x70 * (n))
-#define HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_OFFS(n) (	\
-		IPA_CFG_REG_BASE_OFFS + 0x0000085c + 0x70 * (n))
-#define HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_RMSK 0x7f007f
-#define HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_MAXn 31
-#define HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_ATTR 0x3
-#define HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_INI(n) in_dword_masked( \
-		HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_ADDR(n), \
-		HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_RMSK)
-#define HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_INMI(n, \
-						   mask) in_dword_masked( \
-		HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_ADDR( \
-			n), \
-		mask)
-#define HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_ADDR(n), \
-		val)
-#define HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_OUTMI(n, mask, \
-						    val) \
-	out_dword_masked_ns(HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_ADDR(	\
-				    n),	\
-			    mask, val, \
-			    HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_INI(n))
-#define	\
-	HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_ROUTER_HASH_MSK_METADATA_BMSK \
-	0x400000
-#define	\
-	HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_ROUTER_HASH_MSK_METADATA_SHFT \
-	0x16
-#define	\
-	HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_ROUTER_HASH_MSK_PROTOCOL_BMSK \
-	0x200000
-#define	\
-	HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_ROUTER_HASH_MSK_PROTOCOL_SHFT \
-	0x15
-#define	\
-	HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_ROUTER_HASH_MSK_DST_PORT_BMSK \
-	0x100000
-#define	\
-	HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_ROUTER_HASH_MSK_DST_PORT_SHFT \
-	0x14
-#define	\
-	HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_ROUTER_HASH_MSK_SRC_PORT_BMSK \
-	0x80000
-#define	\
-	HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_ROUTER_HASH_MSK_SRC_PORT_SHFT \
-	0x13
-#define	\
-	HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_ROUTER_HASH_MSK_DST_IP_ADD_BMSK \
-	0x40000
-#define	\
-	HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_ROUTER_HASH_MSK_DST_IP_ADD_SHFT \
-	0x12
-#define	\
-	HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_ROUTER_HASH_MSK_SRC_IP_ADD_BMSK \
-	0x20000
-#define	\
-	HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_ROUTER_HASH_MSK_SRC_IP_ADD_SHFT \
-	0x11
-#define HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_ROUTER_HASH_MSK_SRC_ID_BMSK \
-	0x10000
-#define HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_ROUTER_HASH_MSK_SRC_ID_SHFT \
-	0x10
-#define	\
-	HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_FILTER_HASH_MSK_METADATA_BMSK \
-	0x40
-#define	\
-	HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_FILTER_HASH_MSK_METADATA_SHFT \
-	0x6
-#define	\
-	HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_FILTER_HASH_MSK_PROTOCOL_BMSK \
-	0x20
-#define	\
-	HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_FILTER_HASH_MSK_PROTOCOL_SHFT \
-	0x5
-#define	\
-	HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_FILTER_HASH_MSK_DST_PORT_BMSK \
-	0x10
-#define	\
-	HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_FILTER_HASH_MSK_DST_PORT_SHFT \
-	0x4
-#define	\
-	HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_FILTER_HASH_MSK_SRC_PORT_BMSK \
-	0x8
-#define	\
-	HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_FILTER_HASH_MSK_SRC_PORT_SHFT \
-	0x3
-#define	\
-	HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_FILTER_HASH_MSK_DST_IP_ADD_BMSK \
-	0x4
-#define	\
-	HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_FILTER_HASH_MSK_DST_IP_ADD_SHFT \
-	0x2
-#define	\
-	HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_FILTER_HASH_MSK_SRC_IP_ADD_BMSK \
-	0x2
-#define	\
-	HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_FILTER_HASH_MSK_SRC_IP_ADD_SHFT \
-	0x1
-#define HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_FILTER_HASH_MSK_SRC_ID_BMSK \
-	0x1
-#define HWIO_IPA_ENDP_FILTER_ROUTER_HSH_CFG_n_FILTER_HASH_MSK_SRC_ID_SHFT \
-	0x0
-#define HWIO_IPA_ENDP_YELLOW_RED_MARKER_CFG_n_ADDR(n) (IPA_CFG_REG_BASE + \
-						       0x00000860 + 0x70 * \
-						       (n))
-#define HWIO_IPA_ENDP_YELLOW_RED_MARKER_CFG_n_PHYS(n) (	\
-		IPA_CFG_REG_BASE_PHYS + 0x00000860 + 0x70 * (n))
-#define HWIO_IPA_ENDP_YELLOW_RED_MARKER_CFG_n_OFFS(n) (	\
-		IPA_CFG_REG_BASE_OFFS + 0x00000860 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_CTRL_STATUS_n_ADDR(n) (IPA_CFG_REG_BASE + \
-						  0x00000864 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_CTRL_STATUS_n_PHYS(n) (IPA_CFG_REG_BASE_PHYS + \
-						  0x00000864 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_CTRL_STATUS_n_OFFS(n) (IPA_CFG_REG_BASE_OFFS + \
-						  0x00000864 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_PROD_CFG_n_ADDR(n) (IPA_CFG_REG_BASE + \
-					       0x00000868 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_PROD_CFG_n_PHYS(n) (IPA_CFG_REG_BASE_PHYS + \
-					       0x00000868 + 0x70 * (n))
-#define HWIO_IPA_ENDP_INIT_PROD_CFG_n_OFFS(n) (IPA_CFG_REG_BASE_OFFS + \
-					       0x00000868 + 0x70 * (n))
-#define HWIO_IPA_NLO_PP_CFG1_ADDR (IPA_CFG_REG_BASE + 0x00001680)
-#define HWIO_IPA_NLO_PP_CFG1_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00001680)
-#define HWIO_IPA_NLO_PP_CFG1_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00001680)
-#define HWIO_IPA_NLO_PP_CFG1_RMSK 0x3fffffff
-#define HWIO_IPA_NLO_PP_CFG1_ATTR 0x3
-#define HWIO_IPA_NLO_PP_CFG1_IN in_dword_masked(HWIO_IPA_NLO_PP_CFG1_ADDR, \
-						HWIO_IPA_NLO_PP_CFG1_RMSK)
-#define HWIO_IPA_NLO_PP_CFG1_INM(m) in_dword_masked( \
-		HWIO_IPA_NLO_PP_CFG1_ADDR, \
-		m)
-#define HWIO_IPA_NLO_PP_CFG1_OUT(v) out_dword(HWIO_IPA_NLO_PP_CFG1_ADDR, v)
-#define HWIO_IPA_NLO_PP_CFG1_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_NLO_PP_CFG1_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_NLO_PP_CFG1_IN)
-#define HWIO_IPA_NLO_PP_CFG1_NLO_ACK_MAX_VP_BMSK 0x3f000000
-#define HWIO_IPA_NLO_PP_CFG1_NLO_ACK_MAX_VP_SHFT 0x18
-#define HWIO_IPA_NLO_PP_CFG1_NLO_STATUS_PP_BMSK 0xff0000
-#define HWIO_IPA_NLO_PP_CFG1_NLO_STATUS_PP_SHFT 0x10
-#define HWIO_IPA_NLO_PP_CFG1_NLO_DATA_PP_BMSK 0xff00
-#define HWIO_IPA_NLO_PP_CFG1_NLO_DATA_PP_SHFT 0x8
-#define HWIO_IPA_NLO_PP_CFG1_NLO_ACK_PP_BMSK 0xff
-#define HWIO_IPA_NLO_PP_CFG1_NLO_ACK_PP_SHFT 0x0
-#define HWIO_IPA_NLO_PP_CFG2_ADDR (IPA_CFG_REG_BASE + 0x00001684)
-#define HWIO_IPA_NLO_PP_CFG2_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00001684)
-#define HWIO_IPA_NLO_PP_CFG2_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00001684)
-#define HWIO_IPA_NLO_PP_CFG2_RMSK 0x7ffff
-#define HWIO_IPA_NLO_PP_CFG2_ATTR 0x3
-#define HWIO_IPA_NLO_PP_CFG2_IN in_dword_masked(HWIO_IPA_NLO_PP_CFG2_ADDR, \
-						HWIO_IPA_NLO_PP_CFG2_RMSK)
-#define HWIO_IPA_NLO_PP_CFG2_INM(m) in_dword_masked( \
-		HWIO_IPA_NLO_PP_CFG2_ADDR, \
-		m)
-#define HWIO_IPA_NLO_PP_CFG2_OUT(v) out_dword(HWIO_IPA_NLO_PP_CFG2_ADDR, v)
-#define HWIO_IPA_NLO_PP_CFG2_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_NLO_PP_CFG2_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_NLO_PP_CFG2_IN)
-#define HWIO_IPA_NLO_PP_CFG2_NLO_STATUS_BUFFER_MODE_BMSK 0x40000
-#define HWIO_IPA_NLO_PP_CFG2_NLO_STATUS_BUFFER_MODE_SHFT 0x12
-#define HWIO_IPA_NLO_PP_CFG2_NLO_DATA_BUFFER_MODE_BMSK 0x20000
-#define HWIO_IPA_NLO_PP_CFG2_NLO_DATA_BUFFER_MODE_SHFT 0x11
-#define HWIO_IPA_NLO_PP_CFG2_NLO_ACK_BUFFER_MODE_BMSK 0x10000
-#define HWIO_IPA_NLO_PP_CFG2_NLO_ACK_BUFFER_MODE_SHFT 0x10
-#define HWIO_IPA_NLO_PP_CFG2_NLO_DATA_CLOSE_PADD_BMSK 0xff00
-#define HWIO_IPA_NLO_PP_CFG2_NLO_DATA_CLOSE_PADD_SHFT 0x8
-#define HWIO_IPA_NLO_PP_CFG2_NLO_ACK_CLOSE_PADD_BMSK 0xff
-#define HWIO_IPA_NLO_PP_CFG2_NLO_ACK_CLOSE_PADD_SHFT 0x0
-#define HWIO_IPA_NLO_PP_ACK_LIMIT_CFG_ADDR (IPA_CFG_REG_BASE + 0x00001688)
-#define HWIO_IPA_NLO_PP_ACK_LIMIT_CFG_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					    0x00001688)
-#define HWIO_IPA_NLO_PP_ACK_LIMIT_CFG_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					    0x00001688)
-#define HWIO_IPA_NLO_PP_ACK_LIMIT_CFG_RMSK 0xffffffff
-#define HWIO_IPA_NLO_PP_ACK_LIMIT_CFG_ATTR 0x3
-#define HWIO_IPA_NLO_PP_ACK_LIMIT_CFG_IN in_dword_masked( \
-		HWIO_IPA_NLO_PP_ACK_LIMIT_CFG_ADDR, \
-		HWIO_IPA_NLO_PP_ACK_LIMIT_CFG_RMSK)
-#define HWIO_IPA_NLO_PP_ACK_LIMIT_CFG_INM(m) in_dword_masked( \
-		HWIO_IPA_NLO_PP_ACK_LIMIT_CFG_ADDR, \
-		m)
-#define HWIO_IPA_NLO_PP_ACK_LIMIT_CFG_OUT(v) out_dword(	\
-		HWIO_IPA_NLO_PP_ACK_LIMIT_CFG_ADDR, \
-		v)
-#define HWIO_IPA_NLO_PP_ACK_LIMIT_CFG_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_NLO_PP_ACK_LIMIT_CFG_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_NLO_PP_ACK_LIMIT_CFG_IN)
-#define HWIO_IPA_NLO_PP_ACK_LIMIT_CFG_NLO_ACK_UPPER_SIZE_BMSK 0xffff0000
-#define HWIO_IPA_NLO_PP_ACK_LIMIT_CFG_NLO_ACK_UPPER_SIZE_SHFT 0x10
-#define HWIO_IPA_NLO_PP_ACK_LIMIT_CFG_NLO_ACK_LOWER_SIZE_BMSK 0xffff
-#define HWIO_IPA_NLO_PP_ACK_LIMIT_CFG_NLO_ACK_LOWER_SIZE_SHFT 0x0
-#define HWIO_IPA_NLO_PP_DATA_LIMIT_CFG_ADDR (IPA_CFG_REG_BASE + 0x0000168c)
-#define HWIO_IPA_NLO_PP_DATA_LIMIT_CFG_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					     0x0000168c)
-#define HWIO_IPA_NLO_PP_DATA_LIMIT_CFG_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					     0x0000168c)
-#define HWIO_IPA_NLO_PP_DATA_LIMIT_CFG_RMSK 0xffffffff
-#define HWIO_IPA_NLO_PP_DATA_LIMIT_CFG_ATTR 0x3
-#define HWIO_IPA_NLO_PP_DATA_LIMIT_CFG_IN in_dword_masked( \
-		HWIO_IPA_NLO_PP_DATA_LIMIT_CFG_ADDR, \
-		HWIO_IPA_NLO_PP_DATA_LIMIT_CFG_RMSK)
-#define HWIO_IPA_NLO_PP_DATA_LIMIT_CFG_INM(m) in_dword_masked( \
-		HWIO_IPA_NLO_PP_DATA_LIMIT_CFG_ADDR, \
-		m)
-#define HWIO_IPA_NLO_PP_DATA_LIMIT_CFG_OUT(v) out_dword( \
-		HWIO_IPA_NLO_PP_DATA_LIMIT_CFG_ADDR, \
-		v)
-#define HWIO_IPA_NLO_PP_DATA_LIMIT_CFG_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_NLO_PP_DATA_LIMIT_CFG_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_NLO_PP_DATA_LIMIT_CFG_IN)
-#define HWIO_IPA_NLO_PP_DATA_LIMIT_CFG_NLO_DATA_UPPER_SIZE_BMSK 0xffff0000
-#define HWIO_IPA_NLO_PP_DATA_LIMIT_CFG_NLO_DATA_UPPER_SIZE_SHFT 0x10
-#define HWIO_IPA_NLO_PP_DATA_LIMIT_CFG_NLO_DATA_LOWER_SIZE_BMSK 0xffff
-#define HWIO_IPA_NLO_PP_DATA_LIMIT_CFG_NLO_DATA_LOWER_SIZE_SHFT 0x0
-#define HWIO_IPA_NLO_MIN_DSM_CFG_ADDR (IPA_CFG_REG_BASE + 0x00001690)
-#define HWIO_IPA_NLO_MIN_DSM_CFG_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00001690)
-#define HWIO_IPA_NLO_MIN_DSM_CFG_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00001690)
-#define HWIO_IPA_NLO_MIN_DSM_CFG_RMSK 0xffffffff
-#define HWIO_IPA_NLO_MIN_DSM_CFG_ATTR 0x3
-#define HWIO_IPA_NLO_MIN_DSM_CFG_IN in_dword_masked( \
-		HWIO_IPA_NLO_MIN_DSM_CFG_ADDR, \
-		HWIO_IPA_NLO_MIN_DSM_CFG_RMSK)
-#define HWIO_IPA_NLO_MIN_DSM_CFG_INM(m) in_dword_masked( \
-		HWIO_IPA_NLO_MIN_DSM_CFG_ADDR, \
-		m)
-#define HWIO_IPA_NLO_MIN_DSM_CFG_OUT(v) out_dword( \
-		HWIO_IPA_NLO_MIN_DSM_CFG_ADDR, \
-		v)
-#define HWIO_IPA_NLO_MIN_DSM_CFG_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_NLO_MIN_DSM_CFG_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_NLO_MIN_DSM_CFG_IN)
-#define HWIO_IPA_NLO_MIN_DSM_CFG_NLO_DATA_MIN_DSM_LEN_BMSK 0xffff0000
-#define HWIO_IPA_NLO_MIN_DSM_CFG_NLO_DATA_MIN_DSM_LEN_SHFT 0x10
-#define HWIO_IPA_NLO_MIN_DSM_CFG_NLO_ACK_MIN_DSM_LEN_BMSK 0xffff
-#define HWIO_IPA_NLO_MIN_DSM_CFG_NLO_ACK_MIN_DSM_LEN_SHFT 0x0
-#define HWIO_IPA_NLO_VP_AGGR_CFG_LSB_n_ADDR(n) (IPA_CFG_REG_BASE + \
-						0x00001700 + 0x8 * (n))
-#define HWIO_IPA_NLO_VP_AGGR_CFG_LSB_n_PHYS(n) (IPA_CFG_REG_BASE_PHYS +	\
-						0x00001700 + 0x8 * (n))
-#define HWIO_IPA_NLO_VP_AGGR_CFG_LSB_n_OFFS(n) (IPA_CFG_REG_BASE_OFFS +	\
-						0x00001700 + 0x8 * (n))
-#define HWIO_IPA_NLO_VP_AGGR_CFG_MSB_n_ADDR(n) (IPA_CFG_REG_BASE + \
-						0x00001704 + 0x8 * (n))
-#define HWIO_IPA_NLO_VP_AGGR_CFG_MSB_n_PHYS(n) (IPA_CFG_REG_BASE_PHYS +	\
-						0x00001704 + 0x8 * (n))
-#define HWIO_IPA_NLO_VP_AGGR_CFG_MSB_n_OFFS(n) (IPA_CFG_REG_BASE_OFFS +	\
-						0x00001704 + 0x8 * (n))
-#define HWIO_IPA_SNIFFER_QMB_SEL_ADDR (IPA_CFG_REG_BASE + 0x00001800)
-#define HWIO_IPA_SNIFFER_QMB_SEL_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00001800)
-#define HWIO_IPA_SNIFFER_QMB_SEL_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00001800)
-#define HWIO_IPA_COAL_EVICT_LRU_ADDR (IPA_CFG_REG_BASE + 0x0000180c)
-#define HWIO_IPA_COAL_EVICT_LRU_PHYS (IPA_CFG_REG_BASE_PHYS + 0x0000180c)
-#define HWIO_IPA_COAL_EVICT_LRU_OFFS (IPA_CFG_REG_BASE_OFFS + 0x0000180c)
-#define HWIO_IPA_COAL_QMAP_CFG_ADDR (IPA_CFG_REG_BASE + 0x00001810)
-#define HWIO_IPA_COAL_QMAP_CFG_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00001810)
-#define HWIO_IPA_COAL_QMAP_CFG_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00001810)
-#define HWIO_IPA_NLO_VP_FLUSH_REQ_ADDR (IPA_CFG_REG_BASE + 0x00001814)
-#define HWIO_IPA_NLO_VP_FLUSH_REQ_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00001814)
-#define HWIO_IPA_NLO_VP_FLUSH_REQ_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00001814)
-#define HWIO_IPA_NLO_VP_FLUSH_REQ_RMSK 0x80ff00ff
-#define HWIO_IPA_NLO_VP_FLUSH_REQ_ATTR 0x3
-#define HWIO_IPA_NLO_VP_FLUSH_REQ_IN in_dword_masked( \
-		HWIO_IPA_NLO_VP_FLUSH_REQ_ADDR,	\
-		HWIO_IPA_NLO_VP_FLUSH_REQ_RMSK)
-#define HWIO_IPA_NLO_VP_FLUSH_REQ_INM(m) in_dword_masked( \
-		HWIO_IPA_NLO_VP_FLUSH_REQ_ADDR,	\
-		m)
-#define HWIO_IPA_NLO_VP_FLUSH_REQ_OUT(v) out_dword( \
-		HWIO_IPA_NLO_VP_FLUSH_REQ_ADDR,	\
-		v)
-#define HWIO_IPA_NLO_VP_FLUSH_REQ_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_NLO_VP_FLUSH_REQ_ADDR,	\
-		m, \
-		v, \
-		HWIO_IPA_NLO_VP_FLUSH_REQ_IN)
-#define HWIO_IPA_NLO_VP_FLUSH_REQ_VP_FLUSH_REQ_BMSK 0x80000000
-#define HWIO_IPA_NLO_VP_FLUSH_REQ_VP_FLUSH_REQ_SHFT 0x1f
-#define HWIO_IPA_NLO_VP_FLUSH_REQ_VP_FLUSH_VP_INDX_BMSK 0xff0000
-#define HWIO_IPA_NLO_VP_FLUSH_REQ_VP_FLUSH_VP_INDX_SHFT 0x10
-#define HWIO_IPA_NLO_VP_FLUSH_REQ_VP_FLUSH_PP_INDX_BMSK 0xff
-#define HWIO_IPA_NLO_VP_FLUSH_REQ_VP_FLUSH_PP_INDX_SHFT 0x0
-#define HWIO_IPA_NLO_VP_FLUSH_COOKIE_ADDR (IPA_CFG_REG_BASE + 0x00001818)
-#define HWIO_IPA_NLO_VP_FLUSH_COOKIE_PHYS (IPA_CFG_REG_BASE_PHYS + \
-					   0x00001818)
-#define HWIO_IPA_NLO_VP_FLUSH_COOKIE_OFFS (IPA_CFG_REG_BASE_OFFS + \
-					   0x00001818)
-#define HWIO_IPA_NLO_VP_FLUSH_COOKIE_RMSK 0xffffffff
-#define HWIO_IPA_NLO_VP_FLUSH_COOKIE_ATTR 0x1
-#define HWIO_IPA_NLO_VP_FLUSH_COOKIE_IN in_dword_masked( \
-		HWIO_IPA_NLO_VP_FLUSH_COOKIE_ADDR, \
-		HWIO_IPA_NLO_VP_FLUSH_COOKIE_RMSK)
-#define HWIO_IPA_NLO_VP_FLUSH_COOKIE_INM(m) in_dword_masked( \
-		HWIO_IPA_NLO_VP_FLUSH_COOKIE_ADDR, \
-		m)
-#define HWIO_IPA_NLO_VP_FLUSH_COOKIE_VP_FLUSH_COOKIE_BMSK 0xffffffff
-#define HWIO_IPA_NLO_VP_FLUSH_COOKIE_VP_FLUSH_COOKIE_SHFT 0x0
-#define HWIO_IPA_NLO_VP_FLUSH_ACK_ADDR (IPA_CFG_REG_BASE + 0x0000181c)
-#define HWIO_IPA_NLO_VP_FLUSH_ACK_PHYS (IPA_CFG_REG_BASE_PHYS + 0x0000181c)
-#define HWIO_IPA_NLO_VP_FLUSH_ACK_OFFS (IPA_CFG_REG_BASE_OFFS + 0x0000181c)
-#define HWIO_IPA_NLO_VP_FLUSH_ACK_RMSK 0x1
-#define HWIO_IPA_NLO_VP_FLUSH_ACK_ATTR 0x1
-#define HWIO_IPA_NLO_VP_FLUSH_ACK_IN in_dword_masked( \
-		HWIO_IPA_NLO_VP_FLUSH_ACK_ADDR,	\
-		HWIO_IPA_NLO_VP_FLUSH_ACK_RMSK)
-#define HWIO_IPA_NLO_VP_FLUSH_ACK_INM(m) in_dword_masked( \
-		HWIO_IPA_NLO_VP_FLUSH_ACK_ADDR,	\
-		m)
-#define HWIO_IPA_NLO_VP_FLUSH_ACK_VP_FLUSH_ACK_BMSK 0x1
-#define HWIO_IPA_NLO_VP_FLUSH_ACK_VP_FLUSH_ACK_SHFT 0x0
-#define HWIO_IPA_NLO_VP_DSM_OPEN_ADDR (IPA_CFG_REG_BASE + 0x00001820)
-#define HWIO_IPA_NLO_VP_DSM_OPEN_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00001820)
-#define HWIO_IPA_NLO_VP_DSM_OPEN_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00001820)
-#define HWIO_IPA_NLO_VP_DSM_OPEN_RMSK 0xffffffff
-#define HWIO_IPA_NLO_VP_DSM_OPEN_ATTR 0x1
-#define HWIO_IPA_NLO_VP_DSM_OPEN_IN in_dword_masked( \
-		HWIO_IPA_NLO_VP_DSM_OPEN_ADDR, \
-		HWIO_IPA_NLO_VP_DSM_OPEN_RMSK)
-#define HWIO_IPA_NLO_VP_DSM_OPEN_INM(m) in_dword_masked( \
-		HWIO_IPA_NLO_VP_DSM_OPEN_ADDR, \
-		m)
-#define HWIO_IPA_NLO_VP_DSM_OPEN_VP_DSM_OPEN_BMSK 0xffffffff
-#define HWIO_IPA_NLO_VP_DSM_OPEN_VP_DSM_OPEN_SHFT 0x0
-#define HWIO_IPA_NLO_VP_QBAP_OPEN_ADDR (IPA_CFG_REG_BASE + 0x00001824)
-#define HWIO_IPA_NLO_VP_QBAP_OPEN_PHYS (IPA_CFG_REG_BASE_PHYS + 0x00001824)
-#define HWIO_IPA_NLO_VP_QBAP_OPEN_OFFS (IPA_CFG_REG_BASE_OFFS + 0x00001824)
-#define HWIO_IPA_NLO_VP_QBAP_OPEN_RMSK 0xffffffff
-#define HWIO_IPA_NLO_VP_QBAP_OPEN_ATTR 0x1
-#define HWIO_IPA_NLO_VP_QBAP_OPEN_IN in_dword_masked( \
-		HWIO_IPA_NLO_VP_QBAP_OPEN_ADDR,	\
-		HWIO_IPA_NLO_VP_QBAP_OPEN_RMSK)
-#define HWIO_IPA_NLO_VP_QBAP_OPEN_INM(m) in_dword_masked( \
-		HWIO_IPA_NLO_VP_QBAP_OPEN_ADDR,	\
-		m)
-#define HWIO_IPA_NLO_VP_QBAP_OPEN_VP_QBAP_OPEN_BMSK 0xffffffff
-#define HWIO_IPA_NLO_VP_QBAP_OPEN_VP_QBAP_OPEN_SHFT 0x0
-#define IPA_DEBUG_REG_BASE (IPA_0_IPA_WRAPPER_BASE + 0x00042000)
-#define IPA_DEBUG_REG_BASE_PHYS (IPA_0_IPA_WRAPPER_BASE_PHYS + 0x00042000)
-#define IPA_DEBUG_REG_BASE_OFFS 0x00042000
-#define HWIO_IPA_HPS_FTCH_ARB_DEBUG_STATUS_ADDR (IPA_DEBUG_REG_BASE + \
-						 0x00000000)
-#define HWIO_IPA_HPS_FTCH_ARB_DEBUG_STATUS_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						 0x00000000)
-#define HWIO_IPA_HPS_FTCH_ARB_DEBUG_STATUS_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						 0x00000000)
-#define HWIO_IPA_HPS_FTCH_ARB_DEBUG_BLOCK_STATUS_ADDR (IPA_DEBUG_REG_BASE \
-						       + 0x00000004)
-#define HWIO_IPA_HPS_FTCH_ARB_DEBUG_BLOCK_STATUS_PHYS (	\
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000004)
-#define HWIO_IPA_HPS_FTCH_ARB_DEBUG_BLOCK_STATUS_OFFS (	\
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000004)
-#define HWIO_IPA_HPS_FTCH_ARB_DEBUG_CFG_MASK_ADDR (IPA_DEBUG_REG_BASE +	\
-						   0x00000008)
-#define HWIO_IPA_HPS_FTCH_ARB_DEBUG_CFG_MASK_PHYS (IPA_DEBUG_REG_BASE_PHYS \
-						   + 0x00000008)
-#define HWIO_IPA_HPS_FTCH_ARB_DEBUG_CFG_MASK_OFFS (IPA_DEBUG_REG_BASE_OFFS \
-						   + 0x00000008)
-#define HWIO_IPA_HPS_FTCH_ARB_DEBUG_CFG_BLOCK_ADDR (IPA_DEBUG_REG_BASE + \
-						    0x0000000c)
-#define HWIO_IPA_HPS_FTCH_ARB_DEBUG_CFG_BLOCK_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x0000000c)
-#define HWIO_IPA_HPS_FTCH_ARB_DEBUG_CFG_BLOCK_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x0000000c)
-#define HWIO_IPA_HPS_FTCH_ARB_DEBUG_CMD_ADDR (IPA_DEBUG_REG_BASE + \
-					      0x00000010)
-#define HWIO_IPA_HPS_FTCH_ARB_DEBUG_CMD_PHYS (IPA_DEBUG_REG_BASE_PHYS +	\
-					      0x00000010)
-#define HWIO_IPA_HPS_FTCH_ARB_DEBUG_CMD_OFFS (IPA_DEBUG_REG_BASE_OFFS +	\
-					      0x00000010)
-#define HWIO_IPA_DPS_FTCH_ARB_DEBUG_STATUS_ADDR (IPA_DEBUG_REG_BASE + \
-						 0x00000014)
-#define HWIO_IPA_DPS_FTCH_ARB_DEBUG_STATUS_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						 0x00000014)
-#define HWIO_IPA_DPS_FTCH_ARB_DEBUG_STATUS_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						 0x00000014)
-#define HWIO_IPA_DPS_FTCH_ARB_DEBUG_BLOCK_STATUS_ADDR (IPA_DEBUG_REG_BASE \
-						       + 0x00000018)
-#define HWIO_IPA_DPS_FTCH_ARB_DEBUG_BLOCK_STATUS_PHYS (	\
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000018)
-#define HWIO_IPA_DPS_FTCH_ARB_DEBUG_BLOCK_STATUS_OFFS (	\
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000018)
-#define HWIO_IPA_DPS_FTCH_ARB_DEBUG_CFG_MASK_ADDR (IPA_DEBUG_REG_BASE +	\
-						   0x0000001c)
-#define HWIO_IPA_DPS_FTCH_ARB_DEBUG_CFG_MASK_PHYS (IPA_DEBUG_REG_BASE_PHYS \
-						   + 0x0000001c)
-#define HWIO_IPA_DPS_FTCH_ARB_DEBUG_CFG_MASK_OFFS (IPA_DEBUG_REG_BASE_OFFS \
-						   + 0x0000001c)
-#define HWIO_IPA_DPS_FTCH_ARB_DEBUG_CFG_BLOCK_ADDR (IPA_DEBUG_REG_BASE + \
-						    0x00000020)
-#define HWIO_IPA_DPS_FTCH_ARB_DEBUG_CFG_BLOCK_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000020)
-#define HWIO_IPA_DPS_FTCH_ARB_DEBUG_CFG_BLOCK_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000020)
-#define HWIO_IPA_DPS_FTCH_ARB_DEBUG_CMD_ADDR (IPA_DEBUG_REG_BASE + \
-					      0x00000024)
-#define HWIO_IPA_DPS_FTCH_ARB_DEBUG_CMD_PHYS (IPA_DEBUG_REG_BASE_PHYS +	\
-					      0x00000024)
-#define HWIO_IPA_DPS_FTCH_ARB_DEBUG_CMD_OFFS (IPA_DEBUG_REG_BASE_OFFS +	\
-					      0x00000024)
-#define HWIO_IPA_RSRC_MNGR_FUNC_ARB_DEBUG_STATUS_ADDR (IPA_DEBUG_REG_BASE \
-						       + 0x00000028)
-#define HWIO_IPA_RSRC_MNGR_FUNC_ARB_DEBUG_STATUS_PHYS (	\
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000028)
-#define HWIO_IPA_RSRC_MNGR_FUNC_ARB_DEBUG_STATUS_OFFS (	\
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000028)
-#define HWIO_IPA_RSRC_MNGR_FUNC_ARB_DEBUG_BLOCK_STATUS_ADDR ( \
-		IPA_DEBUG_REG_BASE + 0x0000002c)
-#define HWIO_IPA_RSRC_MNGR_FUNC_ARB_DEBUG_BLOCK_STATUS_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x0000002c)
-#define HWIO_IPA_RSRC_MNGR_FUNC_ARB_DEBUG_BLOCK_STATUS_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x0000002c)
-#define HWIO_IPA_RSRC_MNGR_FUNC_ARB_DEBUG_CFG_MASK_ADDR ( \
-		IPA_DEBUG_REG_BASE + 0x00000030)
-#define HWIO_IPA_RSRC_MNGR_FUNC_ARB_DEBUG_CFG_MASK_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000030)
-#define HWIO_IPA_RSRC_MNGR_FUNC_ARB_DEBUG_CFG_MASK_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000030)
-#define HWIO_IPA_RSRC_MNGR_FUNC_ARB_DEBUG_CFG_BLOCK_ADDR ( \
-		IPA_DEBUG_REG_BASE + 0x00000034)
-#define HWIO_IPA_RSRC_MNGR_FUNC_ARB_DEBUG_CFG_BLOCK_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000034)
-#define HWIO_IPA_RSRC_MNGR_FUNC_ARB_DEBUG_CFG_BLOCK_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000034)
-#define HWIO_IPA_RSRC_MNGR_FUNC_ARB_DEBUG_CMD_ADDR (IPA_DEBUG_REG_BASE + \
-						    0x00000038)
-#define HWIO_IPA_RSRC_MNGR_FUNC_ARB_DEBUG_CMD_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000038)
-#define HWIO_IPA_RSRC_MNGR_FUNC_ARB_DEBUG_CMD_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000038)
-#define HWIO_IPA_RSRC_MNGR_ALLOC_ARB_DEBUG_STATUS_ADDR (IPA_DEBUG_REG_BASE \
-							+ 0x0000003c)
-#define HWIO_IPA_RSRC_MNGR_ALLOC_ARB_DEBUG_STATUS_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x0000003c)
-#define HWIO_IPA_RSRC_MNGR_ALLOC_ARB_DEBUG_STATUS_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x0000003c)
-#define HWIO_IPA_RSRC_MNGR_ALLOC_ARB_DEBUG_BLOCK_STATUS_ADDR ( \
-		IPA_DEBUG_REG_BASE + 0x00000040)
-#define HWIO_IPA_RSRC_MNGR_ALLOC_ARB_DEBUG_BLOCK_STATUS_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000040)
-#define HWIO_IPA_RSRC_MNGR_ALLOC_ARB_DEBUG_BLOCK_STATUS_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000040)
-#define HWIO_IPA_RSRC_MNGR_ALLOC_ARB_DEBUG_CFG_MASK_ADDR ( \
-		IPA_DEBUG_REG_BASE + 0x00000044)
-#define HWIO_IPA_RSRC_MNGR_ALLOC_ARB_DEBUG_CFG_MASK_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000044)
-#define HWIO_IPA_RSRC_MNGR_ALLOC_ARB_DEBUG_CFG_MASK_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000044)
-#define HWIO_IPA_RSRC_MNGR_ALLOC_ARB_DEBUG_CFG_BLOCK_ADDR ( \
-		IPA_DEBUG_REG_BASE + 0x00000048)
-#define HWIO_IPA_RSRC_MNGR_ALLOC_ARB_DEBUG_CFG_BLOCK_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000048)
-#define HWIO_IPA_RSRC_MNGR_ALLOC_ARB_DEBUG_CFG_BLOCK_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000048)
-#define HWIO_IPA_RSRC_MNGR_ALLOC_ARB_DEBUG_CMD_ADDR (IPA_DEBUG_REG_BASE + \
-						     0x0000004c)
-#define HWIO_IPA_RSRC_MNGR_ALLOC_ARB_DEBUG_CMD_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x0000004c)
-#define HWIO_IPA_RSRC_MNGR_ALLOC_ARB_DEBUG_CMD_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x0000004c)
-#define HWIO_IPA_RSRC_MNGR_SRCH_ARB_DEBUG_STATUS_ADDR (IPA_DEBUG_REG_BASE \
-						       + 0x00000050)
-#define HWIO_IPA_RSRC_MNGR_SRCH_ARB_DEBUG_STATUS_PHYS (	\
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000050)
-#define HWIO_IPA_RSRC_MNGR_SRCH_ARB_DEBUG_STATUS_OFFS (	\
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000050)
-#define HWIO_IPA_RSRC_MNGR_SRCH_ARB_DEBUG_BLOCK_STATUS_ADDR ( \
-		IPA_DEBUG_REG_BASE + 0x00000054)
-#define HWIO_IPA_RSRC_MNGR_SRCH_ARB_DEBUG_BLOCK_STATUS_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000054)
-#define HWIO_IPA_RSRC_MNGR_SRCH_ARB_DEBUG_BLOCK_STATUS_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000054)
-#define HWIO_IPA_RSRC_MNGR_SRCH_ARB_DEBUG_CFG_MASK_ADDR ( \
-		IPA_DEBUG_REG_BASE + 0x00000058)
-#define HWIO_IPA_RSRC_MNGR_SRCH_ARB_DEBUG_CFG_MASK_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000058)
-#define HWIO_IPA_RSRC_MNGR_SRCH_ARB_DEBUG_CFG_MASK_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000058)
-#define HWIO_IPA_RSRC_MNGR_SRCH_ARB_DEBUG_CFG_BLOCK_ADDR ( \
-		IPA_DEBUG_REG_BASE + 0x0000005c)
-#define HWIO_IPA_RSRC_MNGR_SRCH_ARB_DEBUG_CFG_BLOCK_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x0000005c)
-#define HWIO_IPA_RSRC_MNGR_SRCH_ARB_DEBUG_CFG_BLOCK_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x0000005c)
-#define HWIO_IPA_RSRC_MNGR_SRCH_ARB_DEBUG_CMD_ADDR (IPA_DEBUG_REG_BASE + \
-						    0x00000060)
-#define HWIO_IPA_RSRC_MNGR_SRCH_ARB_DEBUG_CMD_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000060)
-#define HWIO_IPA_RSRC_MNGR_SRCH_ARB_DEBUG_CMD_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000060)
-#define HWIO_IPA_RSRC_MNGR_REL_ARB_DEBUG_STATUS_ADDR (IPA_DEBUG_REG_BASE + \
-						      0x00000064)
-#define HWIO_IPA_RSRC_MNGR_REL_ARB_DEBUG_STATUS_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000064)
-#define HWIO_IPA_RSRC_MNGR_REL_ARB_DEBUG_STATUS_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000064)
-#define HWIO_IPA_RSRC_MNGR_REL_ARB_DEBUG_BLOCK_STATUS_ADDR ( \
-		IPA_DEBUG_REG_BASE + 0x00000068)
-#define HWIO_IPA_RSRC_MNGR_REL_ARB_DEBUG_BLOCK_STATUS_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000068)
-#define HWIO_IPA_RSRC_MNGR_REL_ARB_DEBUG_BLOCK_STATUS_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000068)
-#define HWIO_IPA_RSRC_MNGR_REL_ARB_DEBUG_CFG_MASK_ADDR (IPA_DEBUG_REG_BASE \
-							+ 0x0000006c)
-#define HWIO_IPA_RSRC_MNGR_REL_ARB_DEBUG_CFG_MASK_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x0000006c)
-#define HWIO_IPA_RSRC_MNGR_REL_ARB_DEBUG_CFG_MASK_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x0000006c)
-#define HWIO_IPA_RSRC_MNGR_REL_ARB_DEBUG_CFG_BLOCK_ADDR ( \
-		IPA_DEBUG_REG_BASE + 0x00000070)
-#define HWIO_IPA_RSRC_MNGR_REL_ARB_DEBUG_CFG_BLOCK_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000070)
-#define HWIO_IPA_RSRC_MNGR_REL_ARB_DEBUG_CFG_BLOCK_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000070)
-#define HWIO_IPA_RSRC_MNGR_REL_ARB_DEBUG_CMD_ADDR (IPA_DEBUG_REG_BASE +	\
-						   0x00000074)
-#define HWIO_IPA_RSRC_MNGR_REL_ARB_DEBUG_CMD_PHYS (IPA_DEBUG_REG_BASE_PHYS \
-						   + 0x00000074)
-#define HWIO_IPA_RSRC_MNGR_REL_ARB_DEBUG_CMD_OFFS (IPA_DEBUG_REG_BASE_OFFS \
-						   + 0x00000074)
-#define HWIO_IPA_TX_ARB_DEBUG_STATUS_ADDR (IPA_DEBUG_REG_BASE + 0x00000078)
-#define HWIO_IPA_TX_ARB_DEBUG_STATUS_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					   0x00000078)
-#define HWIO_IPA_TX_ARB_DEBUG_STATUS_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					   0x00000078)
-#define HWIO_IPA_TX_ARB_DEBUG_BLOCK_STATUS_ADDR (IPA_DEBUG_REG_BASE + \
-						 0x0000007c)
-#define HWIO_IPA_TX_ARB_DEBUG_BLOCK_STATUS_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						 0x0000007c)
-#define HWIO_IPA_TX_ARB_DEBUG_BLOCK_STATUS_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						 0x0000007c)
-#define HWIO_IPA_TX_ARB_DEBUG_CFG_MASK_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x00000080)
-#define HWIO_IPA_TX_ARB_DEBUG_CFG_MASK_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x00000080)
-#define HWIO_IPA_TX_ARB_DEBUG_CFG_MASK_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x00000080)
-#define HWIO_IPA_TX_ARB_DEBUG_CFG_BLOCK_ADDR (IPA_DEBUG_REG_BASE + \
-					      0x00000084)
-#define HWIO_IPA_TX_ARB_DEBUG_CFG_BLOCK_PHYS (IPA_DEBUG_REG_BASE_PHYS +	\
-					      0x00000084)
-#define HWIO_IPA_TX_ARB_DEBUG_CFG_BLOCK_OFFS (IPA_DEBUG_REG_BASE_OFFS +	\
-					      0x00000084)
-#define HWIO_IPA_TX_ARB_DEBUG_CMD_ADDR (IPA_DEBUG_REG_BASE + 0x00000088)
-#define HWIO_IPA_TX_ARB_DEBUG_CMD_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					0x00000088)
-#define HWIO_IPA_TX_ARB_DEBUG_CMD_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					0x00000088)
-#define HWIO_IPA_HPS_SEQ_ARB_DEBUG_STATUS_ADDR (IPA_DEBUG_REG_BASE + \
-						0x0000008c)
-#define HWIO_IPA_HPS_SEQ_ARB_DEBUG_STATUS_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						0x0000008c)
-#define HWIO_IPA_HPS_SEQ_ARB_DEBUG_STATUS_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						0x0000008c)
-#define HWIO_IPA_HPS_SEQ_ARB_DEBUG_BLOCK_STATUS_ADDR (IPA_DEBUG_REG_BASE + \
-						      0x00000090)
-#define HWIO_IPA_HPS_SEQ_ARB_DEBUG_BLOCK_STATUS_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000090)
-#define HWIO_IPA_HPS_SEQ_ARB_DEBUG_BLOCK_STATUS_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000090)
-#define HWIO_IPA_HPS_SEQ_ARB_DEBUG_CFG_MASK_ADDR (IPA_DEBUG_REG_BASE + \
-						  0x00000094)
-#define HWIO_IPA_HPS_SEQ_ARB_DEBUG_CFG_MASK_PHYS (IPA_DEBUG_REG_BASE_PHYS \
-						  + 0x00000094)
-#define HWIO_IPA_HPS_SEQ_ARB_DEBUG_CFG_MASK_OFFS (IPA_DEBUG_REG_BASE_OFFS \
-						  + 0x00000094)
-#define HWIO_IPA_HPS_SEQ_ARB_DEBUG_CFG_BLOCK_ADDR (IPA_DEBUG_REG_BASE +	\
-						   0x00000098)
-#define HWIO_IPA_HPS_SEQ_ARB_DEBUG_CFG_BLOCK_PHYS (IPA_DEBUG_REG_BASE_PHYS \
-						   + 0x00000098)
-#define HWIO_IPA_HPS_SEQ_ARB_DEBUG_CFG_BLOCK_OFFS (IPA_DEBUG_REG_BASE_OFFS \
-						   + 0x00000098)
-#define HWIO_IPA_HPS_SEQ_ARB_DEBUG_CMD_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x0000009c)
-#define HWIO_IPA_HPS_SEQ_ARB_DEBUG_CMD_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x0000009c)
-#define HWIO_IPA_HPS_SEQ_ARB_DEBUG_CMD_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x0000009c)
-#define HWIO_IPA_DPS_SEQ_ARB_DEBUG_STATUS_ADDR (IPA_DEBUG_REG_BASE + \
-						0x00000100)
-#define HWIO_IPA_DPS_SEQ_ARB_DEBUG_STATUS_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						0x00000100)
-#define HWIO_IPA_DPS_SEQ_ARB_DEBUG_STATUS_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						0x00000100)
-#define HWIO_IPA_DPS_SEQ_ARB_DEBUG_BLOCK_STATUS_ADDR (IPA_DEBUG_REG_BASE + \
-						      0x00000104)
-#define HWIO_IPA_DPS_SEQ_ARB_DEBUG_BLOCK_STATUS_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000104)
-#define HWIO_IPA_DPS_SEQ_ARB_DEBUG_BLOCK_STATUS_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000104)
-#define HWIO_IPA_DPS_SEQ_ARB_DEBUG_CFG_MASK_ADDR (IPA_DEBUG_REG_BASE + \
-						  0x00000108)
-#define HWIO_IPA_DPS_SEQ_ARB_DEBUG_CFG_MASK_PHYS (IPA_DEBUG_REG_BASE_PHYS \
-						  + 0x00000108)
-#define HWIO_IPA_DPS_SEQ_ARB_DEBUG_CFG_MASK_OFFS (IPA_DEBUG_REG_BASE_OFFS \
-						  + 0x00000108)
-#define HWIO_IPA_DPS_SEQ_ARB_DEBUG_CFG_BLOCK_ADDR (IPA_DEBUG_REG_BASE +	\
-						   0x0000010c)
-#define HWIO_IPA_DPS_SEQ_ARB_DEBUG_CFG_BLOCK_PHYS (IPA_DEBUG_REG_BASE_PHYS \
-						   + 0x0000010c)
-#define HWIO_IPA_DPS_SEQ_ARB_DEBUG_CFG_BLOCK_OFFS (IPA_DEBUG_REG_BASE_OFFS \
-						   + 0x0000010c)
-#define HWIO_IPA_DPS_SEQ_ARB_DEBUG_CMD_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x00000110)
-#define HWIO_IPA_DPS_SEQ_ARB_DEBUG_CMD_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x00000110)
-#define HWIO_IPA_DPS_SEQ_ARB_DEBUG_CMD_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x00000110)
-#define HWIO_IPA_RSRC_MNGR_SW_ACCESS_ALLOC_CFG_ADDR (IPA_DEBUG_REG_BASE + \
-						     0x00000114)
-#define HWIO_IPA_RSRC_MNGR_SW_ACCESS_ALLOC_CFG_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000114)
-#define HWIO_IPA_RSRC_MNGR_SW_ACCESS_ALLOC_CFG_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000114)
-#define HWIO_IPA_RSRC_MNGR_SW_ACCESS_SRCH_CFG_ADDR (IPA_DEBUG_REG_BASE + \
-						    0x00000118)
-#define HWIO_IPA_RSRC_MNGR_SW_ACCESS_SRCH_CFG_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000118)
-#define HWIO_IPA_RSRC_MNGR_SW_ACCESS_SRCH_CFG_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000118)
-#define HWIO_IPA_RSRC_MNGR_SW_ACCESS_REL_CFG_ADDR (IPA_DEBUG_REG_BASE +	\
-						   0x0000011c)
-#define HWIO_IPA_RSRC_MNGR_SW_ACCESS_REL_CFG_PHYS (IPA_DEBUG_REG_BASE_PHYS \
-						   + 0x0000011c)
-#define HWIO_IPA_RSRC_MNGR_SW_ACCESS_REL_CFG_OFFS (IPA_DEBUG_REG_BASE_OFFS \
-						   + 0x0000011c)
-#define HWIO_IPA_RSRC_MNGR_SW_ACCESS_CMD_ADDR (IPA_DEBUG_REG_BASE + \
-					       0x00000120)
-#define HWIO_IPA_RSRC_MNGR_SW_ACCESS_CMD_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					       0x00000120)
-#define HWIO_IPA_RSRC_MNGR_SW_ACCESS_CMD_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					       0x00000120)
-#define HWIO_IPA_RSRC_MNGR_SW_ACCESS_STATUS_ADDR (IPA_DEBUG_REG_BASE + \
-						  0x00000124)
-#define HWIO_IPA_RSRC_MNGR_SW_ACCESS_STATUS_PHYS (IPA_DEBUG_REG_BASE_PHYS \
-						  + 0x00000124)
-#define HWIO_IPA_RSRC_MNGR_SW_ACCESS_STATUS_OFFS (IPA_DEBUG_REG_BASE_OFFS \
-						  + 0x00000124)
-#define HWIO_IPA_RSRC_MNGR_DB_CFG_ADDR (IPA_DEBUG_REG_BASE + 0x00000128)
-#define HWIO_IPA_RSRC_MNGR_DB_CFG_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					0x00000128)
-#define HWIO_IPA_RSRC_MNGR_DB_CFG_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					0x00000128)
-#define HWIO_IPA_RSRC_MNGR_DB_CFG_RMSK 0x3f77
-#define HWIO_IPA_RSRC_MNGR_DB_CFG_ATTR 0x3
-#define HWIO_IPA_RSRC_MNGR_DB_CFG_IN in_dword_masked( \
-		HWIO_IPA_RSRC_MNGR_DB_CFG_ADDR,	\
-		HWIO_IPA_RSRC_MNGR_DB_CFG_RMSK)
-#define HWIO_IPA_RSRC_MNGR_DB_CFG_INM(m) in_dword_masked( \
-		HWIO_IPA_RSRC_MNGR_DB_CFG_ADDR,	\
-		m)
-#define HWIO_IPA_RSRC_MNGR_DB_CFG_OUT(v) out_dword( \
-		HWIO_IPA_RSRC_MNGR_DB_CFG_ADDR,	\
-		v)
-#define HWIO_IPA_RSRC_MNGR_DB_CFG_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_RSRC_MNGR_DB_CFG_ADDR,	\
-		m, \
-		v, \
-		HWIO_IPA_RSRC_MNGR_DB_CFG_IN)
-#define HWIO_IPA_RSRC_MNGR_DB_CFG_RSRC_ID_SEL_BMSK 0x3f00
-#define HWIO_IPA_RSRC_MNGR_DB_CFG_RSRC_ID_SEL_SHFT 0x8
-#define HWIO_IPA_RSRC_MNGR_DB_CFG_RSRC_TYPE_SEL_BMSK 0x70
-#define HWIO_IPA_RSRC_MNGR_DB_CFG_RSRC_TYPE_SEL_SHFT 0x4
-#define HWIO_IPA_RSRC_MNGR_DB_CFG_RSRC_GRP_SEL_BMSK 0x7
-#define HWIO_IPA_RSRC_MNGR_DB_CFG_RSRC_GRP_SEL_SHFT 0x0
-#define HWIO_IPA_RSRC_MNGR_DB_RSRC_READ_ADDR (IPA_DEBUG_REG_BASE + \
-					      0x0000012c)
-#define HWIO_IPA_RSRC_MNGR_DB_RSRC_READ_PHYS (IPA_DEBUG_REG_BASE_PHYS +	\
-					      0x0000012c)
-#define HWIO_IPA_RSRC_MNGR_DB_RSRC_READ_OFFS (IPA_DEBUG_REG_BASE_OFFS +	\
-					      0x0000012c)
-#define HWIO_IPA_RSRC_MNGR_DB_RSRC_READ_RMSK 0x3f3
-#define HWIO_IPA_RSRC_MNGR_DB_RSRC_READ_ATTR 0x1
-#define HWIO_IPA_RSRC_MNGR_DB_RSRC_READ_IN in_dword_masked( \
-		HWIO_IPA_RSRC_MNGR_DB_RSRC_READ_ADDR, \
-		HWIO_IPA_RSRC_MNGR_DB_RSRC_READ_RMSK)
-#define HWIO_IPA_RSRC_MNGR_DB_RSRC_READ_INM(m) in_dword_masked(	\
-		HWIO_IPA_RSRC_MNGR_DB_RSRC_READ_ADDR, \
-		m)
-#define HWIO_IPA_RSRC_MNGR_DB_RSRC_READ_RSRC_NEXT_INDEX_BMSK 0x3f0
-#define HWIO_IPA_RSRC_MNGR_DB_RSRC_READ_RSRC_NEXT_INDEX_SHFT 0x4
-#define HWIO_IPA_RSRC_MNGR_DB_RSRC_READ_RSRC_NEXT_VALID_BMSK 0x2
-#define HWIO_IPA_RSRC_MNGR_DB_RSRC_READ_RSRC_NEXT_VALID_SHFT 0x1
-#define HWIO_IPA_RSRC_MNGR_DB_RSRC_READ_RSRC_OCCUPIED_BMSK 0x1
-#define HWIO_IPA_RSRC_MNGR_DB_RSRC_READ_RSRC_OCCUPIED_SHFT 0x0
-#define HWIO_IPA_RSRC_MNGR_DB_LIST_READ_ADDR (IPA_DEBUG_REG_BASE + \
-					      0x00000130)
-#define HWIO_IPA_RSRC_MNGR_DB_LIST_READ_PHYS (IPA_DEBUG_REG_BASE_PHYS +	\
-					      0x00000130)
-#define HWIO_IPA_RSRC_MNGR_DB_LIST_READ_OFFS (IPA_DEBUG_REG_BASE_OFFS +	\
-					      0x00000130)
-#define HWIO_IPA_RSRC_MNGR_DB_LIST_READ_RMSK 0x7f7f3f3
-#define HWIO_IPA_RSRC_MNGR_DB_LIST_READ_ATTR 0x1
-#define HWIO_IPA_RSRC_MNGR_DB_LIST_READ_IN in_dword_masked( \
-		HWIO_IPA_RSRC_MNGR_DB_LIST_READ_ADDR, \
-		HWIO_IPA_RSRC_MNGR_DB_LIST_READ_RMSK)
-#define HWIO_IPA_RSRC_MNGR_DB_LIST_READ_INM(m) in_dword_masked(	\
-		HWIO_IPA_RSRC_MNGR_DB_LIST_READ_ADDR, \
-		m)
-#define HWIO_IPA_RSRC_MNGR_DB_LIST_READ_RSRC_LIST_ENTRY_CNT_BMSK 0x7f00000
-#define HWIO_IPA_RSRC_MNGR_DB_LIST_READ_RSRC_LIST_ENTRY_CNT_SHFT 0x14
-#define HWIO_IPA_RSRC_MNGR_DB_LIST_READ_RSRC_LIST_HEAD_CNT_BMSK 0x7f000
-#define HWIO_IPA_RSRC_MNGR_DB_LIST_READ_RSRC_LIST_HEAD_CNT_SHFT 0xc
-#define HWIO_IPA_RSRC_MNGR_DB_LIST_READ_RSRC_LIST_HEAD_RSRC_BMSK 0x3f0
-#define HWIO_IPA_RSRC_MNGR_DB_LIST_READ_RSRC_LIST_HEAD_RSRC_SHFT 0x4
-#define HWIO_IPA_RSRC_MNGR_DB_LIST_READ_RSRC_LIST_HOLD_BMSK 0x2
-#define HWIO_IPA_RSRC_MNGR_DB_LIST_READ_RSRC_LIST_HOLD_SHFT 0x1
-#define HWIO_IPA_RSRC_MNGR_DB_LIST_READ_RSRC_LIST_VALID_BMSK 0x1
-#define HWIO_IPA_RSRC_MNGR_DB_LIST_READ_RSRC_LIST_VALID_SHFT 0x0
-#define HWIO_IPA_RSRC_MNGR_CONTEXTS_ADDR (IPA_DEBUG_REG_BASE + 0x00000134)
-#define HWIO_IPA_RSRC_MNGR_CONTEXTS_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					  0x00000134)
-#define HWIO_IPA_RSRC_MNGR_CONTEXTS_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					  0x00000134)
-#define HWIO_IPA_BRESP_DB_CFG_ADDR (IPA_DEBUG_REG_BASE + 0x00000138)
-#define HWIO_IPA_BRESP_DB_CFG_PHYS (IPA_DEBUG_REG_BASE_PHYS + 0x00000138)
-#define HWIO_IPA_BRESP_DB_CFG_OFFS (IPA_DEBUG_REG_BASE_OFFS + 0x00000138)
-#define HWIO_IPA_BRESP_DB_DATA_ADDR (IPA_DEBUG_REG_BASE + 0x0000013c)
-#define HWIO_IPA_BRESP_DB_DATA_PHYS (IPA_DEBUG_REG_BASE_PHYS + 0x0000013c)
-#define HWIO_IPA_BRESP_DB_DATA_OFFS (IPA_DEBUG_REG_BASE_OFFS + 0x0000013c)
-#define HWIO_IPA_DEBUG_DATA_ADDR (IPA_DEBUG_REG_BASE + 0x00000204)
-#define HWIO_IPA_DEBUG_DATA_PHYS (IPA_DEBUG_REG_BASE_PHYS + 0x00000204)
-#define HWIO_IPA_DEBUG_DATA_OFFS (IPA_DEBUG_REG_BASE_OFFS + 0x00000204)
-#define HWIO_IPA_DEBUG_DATA_RMSK 0xffffffff
-#define HWIO_IPA_DEBUG_DATA_ATTR 0x1
-#define HWIO_IPA_DEBUG_DATA_IN in_dword_masked(HWIO_IPA_DEBUG_DATA_ADDR, \
-					       HWIO_IPA_DEBUG_DATA_RMSK)
-#define HWIO_IPA_DEBUG_DATA_INM(m) in_dword_masked( \
-		HWIO_IPA_DEBUG_DATA_ADDR, \
-		m)
-#define HWIO_IPA_DEBUG_DATA_DEBUG_DATA_BMSK 0xffffffff
-#define HWIO_IPA_DEBUG_DATA_DEBUG_DATA_SHFT 0x0
-#define HWIO_IPA_TESTBUS_SEL_ADDR (IPA_DEBUG_REG_BASE + 0x00000208)
-#define HWIO_IPA_TESTBUS_SEL_PHYS (IPA_DEBUG_REG_BASE_PHYS + 0x00000208)
-#define HWIO_IPA_TESTBUS_SEL_OFFS (IPA_DEBUG_REG_BASE_OFFS + 0x00000208)
-#define HWIO_IPA_TESTBUS_SEL_RMSK 0x1fffff1
-#define HWIO_IPA_TESTBUS_SEL_ATTR 0x3
-#define HWIO_IPA_TESTBUS_SEL_IN in_dword_masked(HWIO_IPA_TESTBUS_SEL_ADDR, \
-						HWIO_IPA_TESTBUS_SEL_RMSK)
-#define HWIO_IPA_TESTBUS_SEL_INM(m) in_dword_masked( \
-		HWIO_IPA_TESTBUS_SEL_ADDR, \
-		m)
-#define HWIO_IPA_TESTBUS_SEL_OUT(v) out_dword(HWIO_IPA_TESTBUS_SEL_ADDR, v)
-#define HWIO_IPA_TESTBUS_SEL_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_TESTBUS_SEL_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_TESTBUS_SEL_IN)
-#define HWIO_IPA_TESTBUS_SEL_PIPE_SELECT_BMSK 0x1f00000
-#define HWIO_IPA_TESTBUS_SEL_PIPE_SELECT_SHFT 0x14
-#define HWIO_IPA_TESTBUS_SEL_INTERNAL_BLOCK_SELECT_BMSK 0xff000
-#define HWIO_IPA_TESTBUS_SEL_INTERNAL_BLOCK_SELECT_SHFT 0xc
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_BMSK 0xff0
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_SHFT 0x4
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_IPA_RX_FVAL 0x0
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_IPA_TX0_FVAL 0x1
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_HPS_FRAG_FVAL 0x2
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_HPS_UCP_FVAL 0x3
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_HPS_ENQUEUER_FVAL 0x4
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_HPS_ROUTER_FVAL 0x5
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_HPS_PKT_PARSER_FVAL 0x6
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_HPS_FILTER_NAT_FVAL 0x7
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_HPS_HDRI_RSRCREL_FVAL \
-	0x8
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_IPA_AHB2AHB_FVAL 0x9
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_IPA_MAXI2AXI_FVAL 0xa
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_DPS_DCMP_FVAL 0xb
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_DPS_DISPATCHER_FVAL 0xc
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_DPS_D_DCPH_FVAL 0xd
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_GSI_TEST_BUS_FVAL 0xe
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_DEADBEAF_FVAL 0xf
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_IPA_MISC_FVAL 0x10
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_IPA_STTS_SNIFFER_FVAL \
-	0x11
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_IPA_QMB_0_FVAL 0x12
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_IPA_QMB_1_FVAL 0x13
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_IPA_UC_ACKQ_FVAL 0x14
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_IPA_RX_ACKQ_FVAL 0x15
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_IPA_TX1_FVAL 0x16
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_HPS_H_DCPH_FVAL 0x17
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_RX_HPS_CMDQ_FVAL 0x18
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_HPS_DPS_CMDQ_FVAL 0x19
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_DPS_TX_CMDQ_FVAL 0x1a
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_IPA_CMDQ_L_FVAL 0x1b
-#define	\
-	HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_IPA_RX_LEGACY_CMDQ_INT_FVAL \
-	0x1c
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_IPA_CTX_HANDLER_FVAL	\
-	0x1d
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_IPA_GSI_FVAL 0x1e
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_ACK_MNGR_CMDQ_FVAL 0x1f
-#define	\
-	HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_ENDP_INIT_CTRL_SUSPEND_FVAL \
-	0x20
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_ACL_WRAPPER_FVAL 0x22
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_IPA_TX_WRAPPER_FVAL \
-	0x23
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_AHB2AHB_BRIDGE_FVAL \
-	0x24
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_RSRC_TYPE_FVAL 0x31
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_RSRC_FVAL 0x32
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_ACKMNGR_FVAL 0x33
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_HPS_SEQ_FVAL 0x34
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_DPS_SEQ_FVAL 0x35
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_HPS_FTCH_FVAL 0x36
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_DPS_FTCH_FVAL 0x37
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_HPS_D_DCPH_2_FVAL 0x38
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_NTF_TX_CMDQ_FVAL 0x39
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_PROD_ACK_MNGR_CMDQ_FVAL \
-	0x3a
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_PROD_ACKMNGR_FVAL 0x3b
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_IPA_GSI_AHB2AHB_FVAL	\
-	0x3c
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_IPA_MAXI2AXI_PCIE_FVAL \
-	0x3d
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_IPA_QSB2AXI_FVAL 0x3e
-#define HWIO_IPA_TESTBUS_SEL_EXTERNAL_BLOCK_SELECT_IPA_UC_FVAL 0x3f
-#define HWIO_IPA_TESTBUS_SEL_TESTBUS_EN_BMSK 0x1
-#define HWIO_IPA_TESTBUS_SEL_TESTBUS_EN_SHFT 0x0
-#define HWIO_IPA_STEP_MODE_BREAKPOINTS_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x0000020c)
-#define HWIO_IPA_STEP_MODE_BREAKPOINTS_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x0000020c)
-#define HWIO_IPA_STEP_MODE_BREAKPOINTS_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x0000020c)
-#define HWIO_IPA_STEP_MODE_BREAKPOINTS_RMSK 0xffffffff
-#define HWIO_IPA_STEP_MODE_BREAKPOINTS_ATTR 0x3
-#define HWIO_IPA_STEP_MODE_BREAKPOINTS_IN in_dword_masked( \
-		HWIO_IPA_STEP_MODE_BREAKPOINTS_ADDR, \
-		HWIO_IPA_STEP_MODE_BREAKPOINTS_RMSK)
-#define HWIO_IPA_STEP_MODE_BREAKPOINTS_INM(m) in_dword_masked( \
-		HWIO_IPA_STEP_MODE_BREAKPOINTS_ADDR, \
-		m)
-#define HWIO_IPA_STEP_MODE_BREAKPOINTS_OUT(v) out_dword( \
-		HWIO_IPA_STEP_MODE_BREAKPOINTS_ADDR, \
-		v)
-#define HWIO_IPA_STEP_MODE_BREAKPOINTS_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_STEP_MODE_BREAKPOINTS_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_STEP_MODE_BREAKPOINTS_IN)
-#define HWIO_IPA_STEP_MODE_BREAKPOINTS_HW_EN_BMSK 0xffffffff
-#define HWIO_IPA_STEP_MODE_BREAKPOINTS_HW_EN_SHFT 0x0
-#define HWIO_IPA_STEP_MODE_STATUS_ADDR (IPA_DEBUG_REG_BASE + 0x00000210)
-#define HWIO_IPA_STEP_MODE_STATUS_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					0x00000210)
-#define HWIO_IPA_STEP_MODE_STATUS_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					0x00000210)
-#define HWIO_IPA_STEP_MODE_STATUS_RMSK 0xffffffff
-#define HWIO_IPA_STEP_MODE_STATUS_ATTR 0x1
-#define HWIO_IPA_STEP_MODE_STATUS_IN in_dword_masked( \
-		HWIO_IPA_STEP_MODE_STATUS_ADDR,	\
-		HWIO_IPA_STEP_MODE_STATUS_RMSK)
-#define HWIO_IPA_STEP_MODE_STATUS_INM(m) in_dword_masked( \
-		HWIO_IPA_STEP_MODE_STATUS_ADDR,	\
-		m)
-#define HWIO_IPA_STEP_MODE_STATUS_HW_EN_BMSK 0xffffffff
-#define HWIO_IPA_STEP_MODE_STATUS_HW_EN_SHFT 0x0
-#define HWIO_IPA_STEP_MODE_GO_ADDR (IPA_DEBUG_REG_BASE + 0x00000214)
-#define HWIO_IPA_STEP_MODE_GO_PHYS (IPA_DEBUG_REG_BASE_PHYS + 0x00000214)
-#define HWIO_IPA_STEP_MODE_GO_OFFS (IPA_DEBUG_REG_BASE_OFFS + 0x00000214)
-#define HWIO_IPA_HW_EVENTS_CFG_ADDR (IPA_DEBUG_REG_BASE + 0x00000218)
-#define HWIO_IPA_HW_EVENTS_CFG_PHYS (IPA_DEBUG_REG_BASE_PHYS + 0x00000218)
-#define HWIO_IPA_HW_EVENTS_CFG_OFFS (IPA_DEBUG_REG_BASE_OFFS + 0x00000218)
-#define HWIO_IPA_LOG_ADDR (IPA_DEBUG_REG_BASE + 0x0000021c)
-#define HWIO_IPA_LOG_PHYS (IPA_DEBUG_REG_BASE_PHYS + 0x0000021c)
-#define HWIO_IPA_LOG_OFFS (IPA_DEBUG_REG_BASE_OFFS + 0x0000021c)
-#define HWIO_IPA_LOG_RMSK 0x3ff1f2
-#define HWIO_IPA_LOG_ATTR 0x3
-#define HWIO_IPA_LOG_IN in_dword_masked(HWIO_IPA_LOG_ADDR, \
-					HWIO_IPA_LOG_RMSK)
-#define HWIO_IPA_LOG_INM(m) in_dword_masked(HWIO_IPA_LOG_ADDR, m)
-#define HWIO_IPA_LOG_OUT(v) out_dword(HWIO_IPA_LOG_ADDR, v)
-#define HWIO_IPA_LOG_OUTM(m, v) out_dword_masked_ns(HWIO_IPA_LOG_ADDR, \
-						    m, \
-						    v, \
-						    HWIO_IPA_LOG_IN)
-#define HWIO_IPA_LOG_LOG_DPL_L2_REMOVE_EN_BMSK 0x200000
-#define HWIO_IPA_LOG_LOG_DPL_L2_REMOVE_EN_SHFT 0x15
-#define HWIO_IPA_LOG_LOG_REDUCTION_EN_BMSK 0x100000
-#define HWIO_IPA_LOG_LOG_REDUCTION_EN_SHFT 0x14
-#define HWIO_IPA_LOG_LOG_LENGTH_BMSK 0xff000
-#define HWIO_IPA_LOG_LOG_LENGTH_SHFT 0xc
-#define HWIO_IPA_LOG_LOG_PIPE_BMSK 0x1f0
-#define HWIO_IPA_LOG_LOG_PIPE_SHFT 0x4
-#define HWIO_IPA_LOG_LOG_EN_BMSK 0x2
-#define HWIO_IPA_LOG_LOG_EN_SHFT 0x1
-#define HWIO_IPA_LOG_BUF_HW_CMD_ADDR_ADDR (IPA_DEBUG_REG_BASE + 0x00000224)
-#define HWIO_IPA_LOG_BUF_HW_CMD_ADDR_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					   0x00000224)
-#define HWIO_IPA_LOG_BUF_HW_CMD_ADDR_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					   0x00000224)
-#define HWIO_IPA_LOG_BUF_HW_CMD_ADDR_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_HW_CMD_ADDR_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_HW_CMD_ADDR_IN in_dword_masked( \
-		HWIO_IPA_LOG_BUF_HW_CMD_ADDR_ADDR, \
-		HWIO_IPA_LOG_BUF_HW_CMD_ADDR_RMSK)
-#define HWIO_IPA_LOG_BUF_HW_CMD_ADDR_INM(m) in_dword_masked( \
-		HWIO_IPA_LOG_BUF_HW_CMD_ADDR_ADDR, \
-		m)
-#define HWIO_IPA_LOG_BUF_HW_CMD_ADDR_OUT(v) out_dword( \
-		HWIO_IPA_LOG_BUF_HW_CMD_ADDR_ADDR, \
-		v)
-#define HWIO_IPA_LOG_BUF_HW_CMD_ADDR_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_HW_CMD_ADDR_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_LOG_BUF_HW_CMD_ADDR_IN)
-#define HWIO_IPA_LOG_BUF_HW_CMD_ADDR_START_ADDR_BMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_HW_CMD_ADDR_START_ADDR_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_HW_CMD_ADDR_MSB_ADDR (IPA_DEBUG_REG_BASE + \
-					       0x00000228)
-#define HWIO_IPA_LOG_BUF_HW_CMD_ADDR_MSB_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					       0x00000228)
-#define HWIO_IPA_LOG_BUF_HW_CMD_ADDR_MSB_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					       0x00000228)
-#define HWIO_IPA_LOG_BUF_HW_CMD_ADDR_MSB_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_HW_CMD_ADDR_MSB_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_HW_CMD_ADDR_MSB_IN in_dword_masked( \
-		HWIO_IPA_LOG_BUF_HW_CMD_ADDR_MSB_ADDR, \
-		HWIO_IPA_LOG_BUF_HW_CMD_ADDR_MSB_RMSK)
-#define HWIO_IPA_LOG_BUF_HW_CMD_ADDR_MSB_INM(m) in_dword_masked( \
-		HWIO_IPA_LOG_BUF_HW_CMD_ADDR_MSB_ADDR, \
-		m)
-#define HWIO_IPA_LOG_BUF_HW_CMD_ADDR_MSB_OUT(v) out_dword( \
-		HWIO_IPA_LOG_BUF_HW_CMD_ADDR_MSB_ADDR, \
-		v)
-#define HWIO_IPA_LOG_BUF_HW_CMD_ADDR_MSB_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_HW_CMD_ADDR_MSB_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_LOG_BUF_HW_CMD_ADDR_MSB_IN)
-#define HWIO_IPA_LOG_BUF_HW_CMD_ADDR_MSB_START_ADDR_BMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_HW_CMD_ADDR_MSB_START_ADDR_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_HW_CMD_WRITE_PTR_ADDR (IPA_DEBUG_REG_BASE + \
-						0x0000022c)
-#define HWIO_IPA_LOG_BUF_HW_CMD_WRITE_PTR_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						0x0000022c)
-#define HWIO_IPA_LOG_BUF_HW_CMD_WRITE_PTR_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						0x0000022c)
-#define HWIO_IPA_LOG_BUF_HW_CMD_WRITE_PTR_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_HW_CMD_WRITE_PTR_ATTR 0x1
-#define HWIO_IPA_LOG_BUF_HW_CMD_WRITE_PTR_IN in_dword_masked( \
-		HWIO_IPA_LOG_BUF_HW_CMD_WRITE_PTR_ADDR,	\
-		HWIO_IPA_LOG_BUF_HW_CMD_WRITE_PTR_RMSK)
-#define HWIO_IPA_LOG_BUF_HW_CMD_WRITE_PTR_INM(m) in_dword_masked( \
-		HWIO_IPA_LOG_BUF_HW_CMD_WRITE_PTR_ADDR,	\
-		m)
-#define HWIO_IPA_LOG_BUF_HW_CMD_WRITE_PTR_WRITR_ADDR_BMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_HW_CMD_WRITE_PTR_WRITR_ADDR_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_HW_CMD_WRITE_PTR_MSB_ADDR (IPA_DEBUG_REG_BASE + \
-						    0x00000230)
-#define HWIO_IPA_LOG_BUF_HW_CMD_WRITE_PTR_MSB_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000230)
-#define HWIO_IPA_LOG_BUF_HW_CMD_WRITE_PTR_MSB_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000230)
-#define HWIO_IPA_LOG_BUF_HW_CMD_WRITE_PTR_MSB_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_HW_CMD_WRITE_PTR_MSB_ATTR 0x1
-#define HWIO_IPA_LOG_BUF_HW_CMD_WRITE_PTR_MSB_IN in_dword_masked( \
-		HWIO_IPA_LOG_BUF_HW_CMD_WRITE_PTR_MSB_ADDR, \
-		HWIO_IPA_LOG_BUF_HW_CMD_WRITE_PTR_MSB_RMSK)
-#define HWIO_IPA_LOG_BUF_HW_CMD_WRITE_PTR_MSB_INM(m) in_dword_masked( \
-		HWIO_IPA_LOG_BUF_HW_CMD_WRITE_PTR_MSB_ADDR, \
-		m)
-#define HWIO_IPA_LOG_BUF_HW_CMD_WRITE_PTR_MSB_WRITR_ADDR_BMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_HW_CMD_WRITE_PTR_MSB_WRITR_ADDR_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_HW_CMD_CFG_ADDR (IPA_DEBUG_REG_BASE + 0x00000234)
-#define HWIO_IPA_LOG_BUF_HW_CMD_CFG_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					  0x00000234)
-#define HWIO_IPA_LOG_BUF_HW_CMD_CFG_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					  0x00000234)
-#define HWIO_IPA_LOG_BUF_HW_CMD_CFG_RMSK 0x3ffff
-#define HWIO_IPA_LOG_BUF_HW_CMD_CFG_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_HW_CMD_CFG_IN in_dword_masked(	\
-		HWIO_IPA_LOG_BUF_HW_CMD_CFG_ADDR, \
-		HWIO_IPA_LOG_BUF_HW_CMD_CFG_RMSK)
-#define HWIO_IPA_LOG_BUF_HW_CMD_CFG_INM(m) in_dword_masked( \
-		HWIO_IPA_LOG_BUF_HW_CMD_CFG_ADDR, \
-		m)
-#define HWIO_IPA_LOG_BUF_HW_CMD_CFG_OUT(v) out_dword( \
-		HWIO_IPA_LOG_BUF_HW_CMD_CFG_ADDR, \
-		v)
-#define HWIO_IPA_LOG_BUF_HW_CMD_CFG_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_HW_CMD_CFG_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_LOG_BUF_HW_CMD_CFG_IN)
-#define HWIO_IPA_LOG_BUF_HW_CMD_CFG_SKIP_DDR_DMA_BMSK 0x20000
-#define HWIO_IPA_LOG_BUF_HW_CMD_CFG_SKIP_DDR_DMA_SHFT 0x11
-#define HWIO_IPA_LOG_BUF_HW_CMD_CFG_ENABLE_BMSK 0x10000
-#define HWIO_IPA_LOG_BUF_HW_CMD_CFG_ENABLE_SHFT 0x10
-#define HWIO_IPA_LOG_BUF_HW_CMD_CFG_SIZE_BMSK 0xffff
-#define HWIO_IPA_LOG_BUF_HW_CMD_CFG_SIZE_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_HW_CMD_RAM_PTR_ADDR (IPA_DEBUG_REG_BASE + \
-					      0x00000238)
-#define HWIO_IPA_LOG_BUF_HW_CMD_RAM_PTR_PHYS (IPA_DEBUG_REG_BASE_PHYS +	\
-					      0x00000238)
-#define HWIO_IPA_LOG_BUF_HW_CMD_RAM_PTR_OFFS (IPA_DEBUG_REG_BASE_OFFS +	\
-					      0x00000238)
-#define HWIO_IPA_LOG_BUF_HW_CMD_RAM_PTR_RMSK 0xbfff3fff
-#define HWIO_IPA_LOG_BUF_HW_CMD_RAM_PTR_ATTR 0x1
-#define HWIO_IPA_LOG_BUF_HW_CMD_RAM_PTR_IN in_dword_masked( \
-		HWIO_IPA_LOG_BUF_HW_CMD_RAM_PTR_ADDR, \
-		HWIO_IPA_LOG_BUF_HW_CMD_RAM_PTR_RMSK)
-#define HWIO_IPA_LOG_BUF_HW_CMD_RAM_PTR_INM(m) in_dword_masked(	\
-		HWIO_IPA_LOG_BUF_HW_CMD_RAM_PTR_ADDR, \
-		m)
-#define HWIO_IPA_LOG_BUF_HW_CMD_RAM_PTR_SKIP_DDR_WRAP_HAPPENED_BMSK \
-	0x80000000
-#define HWIO_IPA_LOG_BUF_HW_CMD_RAM_PTR_SKIP_DDR_WRAP_HAPPENED_SHFT 0x1f
-#define HWIO_IPA_LOG_BUF_HW_CMD_RAM_PTR_WRITE_PTR_BMSK 0x3fff0000
-#define HWIO_IPA_LOG_BUF_HW_CMD_RAM_PTR_WRITE_PTR_SHFT 0x10
-#define HWIO_IPA_LOG_BUF_HW_CMD_RAM_PTR_READ_PTR_BMSK 0x3fff
-#define HWIO_IPA_LOG_BUF_HW_CMD_RAM_PTR_READ_PTR_SHFT 0x0
-#define HWIO_IPA_STEP_MODE_HFETCHER_ADDR_LSB_ADDR (IPA_DEBUG_REG_BASE +	\
-						   0x0000023c)
-#define HWIO_IPA_STEP_MODE_HFETCHER_ADDR_LSB_PHYS (IPA_DEBUG_REG_BASE_PHYS \
-						   + 0x0000023c)
-#define HWIO_IPA_STEP_MODE_HFETCHER_ADDR_LSB_OFFS (IPA_DEBUG_REG_BASE_OFFS \
-						   + 0x0000023c)
-#define HWIO_IPA_STEP_MODE_HFETCHER_ADDR_MSB_ADDR (IPA_DEBUG_REG_BASE +	\
-						   0x00000240)
-#define HWIO_IPA_STEP_MODE_HFETCHER_ADDR_MSB_PHYS (IPA_DEBUG_REG_BASE_PHYS \
-						   + 0x00000240)
-#define HWIO_IPA_STEP_MODE_HFETCHER_ADDR_MSB_OFFS (IPA_DEBUG_REG_BASE_OFFS \
-						   + 0x00000240)
-#define HWIO_IPA_STEP_MODE_HFETCHER_ADDR_RESULT_ADDR (IPA_DEBUG_REG_BASE + \
-						      0x00000244)
-#define HWIO_IPA_STEP_MODE_HFETCHER_ADDR_RESULT_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000244)
-#define HWIO_IPA_STEP_MODE_HFETCHER_ADDR_RESULT_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000244)
-#define HWIO_IPA_STEP_MODE_HSEQ_BREAKPOINT_ADDR (IPA_DEBUG_REG_BASE + \
-						 0x00000248)
-#define HWIO_IPA_STEP_MODE_HSEQ_BREAKPOINT_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						 0x00000248)
-#define HWIO_IPA_STEP_MODE_HSEQ_BREAKPOINT_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						 0x00000248)
-#define HWIO_IPA_STEP_MODE_HSEQ_STATUS_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x0000024c)
-#define HWIO_IPA_STEP_MODE_HSEQ_STATUS_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x0000024c)
-#define HWIO_IPA_STEP_MODE_HSEQ_STATUS_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x0000024c)
-#define HWIO_IPA_STEP_MODE_DSEQ_BREAKPOINT_ADDR (IPA_DEBUG_REG_BASE + \
-						 0x00000250)
-#define HWIO_IPA_STEP_MODE_DSEQ_BREAKPOINT_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						 0x00000250)
-#define HWIO_IPA_STEP_MODE_DSEQ_BREAKPOINT_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						 0x00000250)
-#define HWIO_IPA_STEP_MODE_DSEQ_STATUS_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x00000254)
-#define HWIO_IPA_STEP_MODE_DSEQ_STATUS_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x00000254)
-#define HWIO_IPA_STEP_MODE_DSEQ_STATUS_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x00000254)
-#define HWIO_IPA_RX_ACKQ_CMD_ADDR (IPA_DEBUG_REG_BASE + 0x00000258)
-#define HWIO_IPA_RX_ACKQ_CMD_PHYS (IPA_DEBUG_REG_BASE_PHYS + 0x00000258)
-#define HWIO_IPA_RX_ACKQ_CMD_OFFS (IPA_DEBUG_REG_BASE_OFFS + 0x00000258)
-#define HWIO_IPA_RX_ACKQ_CFG_ADDR (IPA_DEBUG_REG_BASE + 0x0000025c)
-#define HWIO_IPA_RX_ACKQ_CFG_PHYS (IPA_DEBUG_REG_BASE_PHYS + 0x0000025c)
-#define HWIO_IPA_RX_ACKQ_CFG_OFFS (IPA_DEBUG_REG_BASE_OFFS + 0x0000025c)
-#define HWIO_IPA_RX_ACKQ_DATA_WR_0_ADDR (IPA_DEBUG_REG_BASE + 0x00000260)
-#define HWIO_IPA_RX_ACKQ_DATA_WR_0_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					 0x00000260)
-#define HWIO_IPA_RX_ACKQ_DATA_WR_0_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					 0x00000260)
-#define HWIO_IPA_RX_ACKQ_DATA_RD_0_ADDR (IPA_DEBUG_REG_BASE + 0x00000264)
-#define HWIO_IPA_RX_ACKQ_DATA_RD_0_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					 0x00000264)
-#define HWIO_IPA_RX_ACKQ_DATA_RD_0_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					 0x00000264)
-#define HWIO_IPA_RX_ACKQ_STATUS_ADDR (IPA_DEBUG_REG_BASE + 0x00000268)
-#define HWIO_IPA_RX_ACKQ_STATUS_PHYS (IPA_DEBUG_REG_BASE_PHYS + 0x00000268)
-#define HWIO_IPA_RX_ACKQ_STATUS_OFFS (IPA_DEBUG_REG_BASE_OFFS + 0x00000268)
-#define HWIO_IPA_UC_ACKQ_CMD_ADDR (IPA_DEBUG_REG_BASE + 0x0000026c)
-#define HWIO_IPA_UC_ACKQ_CMD_PHYS (IPA_DEBUG_REG_BASE_PHYS + 0x0000026c)
-#define HWIO_IPA_UC_ACKQ_CMD_OFFS (IPA_DEBUG_REG_BASE_OFFS + 0x0000026c)
-#define HWIO_IPA_UC_ACKQ_CFG_ADDR (IPA_DEBUG_REG_BASE + 0x00000270)
-#define HWIO_IPA_UC_ACKQ_CFG_PHYS (IPA_DEBUG_REG_BASE_PHYS + 0x00000270)
-#define HWIO_IPA_UC_ACKQ_CFG_OFFS (IPA_DEBUG_REG_BASE_OFFS + 0x00000270)
-#define HWIO_IPA_UC_ACKQ_DATA_WR_0_ADDR (IPA_DEBUG_REG_BASE + 0x00000274)
-#define HWIO_IPA_UC_ACKQ_DATA_WR_0_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					 0x00000274)
-#define HWIO_IPA_UC_ACKQ_DATA_WR_0_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					 0x00000274)
-#define HWIO_IPA_UC_ACKQ_DATA_RD_0_ADDR (IPA_DEBUG_REG_BASE + 0x00000278)
-#define HWIO_IPA_UC_ACKQ_DATA_RD_0_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					 0x00000278)
-#define HWIO_IPA_UC_ACKQ_DATA_RD_0_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					 0x00000278)
-#define HWIO_IPA_UC_ACKQ_STATUS_ADDR (IPA_DEBUG_REG_BASE + 0x0000027c)
-#define HWIO_IPA_UC_ACKQ_STATUS_PHYS (IPA_DEBUG_REG_BASE_PHYS + 0x0000027c)
-#define HWIO_IPA_UC_ACKQ_STATUS_OFFS (IPA_DEBUG_REG_BASE_OFFS + 0x0000027c)
-#define HWIO_IPA_RX_SPLT_CMDQ_CMD_n_ADDR(n) (IPA_DEBUG_REG_BASE + \
-					     0x00000280 + 0x2C * (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_CMD_n_PHYS(n) (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x00000280 + 0x2C * (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_CMD_n_OFFS(n) (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x00000280 + 0x2C * (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_CMD_n_RMSK 0x7f
-#define HWIO_IPA_RX_SPLT_CMDQ_CMD_n_MAXn 3
-#define HWIO_IPA_RX_SPLT_CMDQ_CMD_n_ATTR 0x2
-#define HWIO_IPA_RX_SPLT_CMDQ_CMD_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_RX_SPLT_CMDQ_CMD_n_ADDR(n), \
-		val)
-#define HWIO_IPA_RX_SPLT_CMDQ_CMD_n_RELEASE_RD_PKT_ENHANCED_BMSK 0x40
-#define HWIO_IPA_RX_SPLT_CMDQ_CMD_n_RELEASE_RD_PKT_ENHANCED_SHFT 0x6
-#define HWIO_IPA_RX_SPLT_CMDQ_CMD_n_RELEASE_WR_PKT_BMSK 0x20
-#define HWIO_IPA_RX_SPLT_CMDQ_CMD_n_RELEASE_WR_PKT_SHFT 0x5
-#define HWIO_IPA_RX_SPLT_CMDQ_CMD_n_RELEASE_RD_PKT_BMSK 0x10
-#define HWIO_IPA_RX_SPLT_CMDQ_CMD_n_RELEASE_RD_PKT_SHFT 0x4
-#define HWIO_IPA_RX_SPLT_CMDQ_CMD_n_RELEASE_WR_CMD_BMSK 0x8
-#define HWIO_IPA_RX_SPLT_CMDQ_CMD_n_RELEASE_WR_CMD_SHFT 0x3
-#define HWIO_IPA_RX_SPLT_CMDQ_CMD_n_RELEASE_RD_CMD_BMSK 0x4
-#define HWIO_IPA_RX_SPLT_CMDQ_CMD_n_RELEASE_RD_CMD_SHFT 0x2
-#define HWIO_IPA_RX_SPLT_CMDQ_CMD_n_POP_CMD_BMSK 0x2
-#define HWIO_IPA_RX_SPLT_CMDQ_CMD_n_POP_CMD_SHFT 0x1
-#define HWIO_IPA_RX_SPLT_CMDQ_CMD_n_WRITE_CMD_BMSK 0x1
-#define HWIO_IPA_RX_SPLT_CMDQ_CMD_n_WRITE_CMD_SHFT 0x0
-#define HWIO_IPA_RX_SPLT_CMDQ_CFG_n_ADDR(n) (IPA_DEBUG_REG_BASE + \
-					     0x00000284 + 0x2C * (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_CFG_n_PHYS(n) (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x00000284 + 0x2C * (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_CFG_n_OFFS(n) (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x00000284 + 0x2C * (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_CFG_n_RMSK 0x3
-#define HWIO_IPA_RX_SPLT_CMDQ_CFG_n_MAXn 3
-#define HWIO_IPA_RX_SPLT_CMDQ_CFG_n_ATTR 0x3
-#define HWIO_IPA_RX_SPLT_CMDQ_CFG_n_INI(n) in_dword_masked( \
-		HWIO_IPA_RX_SPLT_CMDQ_CFG_n_ADDR(n), \
-		HWIO_IPA_RX_SPLT_CMDQ_CFG_n_RMSK)
-#define HWIO_IPA_RX_SPLT_CMDQ_CFG_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_RX_SPLT_CMDQ_CFG_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_RX_SPLT_CMDQ_CFG_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_RX_SPLT_CMDQ_CFG_n_ADDR(n), \
-		val)
-#define HWIO_IPA_RX_SPLT_CMDQ_CFG_n_OUTMI(n, mask, \
-					  val) out_dword_masked_ns( \
-		HWIO_IPA_RX_SPLT_CMDQ_CFG_n_ADDR( \
-			n), \
-		mask, \
-		val, \
-		HWIO_IPA_RX_SPLT_CMDQ_CFG_n_INI(n))
-#define HWIO_IPA_RX_SPLT_CMDQ_CFG_n_BLOCK_WR_BMSK 0x2
-#define HWIO_IPA_RX_SPLT_CMDQ_CFG_n_BLOCK_WR_SHFT 0x1
-#define HWIO_IPA_RX_SPLT_CMDQ_CFG_n_BLOCK_RD_BMSK 0x1
-#define HWIO_IPA_RX_SPLT_CMDQ_CFG_n_BLOCK_RD_SHFT 0x0
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_0_n_ADDR(n) (IPA_DEBUG_REG_BASE +	\
-						   0x00000288 + 0x2C * (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_0_n_PHYS(n) (IPA_DEBUG_REG_BASE_PHYS \
-						   + 0x00000288 + 0x2C * \
-						   (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_0_n_OFFS(n) (IPA_DEBUG_REG_BASE_OFFS \
-						   + 0x00000288 + 0x2C * \
-						   (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_0_n_RMSK 0xffffffff
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_0_n_MAXn 3
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_0_n_ATTR 0x3
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_0_n_INI(n) in_dword_masked( \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_0_n_ADDR(n), \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_0_n_RMSK)
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_0_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_0_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_0_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_0_n_ADDR(n), \
-		val)
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_0_n_OUTMI(n, mask, \
-						val) out_dword_masked_ns( \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_0_n_ADDR(	\
-			n), \
-		mask, \
-		val, \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_0_n_INI(n))
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_0_n_CMDQ_SRC_LEN_F_BMSK 0xffff0000
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_0_n_CMDQ_SRC_LEN_F_SHFT 0x10
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_0_n_CMDQ_PACKET_LEN_F_BMSK 0xffff
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_0_n_CMDQ_PACKET_LEN_F_SHFT 0x0
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_1_n_ADDR(n) (IPA_DEBUG_REG_BASE +	\
-						   0x0000028c + 0x2C * (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_1_n_PHYS(n) (IPA_DEBUG_REG_BASE_PHYS \
-						   + 0x0000028c + 0x2C * \
-						   (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_1_n_OFFS(n) (IPA_DEBUG_REG_BASE_OFFS \
-						   + 0x0000028c + 0x2C * \
-						   (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_1_n_RMSK 0xffffffff
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_1_n_MAXn 3
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_1_n_ATTR 0x3
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_1_n_INI(n) in_dword_masked( \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_1_n_ADDR(n), \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_1_n_RMSK)
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_1_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_1_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_1_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_1_n_ADDR(n), \
-		val)
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_1_n_OUTMI(n, mask, \
-						val) out_dword_masked_ns( \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_1_n_ADDR(	\
-			n), \
-		mask, \
-		val, \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_1_n_INI(n))
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_1_n_CMDQ_METADATA_F_BMSK 0xff000000
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_1_n_CMDQ_METADATA_F_SHFT 0x18
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_1_n_CMDQ_OPCODE_F_BMSK 0xff0000
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_1_n_CMDQ_OPCODE_F_SHFT 0x10
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_1_n_CMDQ_FLAGS_F_BMSK 0xfc00
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_1_n_CMDQ_FLAGS_F_SHFT 0xa
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_1_n_CMDQ_ORDER_F_BMSK 0x300
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_1_n_CMDQ_ORDER_F_SHFT 0x8
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_1_n_CMDQ_SRC_PIPE_F_BMSK 0xff
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_1_n_CMDQ_SRC_PIPE_F_SHFT 0x0
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_2_n_ADDR(n) (IPA_DEBUG_REG_BASE +	\
-						   0x00000290 + 0x2C * (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_2_n_PHYS(n) (IPA_DEBUG_REG_BASE_PHYS \
-						   + 0x00000290 + 0x2C * \
-						   (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_2_n_OFFS(n) (IPA_DEBUG_REG_BASE_OFFS \
-						   + 0x00000290 + 0x2C * \
-						   (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_2_n_RMSK 0xffffffff
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_2_n_MAXn 3
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_2_n_ATTR 0x3
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_2_n_INI(n) in_dword_masked( \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_2_n_ADDR(n), \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_2_n_RMSK)
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_2_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_2_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_2_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_2_n_ADDR(n), \
-		val)
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_2_n_OUTMI(n, mask, \
-						val) out_dword_masked_ns( \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_2_n_ADDR(	\
-			n), \
-		mask, \
-		val, \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_2_n_INI(n))
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_2_n_CMDQ_ADDR_LSB_F_BMSK 0xffffffff
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_2_n_CMDQ_ADDR_LSB_F_SHFT 0x0
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_3_n_ADDR(n) (IPA_DEBUG_REG_BASE +	\
-						   0x00000294 + 0x2C * (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_3_n_PHYS(n) (IPA_DEBUG_REG_BASE_PHYS \
-						   + 0x00000294 + 0x2C * \
-						   (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_3_n_OFFS(n) (IPA_DEBUG_REG_BASE_OFFS \
-						   + 0x00000294 + 0x2C * \
-						   (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_3_n_RMSK 0xffffffff
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_3_n_MAXn 3
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_3_n_ATTR 0x3
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_3_n_INI(n) in_dword_masked( \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_3_n_ADDR(n), \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_3_n_RMSK)
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_3_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_3_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_3_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_3_n_ADDR(n), \
-		val)
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_3_n_OUTMI(n, mask, \
-						val) out_dword_masked_ns( \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_3_n_ADDR(	\
-			n), \
-		mask, \
-		val, \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_3_n_INI(n))
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_3_n_CMDQ_ADDR_MSB_F_BMSK 0xffffffff
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_WR_3_n_CMDQ_ADDR_MSB_F_SHFT 0x0
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_0_n_ADDR(n) (IPA_DEBUG_REG_BASE +	\
-						   0x00000298 + 0x2C * (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_0_n_PHYS(n) (IPA_DEBUG_REG_BASE_PHYS \
-						   + 0x00000298 + 0x2C * \
-						   (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_0_n_OFFS(n) (IPA_DEBUG_REG_BASE_OFFS \
-						   + 0x00000298 + 0x2C * \
-						   (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_0_n_RMSK 0xffffffff
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_0_n_MAXn 3
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_0_n_ATTR 0x1
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_0_n_INI(n) in_dword_masked( \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_0_n_ADDR(n), \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_0_n_RMSK)
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_0_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_0_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_0_n_CMDQ_SRC_LEN_F_BMSK 0xffff0000
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_0_n_CMDQ_SRC_LEN_F_SHFT 0x10
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_0_n_CMDQ_PACKET_LEN_F_BMSK 0xffff
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_0_n_CMDQ_PACKET_LEN_F_SHFT 0x0
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_1_n_ADDR(n) (IPA_DEBUG_REG_BASE +	\
-						   0x0000029c + 0x2C * (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_1_n_PHYS(n) (IPA_DEBUG_REG_BASE_PHYS \
-						   + 0x0000029c + 0x2C * \
-						   (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_1_n_OFFS(n) (IPA_DEBUG_REG_BASE_OFFS \
-						   + 0x0000029c + 0x2C * \
-						   (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_1_n_RMSK 0xffffffff
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_1_n_MAXn 3
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_1_n_ATTR 0x1
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_1_n_INI(n) in_dword_masked( \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_1_n_ADDR(n), \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_1_n_RMSK)
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_1_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_1_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_1_n_CMDQ_METADATA_F_BMSK 0xff000000
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_1_n_CMDQ_METADATA_F_SHFT 0x18
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_1_n_CMDQ_OPCODE_F_BMSK 0xff0000
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_1_n_CMDQ_OPCODE_F_SHFT 0x10
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_1_n_CMDQ_FLAGS_F_BMSK 0xfc00
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_1_n_CMDQ_FLAGS_F_SHFT 0xa
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_1_n_CMDQ_ORDER_F_BMSK 0x300
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_1_n_CMDQ_ORDER_F_SHFT 0x8
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_1_n_CMDQ_SRC_PIPE_F_BMSK 0xff
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_1_n_CMDQ_SRC_PIPE_F_SHFT 0x0
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_2_n_ADDR(n) (IPA_DEBUG_REG_BASE +	\
-						   0x000002a0 + 0x2C * (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_2_n_PHYS(n) (IPA_DEBUG_REG_BASE_PHYS \
-						   + 0x000002a0 + 0x2C * \
-						   (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_2_n_OFFS(n) (IPA_DEBUG_REG_BASE_OFFS \
-						   + 0x000002a0 + 0x2C * \
-						   (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_2_n_RMSK 0xffffffff
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_2_n_MAXn 3
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_2_n_ATTR 0x1
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_2_n_INI(n) in_dword_masked( \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_2_n_ADDR(n), \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_2_n_RMSK)
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_2_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_2_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_2_n_CMDQ_ADDR_LSB_F_BMSK 0xffffffff
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_2_n_CMDQ_ADDR_LSB_F_SHFT 0x0
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_3_n_ADDR(n) (IPA_DEBUG_REG_BASE +	\
-						   0x000002a4 + 0x2C * (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_3_n_PHYS(n) (IPA_DEBUG_REG_BASE_PHYS \
-						   + 0x000002a4 + 0x2C * \
-						   (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_3_n_OFFS(n) (IPA_DEBUG_REG_BASE_OFFS \
-						   + 0x000002a4 + 0x2C * \
-						   (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_3_n_RMSK 0xffffffff
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_3_n_MAXn 3
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_3_n_ATTR 0x1
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_3_n_INI(n) in_dword_masked( \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_3_n_ADDR(n), \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_3_n_RMSK)
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_3_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_3_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_3_n_CMDQ_ADDR_MSB_F_BMSK 0xffffffff
-#define HWIO_IPA_RX_SPLT_CMDQ_DATA_RD_3_n_CMDQ_ADDR_MSB_F_SHFT 0x0
-#define HWIO_IPA_RX_SPLT_CMDQ_STATUS_n_ADDR(n) (IPA_DEBUG_REG_BASE + \
-						0x000002a8 + 0x2C * (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_STATUS_n_PHYS(n) (IPA_DEBUG_REG_BASE_PHYS + \
-						0x000002a8 + 0x2C * (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_STATUS_n_OFFS(n) (IPA_DEBUG_REG_BASE_OFFS + \
-						0x000002a8 + 0x2C * (n))
-#define HWIO_IPA_RX_SPLT_CMDQ_STATUS_n_RMSK 0x7f
-#define HWIO_IPA_RX_SPLT_CMDQ_STATUS_n_MAXn 3
-#define HWIO_IPA_RX_SPLT_CMDQ_STATUS_n_ATTR 0x1
-#define HWIO_IPA_RX_SPLT_CMDQ_STATUS_n_INI(n) in_dword_masked( \
-		HWIO_IPA_RX_SPLT_CMDQ_STATUS_n_ADDR(n),	\
-		HWIO_IPA_RX_SPLT_CMDQ_STATUS_n_RMSK)
-#define HWIO_IPA_RX_SPLT_CMDQ_STATUS_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_RX_SPLT_CMDQ_STATUS_n_ADDR(n),	\
-		mask)
-#define HWIO_IPA_RX_SPLT_CMDQ_STATUS_n_CMDQ_DEPTH_BMSK 0x60
-#define HWIO_IPA_RX_SPLT_CMDQ_STATUS_n_CMDQ_DEPTH_SHFT 0x5
-#define HWIO_IPA_RX_SPLT_CMDQ_STATUS_n_CMDQ_COUNT_BMSK 0x18
-#define HWIO_IPA_RX_SPLT_CMDQ_STATUS_n_CMDQ_COUNT_SHFT 0x3
-#define HWIO_IPA_RX_SPLT_CMDQ_STATUS_n_CMDQ_FULL_BMSK 0x4
-#define HWIO_IPA_RX_SPLT_CMDQ_STATUS_n_CMDQ_FULL_SHFT 0x2
-#define HWIO_IPA_RX_SPLT_CMDQ_STATUS_n_CMDQ_EMPTY_BMSK 0x2
-#define HWIO_IPA_RX_SPLT_CMDQ_STATUS_n_CMDQ_EMPTY_SHFT 0x1
-#define HWIO_IPA_RX_SPLT_CMDQ_STATUS_n_STATUS_BMSK 0x1
-#define HWIO_IPA_RX_SPLT_CMDQ_STATUS_n_STATUS_SHFT 0x0
-#define HWIO_IPA_TX_COMMANDER_CMDQ_CMD_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x0000035c)
-#define HWIO_IPA_TX_COMMANDER_CMDQ_CMD_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x0000035c)
-#define HWIO_IPA_TX_COMMANDER_CMDQ_CMD_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x0000035c)
-#define HWIO_IPA_TX_COMMANDER_CMDQ_CFG_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x00000360)
-#define HWIO_IPA_TX_COMMANDER_CMDQ_CFG_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x00000360)
-#define HWIO_IPA_TX_COMMANDER_CMDQ_CFG_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x00000360)
-#define HWIO_IPA_TX_COMMANDER_CMDQ_DATA_WR_0_ADDR (IPA_DEBUG_REG_BASE +	\
-						   0x00000364)
-#define HWIO_IPA_TX_COMMANDER_CMDQ_DATA_WR_0_PHYS (IPA_DEBUG_REG_BASE_PHYS \
-						   + 0x00000364)
-#define HWIO_IPA_TX_COMMANDER_CMDQ_DATA_WR_0_OFFS (IPA_DEBUG_REG_BASE_OFFS \
-						   + 0x00000364)
-#define HWIO_IPA_TX_COMMANDER_CMDQ_DATA_WR_1_ADDR (IPA_DEBUG_REG_BASE +	\
-						   0x00000368)
-#define HWIO_IPA_TX_COMMANDER_CMDQ_DATA_WR_1_PHYS (IPA_DEBUG_REG_BASE_PHYS \
-						   + 0x00000368)
-#define HWIO_IPA_TX_COMMANDER_CMDQ_DATA_WR_1_OFFS (IPA_DEBUG_REG_BASE_OFFS \
-						   + 0x00000368)
-#define HWIO_IPA_TX_COMMANDER_CMDQ_DATA_WR_2_ADDR (IPA_DEBUG_REG_BASE +	\
-						   0x0000036c)
-#define HWIO_IPA_TX_COMMANDER_CMDQ_DATA_WR_2_PHYS (IPA_DEBUG_REG_BASE_PHYS \
-						   + 0x0000036c)
-#define HWIO_IPA_TX_COMMANDER_CMDQ_DATA_WR_2_OFFS (IPA_DEBUG_REG_BASE_OFFS \
-						   + 0x0000036c)
-#define HWIO_IPA_TX_COMMANDER_CMDQ_DATA_RD_0_ADDR (IPA_DEBUG_REG_BASE +	\
-						   0x00000370)
-#define HWIO_IPA_TX_COMMANDER_CMDQ_DATA_RD_0_PHYS (IPA_DEBUG_REG_BASE_PHYS \
-						   + 0x00000370)
-#define HWIO_IPA_TX_COMMANDER_CMDQ_DATA_RD_0_OFFS (IPA_DEBUG_REG_BASE_OFFS \
-						   + 0x00000370)
-#define HWIO_IPA_TX_COMMANDER_CMDQ_DATA_RD_1_ADDR (IPA_DEBUG_REG_BASE +	\
-						   0x00000374)
-#define HWIO_IPA_TX_COMMANDER_CMDQ_DATA_RD_1_PHYS (IPA_DEBUG_REG_BASE_PHYS \
-						   + 0x00000374)
-#define HWIO_IPA_TX_COMMANDER_CMDQ_DATA_RD_1_OFFS (IPA_DEBUG_REG_BASE_OFFS \
-						   + 0x00000374)
-#define HWIO_IPA_TX_COMMANDER_CMDQ_DATA_RD_2_ADDR (IPA_DEBUG_REG_BASE +	\
-						   0x00000378)
-#define HWIO_IPA_TX_COMMANDER_CMDQ_DATA_RD_2_PHYS (IPA_DEBUG_REG_BASE_PHYS \
-						   + 0x00000378)
-#define HWIO_IPA_TX_COMMANDER_CMDQ_DATA_RD_2_OFFS (IPA_DEBUG_REG_BASE_OFFS \
-						   + 0x00000378)
-#define HWIO_IPA_TX_COMMANDER_CMDQ_STATUS_ADDR (IPA_DEBUG_REG_BASE + \
-						0x0000037c)
-#define HWIO_IPA_TX_COMMANDER_CMDQ_STATUS_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						0x0000037c)
-#define HWIO_IPA_TX_COMMANDER_CMDQ_STATUS_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						0x0000037c)
-#define HWIO_IPA_TX_COMMANDER_CMDQ_STATUS_RMSK 0x7
-#define HWIO_IPA_TX_COMMANDER_CMDQ_STATUS_ATTR 0x1
-#define HWIO_IPA_TX_COMMANDER_CMDQ_STATUS_IN in_dword_masked( \
-		HWIO_IPA_TX_COMMANDER_CMDQ_STATUS_ADDR,	\
-		HWIO_IPA_TX_COMMANDER_CMDQ_STATUS_RMSK)
-#define HWIO_IPA_TX_COMMANDER_CMDQ_STATUS_INM(m) in_dword_masked( \
-		HWIO_IPA_TX_COMMANDER_CMDQ_STATUS_ADDR,	\
-		m)
-#define HWIO_IPA_TX_COMMANDER_CMDQ_STATUS_CMDQ_FULL_BMSK 0x4
-#define HWIO_IPA_TX_COMMANDER_CMDQ_STATUS_CMDQ_FULL_SHFT 0x2
-#define HWIO_IPA_TX_COMMANDER_CMDQ_STATUS_CMDQ_EMPTY_BMSK 0x2
-#define HWIO_IPA_TX_COMMANDER_CMDQ_STATUS_CMDQ_EMPTY_SHFT 0x1
-#define HWIO_IPA_TX_COMMANDER_CMDQ_STATUS_STATUS_BMSK 0x1
-#define HWIO_IPA_TX_COMMANDER_CMDQ_STATUS_STATUS_SHFT 0x0
-#define HWIO_IPA_RX_HPS_CMDQ_CMD_ADDR (IPA_DEBUG_REG_BASE + 0x00000380)
-#define HWIO_IPA_RX_HPS_CMDQ_CMD_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-				       0x00000380)
-#define HWIO_IPA_RX_HPS_CMDQ_CMD_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-				       0x00000380)
-#define HWIO_IPA_RX_HPS_CMDQ_CMD_RMSK 0x3f
-#define HWIO_IPA_RX_HPS_CMDQ_CMD_ATTR 0x3
-#define HWIO_IPA_RX_HPS_CMDQ_CMD_IN in_dword_masked( \
-		HWIO_IPA_RX_HPS_CMDQ_CMD_ADDR, \
-		HWIO_IPA_RX_HPS_CMDQ_CMD_RMSK)
-#define HWIO_IPA_RX_HPS_CMDQ_CMD_INM(m) in_dword_masked( \
-		HWIO_IPA_RX_HPS_CMDQ_CMD_ADDR, \
-		m)
-#define HWIO_IPA_RX_HPS_CMDQ_CMD_OUT(v) out_dword( \
-		HWIO_IPA_RX_HPS_CMDQ_CMD_ADDR, \
-		v)
-#define HWIO_IPA_RX_HPS_CMDQ_CMD_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_RX_HPS_CMDQ_CMD_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_RX_HPS_CMDQ_CMD_IN)
-#define HWIO_IPA_RX_HPS_CMDQ_CMD_RD_REQ_BMSK 0x20
-#define HWIO_IPA_RX_HPS_CMDQ_CMD_RD_REQ_SHFT 0x5
-#define HWIO_IPA_RX_HPS_CMDQ_CMD_CMD_CLIENT_BMSK 0x1c
-#define HWIO_IPA_RX_HPS_CMDQ_CMD_CMD_CLIENT_SHFT 0x2
-#define HWIO_IPA_RX_HPS_CMDQ_CMD_POP_CMD_BMSK 0x2
-#define HWIO_IPA_RX_HPS_CMDQ_CMD_POP_CMD_SHFT 0x1
-#define HWIO_IPA_RX_HPS_CMDQ_CMD_WRITE_CMD_BMSK 0x1
-#define HWIO_IPA_RX_HPS_CMDQ_CMD_WRITE_CMD_SHFT 0x0
-#define HWIO_IPA_RX_HPS_CMDQ_RELEASE_WR_ADDR (IPA_DEBUG_REG_BASE + \
-					      0x00000384)
-#define HWIO_IPA_RX_HPS_CMDQ_RELEASE_WR_PHYS (IPA_DEBUG_REG_BASE_PHYS +	\
-					      0x00000384)
-#define HWIO_IPA_RX_HPS_CMDQ_RELEASE_WR_OFFS (IPA_DEBUG_REG_BASE_OFFS +	\
-					      0x00000384)
-#define HWIO_IPA_RX_HPS_CMDQ_RELEASE_RD_ADDR (IPA_DEBUG_REG_BASE + \
-					      0x00000388)
-#define HWIO_IPA_RX_HPS_CMDQ_RELEASE_RD_PHYS (IPA_DEBUG_REG_BASE_PHYS +	\
-					      0x00000388)
-#define HWIO_IPA_RX_HPS_CMDQ_RELEASE_RD_OFFS (IPA_DEBUG_REG_BASE_OFFS +	\
-					      0x00000388)
-#define HWIO_IPA_RX_HPS_CMDQ_CFG_WR_ADDR (IPA_DEBUG_REG_BASE + 0x0000038c)
-#define HWIO_IPA_RX_HPS_CMDQ_CFG_WR_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					  0x0000038c)
-#define HWIO_IPA_RX_HPS_CMDQ_CFG_WR_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					  0x0000038c)
-#define HWIO_IPA_RX_HPS_CMDQ_CFG_WR_RMSK 0x1f
-#define HWIO_IPA_RX_HPS_CMDQ_CFG_WR_ATTR 0x3
-#define HWIO_IPA_RX_HPS_CMDQ_CFG_WR_IN in_dword_masked(	\
-		HWIO_IPA_RX_HPS_CMDQ_CFG_WR_ADDR, \
-		HWIO_IPA_RX_HPS_CMDQ_CFG_WR_RMSK)
-#define HWIO_IPA_RX_HPS_CMDQ_CFG_WR_INM(m) in_dword_masked( \
-		HWIO_IPA_RX_HPS_CMDQ_CFG_WR_ADDR, \
-		m)
-#define HWIO_IPA_RX_HPS_CMDQ_CFG_WR_OUT(v) out_dword( \
-		HWIO_IPA_RX_HPS_CMDQ_CFG_WR_ADDR, \
-		v)
-#define HWIO_IPA_RX_HPS_CMDQ_CFG_WR_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_RX_HPS_CMDQ_CFG_WR_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_RX_HPS_CMDQ_CFG_WR_IN)
-#define HWIO_IPA_RX_HPS_CMDQ_CFG_WR_BLOCK_WR_BMSK 0x1f
-#define HWIO_IPA_RX_HPS_CMDQ_CFG_WR_BLOCK_WR_SHFT 0x0
-#define HWIO_IPA_RX_HPS_CMDQ_CFG_RD_ADDR (IPA_DEBUG_REG_BASE + 0x00000390)
-#define HWIO_IPA_RX_HPS_CMDQ_CFG_RD_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					  0x00000390)
-#define HWIO_IPA_RX_HPS_CMDQ_CFG_RD_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					  0x00000390)
-#define HWIO_IPA_RX_HPS_CMDQ_CFG_RD_RMSK 0x1f
-#define HWIO_IPA_RX_HPS_CMDQ_CFG_RD_ATTR 0x3
-#define HWIO_IPA_RX_HPS_CMDQ_CFG_RD_IN in_dword_masked(	\
-		HWIO_IPA_RX_HPS_CMDQ_CFG_RD_ADDR, \
-		HWIO_IPA_RX_HPS_CMDQ_CFG_RD_RMSK)
-#define HWIO_IPA_RX_HPS_CMDQ_CFG_RD_INM(m) in_dword_masked( \
-		HWIO_IPA_RX_HPS_CMDQ_CFG_RD_ADDR, \
-		m)
-#define HWIO_IPA_RX_HPS_CMDQ_CFG_RD_OUT(v) out_dword( \
-		HWIO_IPA_RX_HPS_CMDQ_CFG_RD_ADDR, \
-		v)
-#define HWIO_IPA_RX_HPS_CMDQ_CFG_RD_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_RX_HPS_CMDQ_CFG_RD_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_RX_HPS_CMDQ_CFG_RD_IN)
-#define HWIO_IPA_RX_HPS_CMDQ_CFG_RD_BLOCK_RD_BMSK 0x1f
-#define HWIO_IPA_RX_HPS_CMDQ_CFG_RD_BLOCK_RD_SHFT 0x0
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_WR_0_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x00000394)
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_WR_0_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x00000394)
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_WR_0_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x00000394)
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_WR_1_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x00000398)
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_WR_1_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x00000398)
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_WR_1_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x00000398)
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_WR_2_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x0000039c)
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_WR_2_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x0000039c)
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_WR_2_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x0000039c)
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_WR_3_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x000003a0)
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_WR_3_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x000003a0)
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_WR_3_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x000003a0)
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_0_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x000003a4)
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_0_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x000003a4)
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_0_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x000003a4)
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_0_RMSK 0xffffffff
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_0_ATTR 0x1
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_0_IN in_dword_masked( \
-		HWIO_IPA_RX_HPS_CMDQ_DATA_RD_0_ADDR, \
-		HWIO_IPA_RX_HPS_CMDQ_DATA_RD_0_RMSK)
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_0_INM(m) in_dword_masked( \
-		HWIO_IPA_RX_HPS_CMDQ_DATA_RD_0_ADDR, \
-		m)
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_0_CMDQ_DEST_LEN_F_BMSK 0xffff0000
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_0_CMDQ_DEST_LEN_F_SHFT 0x10
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_0_CMDQ_PACKET_LEN_F_BMSK 0xffff
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_0_CMDQ_PACKET_LEN_F_SHFT 0x0
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_1_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x000003a8)
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_1_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x000003a8)
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_1_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x000003a8)
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_1_RMSK 0xffffffff
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_1_ATTR 0x1
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_1_IN in_dword_masked( \
-		HWIO_IPA_RX_HPS_CMDQ_DATA_RD_1_ADDR, \
-		HWIO_IPA_RX_HPS_CMDQ_DATA_RD_1_RMSK)
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_1_INM(m) in_dword_masked( \
-		HWIO_IPA_RX_HPS_CMDQ_DATA_RD_1_ADDR, \
-		m)
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_1_CMDQ_METADATA_F_BMSK 0xff000000
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_1_CMDQ_METADATA_F_SHFT 0x18
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_1_CMDQ_OPCODE_F_BMSK 0xff0000
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_1_CMDQ_OPCODE_F_SHFT 0x10
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_1_CMDQ_FLAGS_F_BMSK 0xfc00
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_1_CMDQ_FLAGS_F_SHFT 0xa
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_1_CMDQ_ORDER_F_BMSK 0x300
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_1_CMDQ_ORDER_F_SHFT 0x8
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_1_CMDQ_SRC_PIPE_F_BMSK 0xff
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_1_CMDQ_SRC_PIPE_F_SHFT 0x0
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_2_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x000003ac)
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_2_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x000003ac)
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_2_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x000003ac)
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_2_RMSK 0xffffffff
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_2_ATTR 0x1
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_2_IN in_dword_masked( \
-		HWIO_IPA_RX_HPS_CMDQ_DATA_RD_2_ADDR, \
-		HWIO_IPA_RX_HPS_CMDQ_DATA_RD_2_RMSK)
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_2_INM(m) in_dword_masked( \
-		HWIO_IPA_RX_HPS_CMDQ_DATA_RD_2_ADDR, \
-		m)
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_2_CMDQ_ADDR_LSB_F_BMSK 0xffffffff
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_2_CMDQ_ADDR_LSB_F_SHFT 0x0
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_3_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x000003b0)
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_3_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x000003b0)
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_3_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x000003b0)
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_3_RMSK 0xffffffff
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_3_ATTR 0x1
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_3_IN in_dword_masked( \
-		HWIO_IPA_RX_HPS_CMDQ_DATA_RD_3_ADDR, \
-		HWIO_IPA_RX_HPS_CMDQ_DATA_RD_3_RMSK)
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_3_INM(m) in_dword_masked( \
-		HWIO_IPA_RX_HPS_CMDQ_DATA_RD_3_ADDR, \
-		m)
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_3_CMDQ_ADDR_MSB_F_BMSK 0xffffffff
-#define HWIO_IPA_RX_HPS_CMDQ_DATA_RD_3_CMDQ_ADDR_MSB_F_SHFT 0x0
-#define HWIO_IPA_RX_HPS_CMDQ_STATUS_ADDR (IPA_DEBUG_REG_BASE + 0x000003b4)
-#define HWIO_IPA_RX_HPS_CMDQ_STATUS_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					  0x000003b4)
-#define HWIO_IPA_RX_HPS_CMDQ_STATUS_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					  0x000003b4)
-#define HWIO_IPA_RX_HPS_CMDQ_STATUS_RMSK 0x1ff
-#define HWIO_IPA_RX_HPS_CMDQ_STATUS_ATTR 0x1
-#define HWIO_IPA_RX_HPS_CMDQ_STATUS_IN in_dword_masked(	\
-		HWIO_IPA_RX_HPS_CMDQ_STATUS_ADDR, \
-		HWIO_IPA_RX_HPS_CMDQ_STATUS_RMSK)
-#define HWIO_IPA_RX_HPS_CMDQ_STATUS_INM(m) in_dword_masked( \
-		HWIO_IPA_RX_HPS_CMDQ_STATUS_ADDR, \
-		m)
-#define HWIO_IPA_RX_HPS_CMDQ_STATUS_CMDQ_DEPTH_BMSK 0x1fc
-#define HWIO_IPA_RX_HPS_CMDQ_STATUS_CMDQ_DEPTH_SHFT 0x2
-#define HWIO_IPA_RX_HPS_CMDQ_STATUS_CMDQ_FULL_BMSK 0x2
-#define HWIO_IPA_RX_HPS_CMDQ_STATUS_CMDQ_FULL_SHFT 0x1
-#define HWIO_IPA_RX_HPS_CMDQ_STATUS_STATUS_BMSK 0x1
-#define HWIO_IPA_RX_HPS_CMDQ_STATUS_STATUS_SHFT 0x0
-#define HWIO_IPA_RX_HPS_CMDQ_STATUS_EMPTY_ADDR (IPA_DEBUG_REG_BASE + \
-						0x000003b8)
-#define HWIO_IPA_RX_HPS_CMDQ_STATUS_EMPTY_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						0x000003b8)
-#define HWIO_IPA_RX_HPS_CMDQ_STATUS_EMPTY_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						0x000003b8)
-#define HWIO_IPA_RX_HPS_CMDQ_STATUS_EMPTY_RMSK 0x1f
-#define HWIO_IPA_RX_HPS_CMDQ_STATUS_EMPTY_ATTR 0x1
-#define HWIO_IPA_RX_HPS_CMDQ_STATUS_EMPTY_IN in_dword_masked( \
-		HWIO_IPA_RX_HPS_CMDQ_STATUS_EMPTY_ADDR,	\
-		HWIO_IPA_RX_HPS_CMDQ_STATUS_EMPTY_RMSK)
-#define HWIO_IPA_RX_HPS_CMDQ_STATUS_EMPTY_INM(m) in_dword_masked( \
-		HWIO_IPA_RX_HPS_CMDQ_STATUS_EMPTY_ADDR,	\
-		m)
-#define HWIO_IPA_RX_HPS_CMDQ_STATUS_EMPTY_CMDQ_EMPTY_BMSK 0x1f
-#define HWIO_IPA_RX_HPS_CMDQ_STATUS_EMPTY_CMDQ_EMPTY_SHFT 0x0
-#define HWIO_IPA_RX_HPS_SNP_ADDR (IPA_DEBUG_REG_BASE + 0x000003bc)
-#define HWIO_IPA_RX_HPS_SNP_PHYS (IPA_DEBUG_REG_BASE_PHYS + 0x000003bc)
-#define HWIO_IPA_RX_HPS_SNP_OFFS (IPA_DEBUG_REG_BASE_OFFS + 0x000003bc)
-#define HWIO_IPA_RX_HPS_CMDQ_COUNT_ADDR (IPA_DEBUG_REG_BASE + 0x000003c0)
-#define HWIO_IPA_RX_HPS_CMDQ_COUNT_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					 0x000003c0)
-#define HWIO_IPA_RX_HPS_CMDQ_COUNT_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					 0x000003c0)
-#define HWIO_IPA_RX_HPS_CMDQ_COUNT_RMSK 0x7f
-#define HWIO_IPA_RX_HPS_CMDQ_COUNT_ATTR 0x1
-#define HWIO_IPA_RX_HPS_CMDQ_COUNT_IN in_dword_masked( \
-		HWIO_IPA_RX_HPS_CMDQ_COUNT_ADDR, \
-		HWIO_IPA_RX_HPS_CMDQ_COUNT_RMSK)
-#define HWIO_IPA_RX_HPS_CMDQ_COUNT_INM(m) in_dword_masked( \
-		HWIO_IPA_RX_HPS_CMDQ_COUNT_ADDR, \
-		m)
-#define HWIO_IPA_RX_HPS_CMDQ_COUNT_FIFO_COUNT_BMSK 0x7f
-#define HWIO_IPA_RX_HPS_CMDQ_COUNT_FIFO_COUNT_SHFT 0x0
-#define HWIO_IPA_RX_HPS_CLIENTS_MIN_DEPTH_0_ADDR (IPA_DEBUG_REG_BASE + \
-						  0x000003c4)
-#define HWIO_IPA_RX_HPS_CLIENTS_MIN_DEPTH_0_PHYS (IPA_DEBUG_REG_BASE_PHYS \
-						  + 0x000003c4)
-#define HWIO_IPA_RX_HPS_CLIENTS_MIN_DEPTH_0_OFFS (IPA_DEBUG_REG_BASE_OFFS \
-						  + 0x000003c4)
-#define HWIO_IPA_RX_HPS_CLIENTS_MIN_DEPTH_0_RMSK 0xff0f0f0f
-#define HWIO_IPA_RX_HPS_CLIENTS_MIN_DEPTH_0_ATTR 0x3
-#define HWIO_IPA_RX_HPS_CLIENTS_MIN_DEPTH_0_IN in_dword_masked(	\
-		HWIO_IPA_RX_HPS_CLIENTS_MIN_DEPTH_0_ADDR, \
-		HWIO_IPA_RX_HPS_CLIENTS_MIN_DEPTH_0_RMSK)
-#define HWIO_IPA_RX_HPS_CLIENTS_MIN_DEPTH_0_INM(m) in_dword_masked( \
-		HWIO_IPA_RX_HPS_CLIENTS_MIN_DEPTH_0_ADDR, \
-		m)
-#define HWIO_IPA_RX_HPS_CLIENTS_MIN_DEPTH_0_OUT(v) out_dword( \
-		HWIO_IPA_RX_HPS_CLIENTS_MIN_DEPTH_0_ADDR, \
-		v)
-#define HWIO_IPA_RX_HPS_CLIENTS_MIN_DEPTH_0_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_RX_HPS_CLIENTS_MIN_DEPTH_0_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_RX_HPS_CLIENTS_MIN_DEPTH_0_IN)
-#define HWIO_IPA_RX_HPS_CLIENTS_MIN_DEPTH_0_CLIENT_4_MIN_DEPTH_BMSK \
-	0xf0000000
-#define HWIO_IPA_RX_HPS_CLIENTS_MIN_DEPTH_0_CLIENT_4_MIN_DEPTH_SHFT 0x1c
-#define HWIO_IPA_RX_HPS_CLIENTS_MIN_DEPTH_0_CLIENT_3_MIN_DEPTH_BMSK \
-	0xf000000
-#define HWIO_IPA_RX_HPS_CLIENTS_MIN_DEPTH_0_CLIENT_3_MIN_DEPTH_SHFT 0x18
-#define HWIO_IPA_RX_HPS_CLIENTS_MIN_DEPTH_0_CLIENT_2_MIN_DEPTH_BMSK \
-	0xf0000
-#define HWIO_IPA_RX_HPS_CLIENTS_MIN_DEPTH_0_CLIENT_2_MIN_DEPTH_SHFT 0x10
-#define HWIO_IPA_RX_HPS_CLIENTS_MIN_DEPTH_0_CLIENT_1_MIN_DEPTH_BMSK 0xf00
-#define HWIO_IPA_RX_HPS_CLIENTS_MIN_DEPTH_0_CLIENT_1_MIN_DEPTH_SHFT 0x8
-#define HWIO_IPA_RX_HPS_CLIENTS_MIN_DEPTH_0_CLIENT_0_MIN_DEPTH_BMSK 0xf
-#define HWIO_IPA_RX_HPS_CLIENTS_MIN_DEPTH_0_CLIENT_0_MIN_DEPTH_SHFT 0x0
-#define HWIO_IPA_RX_HPS_CLIENTS_MAX_DEPTH_0_ADDR (IPA_DEBUG_REG_BASE + \
-						  0x000003cc)
-#define HWIO_IPA_RX_HPS_CLIENTS_MAX_DEPTH_0_PHYS (IPA_DEBUG_REG_BASE_PHYS \
-						  + 0x000003cc)
-#define HWIO_IPA_RX_HPS_CLIENTS_MAX_DEPTH_0_OFFS (IPA_DEBUG_REG_BASE_OFFS \
-						  + 0x000003cc)
-#define HWIO_IPA_RX_HPS_CLIENTS_MAX_DEPTH_0_RMSK 0xff0f0f0f
-#define HWIO_IPA_RX_HPS_CLIENTS_MAX_DEPTH_0_ATTR 0x3
-#define HWIO_IPA_RX_HPS_CLIENTS_MAX_DEPTH_0_IN in_dword_masked(	\
-		HWIO_IPA_RX_HPS_CLIENTS_MAX_DEPTH_0_ADDR, \
-		HWIO_IPA_RX_HPS_CLIENTS_MAX_DEPTH_0_RMSK)
-#define HWIO_IPA_RX_HPS_CLIENTS_MAX_DEPTH_0_INM(m) in_dword_masked( \
-		HWIO_IPA_RX_HPS_CLIENTS_MAX_DEPTH_0_ADDR, \
-		m)
-#define HWIO_IPA_RX_HPS_CLIENTS_MAX_DEPTH_0_OUT(v) out_dword( \
-		HWIO_IPA_RX_HPS_CLIENTS_MAX_DEPTH_0_ADDR, \
-		v)
-#define HWIO_IPA_RX_HPS_CLIENTS_MAX_DEPTH_0_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_RX_HPS_CLIENTS_MAX_DEPTH_0_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_RX_HPS_CLIENTS_MAX_DEPTH_0_IN)
-#define HWIO_IPA_RX_HPS_CLIENTS_MAX_DEPTH_0_CLIENT_4_MAX_DEPTH_BMSK \
-	0xf0000000
-#define HWIO_IPA_RX_HPS_CLIENTS_MAX_DEPTH_0_CLIENT_4_MAX_DEPTH_SHFT 0x1c
-#define HWIO_IPA_RX_HPS_CLIENTS_MAX_DEPTH_0_CLIENT_3_MAX_DEPTH_BMSK \
-	0xf000000
-#define HWIO_IPA_RX_HPS_CLIENTS_MAX_DEPTH_0_CLIENT_3_MAX_DEPTH_SHFT 0x18
-#define HWIO_IPA_RX_HPS_CLIENTS_MAX_DEPTH_0_CLIENT_2_MAX_DEPTH_BMSK \
-	0xf0000
-#define HWIO_IPA_RX_HPS_CLIENTS_MAX_DEPTH_0_CLIENT_2_MAX_DEPTH_SHFT 0x10
-#define HWIO_IPA_RX_HPS_CLIENTS_MAX_DEPTH_0_CLIENT_1_MAX_DEPTH_BMSK 0xf00
-#define HWIO_IPA_RX_HPS_CLIENTS_MAX_DEPTH_0_CLIENT_1_MAX_DEPTH_SHFT 0x8
-#define HWIO_IPA_RX_HPS_CLIENTS_MAX_DEPTH_0_CLIENT_0_MAX_DEPTH_BMSK 0xf
-#define HWIO_IPA_RX_HPS_CLIENTS_MAX_DEPTH_0_CLIENT_0_MAX_DEPTH_SHFT 0x0
-#define HWIO_IPA_HPS_DPS_CMDQ_CMD_ADDR (IPA_DEBUG_REG_BASE + 0x000003d4)
-#define HWIO_IPA_HPS_DPS_CMDQ_CMD_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					0x000003d4)
-#define HWIO_IPA_HPS_DPS_CMDQ_CMD_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					0x000003d4)
-#define HWIO_IPA_HPS_DPS_CMDQ_CMD_RMSK 0xff
-#define HWIO_IPA_HPS_DPS_CMDQ_CMD_ATTR 0x3
-#define HWIO_IPA_HPS_DPS_CMDQ_CMD_IN in_dword_masked( \
-		HWIO_IPA_HPS_DPS_CMDQ_CMD_ADDR,	\
-		HWIO_IPA_HPS_DPS_CMDQ_CMD_RMSK)
-#define HWIO_IPA_HPS_DPS_CMDQ_CMD_INM(m) in_dword_masked( \
-		HWIO_IPA_HPS_DPS_CMDQ_CMD_ADDR,	\
-		m)
-#define HWIO_IPA_HPS_DPS_CMDQ_CMD_OUT(v) out_dword( \
-		HWIO_IPA_HPS_DPS_CMDQ_CMD_ADDR,	\
-		v)
-#define HWIO_IPA_HPS_DPS_CMDQ_CMD_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_HPS_DPS_CMDQ_CMD_ADDR,	\
-		m, \
-		v, \
-		HWIO_IPA_HPS_DPS_CMDQ_CMD_IN)
-#define HWIO_IPA_HPS_DPS_CMDQ_CMD_RD_REQ_BMSK 0x80
-#define HWIO_IPA_HPS_DPS_CMDQ_CMD_RD_REQ_SHFT 0x7
-#define HWIO_IPA_HPS_DPS_CMDQ_CMD_CMD_CLIENT_BMSK 0x7c
-#define HWIO_IPA_HPS_DPS_CMDQ_CMD_CMD_CLIENT_SHFT 0x2
-#define HWIO_IPA_HPS_DPS_CMDQ_CMD_POP_CMD_BMSK 0x2
-#define HWIO_IPA_HPS_DPS_CMDQ_CMD_POP_CMD_SHFT 0x1
-#define HWIO_IPA_HPS_DPS_CMDQ_CMD_WRITE_CMD_BMSK 0x1
-#define HWIO_IPA_HPS_DPS_CMDQ_CMD_WRITE_CMD_SHFT 0x0
-#define HWIO_IPA_HPS_DPS_CMDQ_RELEASE_WR_ADDR (IPA_DEBUG_REG_BASE + \
-					       0x000003d8)
-#define HWIO_IPA_HPS_DPS_CMDQ_RELEASE_WR_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					       0x000003d8)
-#define HWIO_IPA_HPS_DPS_CMDQ_RELEASE_WR_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					       0x000003d8)
-#define HWIO_IPA_HPS_DPS_CMDQ_RELEASE_RD_ADDR (IPA_DEBUG_REG_BASE + \
-					       0x000003dc)
-#define HWIO_IPA_HPS_DPS_CMDQ_RELEASE_RD_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					       0x000003dc)
-#define HWIO_IPA_HPS_DPS_CMDQ_RELEASE_RD_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					       0x000003dc)
-#define HWIO_IPA_HPS_DPS_CMDQ_CFG_WR_ADDR (IPA_DEBUG_REG_BASE + 0x000003e0)
-#define HWIO_IPA_HPS_DPS_CMDQ_CFG_WR_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					   0x000003e0)
-#define HWIO_IPA_HPS_DPS_CMDQ_CFG_WR_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					   0x000003e0)
-#define HWIO_IPA_HPS_DPS_CMDQ_CFG_RD_ADDR (IPA_DEBUG_REG_BASE + 0x000003e4)
-#define HWIO_IPA_HPS_DPS_CMDQ_CFG_RD_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					   0x000003e4)
-#define HWIO_IPA_HPS_DPS_CMDQ_CFG_RD_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					   0x000003e4)
-#define HWIO_IPA_HPS_DPS_CMDQ_DATA_WR_0_ADDR (IPA_DEBUG_REG_BASE + \
-					      0x000003e8)
-#define HWIO_IPA_HPS_DPS_CMDQ_DATA_WR_0_PHYS (IPA_DEBUG_REG_BASE_PHYS +	\
-					      0x000003e8)
-#define HWIO_IPA_HPS_DPS_CMDQ_DATA_WR_0_OFFS (IPA_DEBUG_REG_BASE_OFFS +	\
-					      0x000003e8)
-#define HWIO_IPA_HPS_DPS_CMDQ_DATA_RD_0_ADDR (IPA_DEBUG_REG_BASE + \
-					      0x000003ec)
-#define HWIO_IPA_HPS_DPS_CMDQ_DATA_RD_0_PHYS (IPA_DEBUG_REG_BASE_PHYS +	\
-					      0x000003ec)
-#define HWIO_IPA_HPS_DPS_CMDQ_DATA_RD_0_OFFS (IPA_DEBUG_REG_BASE_OFFS +	\
-					      0x000003ec)
-#define HWIO_IPA_HPS_DPS_CMDQ_DATA_RD_0_RMSK 0xfffff
-#define HWIO_IPA_HPS_DPS_CMDQ_DATA_RD_0_ATTR 0x1
-#define HWIO_IPA_HPS_DPS_CMDQ_DATA_RD_0_IN in_dword_masked( \
-		HWIO_IPA_HPS_DPS_CMDQ_DATA_RD_0_ADDR, \
-		HWIO_IPA_HPS_DPS_CMDQ_DATA_RD_0_RMSK)
-#define HWIO_IPA_HPS_DPS_CMDQ_DATA_RD_0_INM(m) in_dword_masked(	\
-		HWIO_IPA_HPS_DPS_CMDQ_DATA_RD_0_ADDR, \
-		m)
-#define HWIO_IPA_HPS_DPS_CMDQ_DATA_RD_0_CMDQ_REP_F_BMSK 0x80000
-#define HWIO_IPA_HPS_DPS_CMDQ_DATA_RD_0_CMDQ_REP_F_SHFT 0x13
-#define HWIO_IPA_HPS_DPS_CMDQ_DATA_RD_0_CMDQ_OPCODE_F_BMSK 0x60000
-#define HWIO_IPA_HPS_DPS_CMDQ_DATA_RD_0_CMDQ_OPCODE_F_SHFT 0x11
-#define HWIO_IPA_HPS_DPS_CMDQ_DATA_RD_0_CMDQ_SRC_PIPE_F_BMSK 0x1f000
-#define HWIO_IPA_HPS_DPS_CMDQ_DATA_RD_0_CMDQ_SRC_PIPE_F_SHFT 0xc
-#define HWIO_IPA_HPS_DPS_CMDQ_DATA_RD_0_CMDQ_SRC_ID_F_BMSK 0xff0
-#define HWIO_IPA_HPS_DPS_CMDQ_DATA_RD_0_CMDQ_SRC_ID_F_SHFT 0x4
-#define HWIO_IPA_HPS_DPS_CMDQ_DATA_RD_0_CMDQ_CTX_ID_F_BMSK 0xf
-#define HWIO_IPA_HPS_DPS_CMDQ_DATA_RD_0_CMDQ_CTX_ID_F_SHFT 0x0
-#define HWIO_IPA_HPS_DPS_CMDQ_STATUS_ADDR (IPA_DEBUG_REG_BASE + 0x000003f0)
-#define HWIO_IPA_HPS_DPS_CMDQ_STATUS_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					   0x000003f0)
-#define HWIO_IPA_HPS_DPS_CMDQ_STATUS_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					   0x000003f0)
-#define HWIO_IPA_HPS_DPS_CMDQ_STATUS_RMSK 0xff
-#define HWIO_IPA_HPS_DPS_CMDQ_STATUS_ATTR 0x1
-#define HWIO_IPA_HPS_DPS_CMDQ_STATUS_IN in_dword_masked( \
-		HWIO_IPA_HPS_DPS_CMDQ_STATUS_ADDR, \
-		HWIO_IPA_HPS_DPS_CMDQ_STATUS_RMSK)
-#define HWIO_IPA_HPS_DPS_CMDQ_STATUS_INM(m) in_dword_masked( \
-		HWIO_IPA_HPS_DPS_CMDQ_STATUS_ADDR, \
-		m)
-#define HWIO_IPA_HPS_DPS_CMDQ_STATUS_CMDQ_DEPTH_BMSK 0xfc
-#define HWIO_IPA_HPS_DPS_CMDQ_STATUS_CMDQ_DEPTH_SHFT 0x2
-#define HWIO_IPA_HPS_DPS_CMDQ_STATUS_CMDQ_FULL_BMSK 0x2
-#define HWIO_IPA_HPS_DPS_CMDQ_STATUS_CMDQ_FULL_SHFT 0x1
-#define HWIO_IPA_HPS_DPS_CMDQ_STATUS_STATUS_BMSK 0x1
-#define HWIO_IPA_HPS_DPS_CMDQ_STATUS_STATUS_SHFT 0x0
-#define HWIO_IPA_HPS_DPS_CMDQ_STATUS_EMPTY_ADDR (IPA_DEBUG_REG_BASE + \
-						 0x000003f4)
-#define HWIO_IPA_HPS_DPS_CMDQ_STATUS_EMPTY_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						 0x000003f4)
-#define HWIO_IPA_HPS_DPS_CMDQ_STATUS_EMPTY_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						 0x000003f4)
-#define HWIO_IPA_HPS_DPS_CMDQ_STATUS_EMPTY_RMSK 0x7fffffff
-#define HWIO_IPA_HPS_DPS_CMDQ_STATUS_EMPTY_ATTR 0x1
-#define HWIO_IPA_HPS_DPS_CMDQ_STATUS_EMPTY_IN in_dword_masked( \
-		HWIO_IPA_HPS_DPS_CMDQ_STATUS_EMPTY_ADDR, \
-		HWIO_IPA_HPS_DPS_CMDQ_STATUS_EMPTY_RMSK)
-#define HWIO_IPA_HPS_DPS_CMDQ_STATUS_EMPTY_INM(m) in_dword_masked( \
-		HWIO_IPA_HPS_DPS_CMDQ_STATUS_EMPTY_ADDR, \
-		m)
-#define HWIO_IPA_HPS_DPS_CMDQ_STATUS_EMPTY_CMDQ_EMPTY_BMSK 0x7fffffff
-#define HWIO_IPA_HPS_DPS_CMDQ_STATUS_EMPTY_CMDQ_EMPTY_SHFT 0x0
-#define HWIO_IPA_HPS_DPS_SNP_ADDR (IPA_DEBUG_REG_BASE + 0x000003f8)
-#define HWIO_IPA_HPS_DPS_SNP_PHYS (IPA_DEBUG_REG_BASE_PHYS + 0x000003f8)
-#define HWIO_IPA_HPS_DPS_SNP_OFFS (IPA_DEBUG_REG_BASE_OFFS + 0x000003f8)
-#define HWIO_IPA_HPS_DPS_CMDQ_COUNT_ADDR (IPA_DEBUG_REG_BASE + 0x000003fc)
-#define HWIO_IPA_HPS_DPS_CMDQ_COUNT_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					  0x000003fc)
-#define HWIO_IPA_HPS_DPS_CMDQ_COUNT_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					  0x000003fc)
-#define HWIO_IPA_HPS_DPS_CMDQ_COUNT_RMSK 0x3f
-#define HWIO_IPA_HPS_DPS_CMDQ_COUNT_ATTR 0x1
-#define HWIO_IPA_HPS_DPS_CMDQ_COUNT_IN in_dword_masked(	\
-		HWIO_IPA_HPS_DPS_CMDQ_COUNT_ADDR, \
-		HWIO_IPA_HPS_DPS_CMDQ_COUNT_RMSK)
-#define HWIO_IPA_HPS_DPS_CMDQ_COUNT_INM(m) in_dword_masked( \
-		HWIO_IPA_HPS_DPS_CMDQ_COUNT_ADDR, \
-		m)
-#define HWIO_IPA_HPS_DPS_CMDQ_COUNT_FIFO_COUNT_BMSK 0x3f
-#define HWIO_IPA_HPS_DPS_CMDQ_COUNT_FIFO_COUNT_SHFT 0x0
-#define HWIO_IPA_DPS_TX_CMDQ_CMD_ADDR (IPA_DEBUG_REG_BASE + 0x00000400)
-#define HWIO_IPA_DPS_TX_CMDQ_CMD_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-				       0x00000400)
-#define HWIO_IPA_DPS_TX_CMDQ_CMD_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-				       0x00000400)
-#define HWIO_IPA_DPS_TX_CMDQ_CMD_RMSK 0xbf
-#define HWIO_IPA_DPS_TX_CMDQ_CMD_ATTR 0x3
-#define HWIO_IPA_DPS_TX_CMDQ_CMD_IN in_dword_masked( \
-		HWIO_IPA_DPS_TX_CMDQ_CMD_ADDR, \
-		HWIO_IPA_DPS_TX_CMDQ_CMD_RMSK)
-#define HWIO_IPA_DPS_TX_CMDQ_CMD_INM(m) in_dword_masked( \
-		HWIO_IPA_DPS_TX_CMDQ_CMD_ADDR, \
-		m)
-#define HWIO_IPA_DPS_TX_CMDQ_CMD_OUT(v) out_dword( \
-		HWIO_IPA_DPS_TX_CMDQ_CMD_ADDR, \
-		v)
-#define HWIO_IPA_DPS_TX_CMDQ_CMD_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_DPS_TX_CMDQ_CMD_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_DPS_TX_CMDQ_CMD_IN)
-#define HWIO_IPA_DPS_TX_CMDQ_CMD_RD_REQ_BMSK 0x80
-#define HWIO_IPA_DPS_TX_CMDQ_CMD_RD_REQ_SHFT 0x7
-#define HWIO_IPA_DPS_TX_CMDQ_CMD_CMD_CLIENT_BMSK 0x3c
-#define HWIO_IPA_DPS_TX_CMDQ_CMD_CMD_CLIENT_SHFT 0x2
-#define HWIO_IPA_DPS_TX_CMDQ_CMD_POP_CMD_BMSK 0x2
-#define HWIO_IPA_DPS_TX_CMDQ_CMD_POP_CMD_SHFT 0x1
-#define HWIO_IPA_DPS_TX_CMDQ_CMD_WRITE_CMD_BMSK 0x1
-#define HWIO_IPA_DPS_TX_CMDQ_CMD_WRITE_CMD_SHFT 0x0
-#define HWIO_IPA_DPS_TX_CMDQ_RELEASE_WR_ADDR (IPA_DEBUG_REG_BASE + \
-					      0x00000404)
-#define HWIO_IPA_DPS_TX_CMDQ_RELEASE_WR_PHYS (IPA_DEBUG_REG_BASE_PHYS +	\
-					      0x00000404)
-#define HWIO_IPA_DPS_TX_CMDQ_RELEASE_WR_OFFS (IPA_DEBUG_REG_BASE_OFFS +	\
-					      0x00000404)
-#define HWIO_IPA_DPS_TX_CMDQ_RELEASE_RD_ADDR (IPA_DEBUG_REG_BASE + \
-					      0x00000408)
-#define HWIO_IPA_DPS_TX_CMDQ_RELEASE_RD_PHYS (IPA_DEBUG_REG_BASE_PHYS +	\
-					      0x00000408)
-#define HWIO_IPA_DPS_TX_CMDQ_RELEASE_RD_OFFS (IPA_DEBUG_REG_BASE_OFFS +	\
-					      0x00000408)
-#define HWIO_IPA_DPS_TX_CMDQ_CFG_WR_ADDR (IPA_DEBUG_REG_BASE + 0x0000040c)
-#define HWIO_IPA_DPS_TX_CMDQ_CFG_WR_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					  0x0000040c)
-#define HWIO_IPA_DPS_TX_CMDQ_CFG_WR_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					  0x0000040c)
-#define HWIO_IPA_DPS_TX_CMDQ_CFG_RD_ADDR (IPA_DEBUG_REG_BASE + 0x00000410)
-#define HWIO_IPA_DPS_TX_CMDQ_CFG_RD_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					  0x00000410)
-#define HWIO_IPA_DPS_TX_CMDQ_CFG_RD_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					  0x00000410)
-#define HWIO_IPA_DPS_TX_CMDQ_DATA_WR_0_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x00000414)
-#define HWIO_IPA_DPS_TX_CMDQ_DATA_WR_0_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x00000414)
-#define HWIO_IPA_DPS_TX_CMDQ_DATA_WR_0_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x00000414)
-#define HWIO_IPA_DPS_TX_CMDQ_DATA_RD_0_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x00000418)
-#define HWIO_IPA_DPS_TX_CMDQ_DATA_RD_0_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x00000418)
-#define HWIO_IPA_DPS_TX_CMDQ_DATA_RD_0_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x00000418)
-#define HWIO_IPA_DPS_TX_CMDQ_DATA_RD_0_RMSK 0xfffff
-#define HWIO_IPA_DPS_TX_CMDQ_DATA_RD_0_ATTR 0x1
-#define HWIO_IPA_DPS_TX_CMDQ_DATA_RD_0_IN in_dword_masked( \
-		HWIO_IPA_DPS_TX_CMDQ_DATA_RD_0_ADDR, \
-		HWIO_IPA_DPS_TX_CMDQ_DATA_RD_0_RMSK)
-#define HWIO_IPA_DPS_TX_CMDQ_DATA_RD_0_INM(m) in_dword_masked( \
-		HWIO_IPA_DPS_TX_CMDQ_DATA_RD_0_ADDR, \
-		m)
-#define HWIO_IPA_DPS_TX_CMDQ_DATA_RD_0_CMDQ_REP_F_BMSK 0x80000
-#define HWIO_IPA_DPS_TX_CMDQ_DATA_RD_0_CMDQ_REP_F_SHFT 0x13
-#define HWIO_IPA_DPS_TX_CMDQ_DATA_RD_0_CMDQ_OPCODE_F_BMSK 0x60000
-#define HWIO_IPA_DPS_TX_CMDQ_DATA_RD_0_CMDQ_OPCODE_F_SHFT 0x11
-#define HWIO_IPA_DPS_TX_CMDQ_DATA_RD_0_CMDQ_SRC_PIPE_F_BMSK 0x1f000
-#define HWIO_IPA_DPS_TX_CMDQ_DATA_RD_0_CMDQ_SRC_PIPE_F_SHFT 0xc
-#define HWIO_IPA_DPS_TX_CMDQ_DATA_RD_0_CMDQ_SRC_ID_F_BMSK 0xff0
-#define HWIO_IPA_DPS_TX_CMDQ_DATA_RD_0_CMDQ_SRC_ID_F_SHFT 0x4
-#define HWIO_IPA_DPS_TX_CMDQ_DATA_RD_0_CMDQ_CTX_ID_F_BMSK 0xf
-#define HWIO_IPA_DPS_TX_CMDQ_DATA_RD_0_CMDQ_CTX_ID_F_SHFT 0x0
-#define HWIO_IPA_DPS_TX_CMDQ_STATUS_ADDR (IPA_DEBUG_REG_BASE + 0x0000041c)
-#define HWIO_IPA_DPS_TX_CMDQ_STATUS_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					  0x0000041c)
-#define HWIO_IPA_DPS_TX_CMDQ_STATUS_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					  0x0000041c)
-#define HWIO_IPA_DPS_TX_CMDQ_STATUS_RMSK 0x1ff
-#define HWIO_IPA_DPS_TX_CMDQ_STATUS_ATTR 0x1
-#define HWIO_IPA_DPS_TX_CMDQ_STATUS_IN in_dword_masked(	\
-		HWIO_IPA_DPS_TX_CMDQ_STATUS_ADDR, \
-		HWIO_IPA_DPS_TX_CMDQ_STATUS_RMSK)
-#define HWIO_IPA_DPS_TX_CMDQ_STATUS_INM(m) in_dword_masked( \
-		HWIO_IPA_DPS_TX_CMDQ_STATUS_ADDR, \
-		m)
-#define HWIO_IPA_DPS_TX_CMDQ_STATUS_CMDQ_DEPTH_BMSK 0x1fc
-#define HWIO_IPA_DPS_TX_CMDQ_STATUS_CMDQ_DEPTH_SHFT 0x2
-#define HWIO_IPA_DPS_TX_CMDQ_STATUS_CMDQ_FULL_BMSK 0x2
-#define HWIO_IPA_DPS_TX_CMDQ_STATUS_CMDQ_FULL_SHFT 0x1
-#define HWIO_IPA_DPS_TX_CMDQ_STATUS_STATUS_BMSK 0x1
-#define HWIO_IPA_DPS_TX_CMDQ_STATUS_STATUS_SHFT 0x0
-#define HWIO_IPA_DPS_TX_CMDQ_STATUS_EMPTY_ADDR (IPA_DEBUG_REG_BASE + \
-						0x00000420)
-#define HWIO_IPA_DPS_TX_CMDQ_STATUS_EMPTY_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						0x00000420)
-#define HWIO_IPA_DPS_TX_CMDQ_STATUS_EMPTY_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						0x00000420)
-#define HWIO_IPA_DPS_TX_CMDQ_STATUS_EMPTY_RMSK 0x3ff
-#define HWIO_IPA_DPS_TX_CMDQ_STATUS_EMPTY_ATTR 0x1
-#define HWIO_IPA_DPS_TX_CMDQ_STATUS_EMPTY_IN in_dword_masked( \
-		HWIO_IPA_DPS_TX_CMDQ_STATUS_EMPTY_ADDR,	\
-		HWIO_IPA_DPS_TX_CMDQ_STATUS_EMPTY_RMSK)
-#define HWIO_IPA_DPS_TX_CMDQ_STATUS_EMPTY_INM(m) in_dword_masked( \
-		HWIO_IPA_DPS_TX_CMDQ_STATUS_EMPTY_ADDR,	\
-		m)
-#define HWIO_IPA_DPS_TX_CMDQ_STATUS_EMPTY_CMDQ_EMPTY_BMSK 0x3ff
-#define HWIO_IPA_DPS_TX_CMDQ_STATUS_EMPTY_CMDQ_EMPTY_SHFT 0x0
-#define HWIO_IPA_DPS_TX_SNP_ADDR (IPA_DEBUG_REG_BASE + 0x00000424)
-#define HWIO_IPA_DPS_TX_SNP_PHYS (IPA_DEBUG_REG_BASE_PHYS + 0x00000424)
-#define HWIO_IPA_DPS_TX_SNP_OFFS (IPA_DEBUG_REG_BASE_OFFS + 0x00000424)
-#define HWIO_IPA_DPS_TX_CMDQ_COUNT_ADDR (IPA_DEBUG_REG_BASE + 0x00000428)
-#define HWIO_IPA_DPS_TX_CMDQ_COUNT_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					 0x00000428)
-#define HWIO_IPA_DPS_TX_CMDQ_COUNT_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					 0x00000428)
-#define HWIO_IPA_DPS_TX_CMDQ_COUNT_RMSK 0x7f
-#define HWIO_IPA_DPS_TX_CMDQ_COUNT_ATTR 0x1
-#define HWIO_IPA_DPS_TX_CMDQ_COUNT_IN in_dword_masked( \
-		HWIO_IPA_DPS_TX_CMDQ_COUNT_ADDR, \
-		HWIO_IPA_DPS_TX_CMDQ_COUNT_RMSK)
-#define HWIO_IPA_DPS_TX_CMDQ_COUNT_INM(m) in_dword_masked( \
-		HWIO_IPA_DPS_TX_CMDQ_COUNT_ADDR, \
-		m)
-#define HWIO_IPA_DPS_TX_CMDQ_COUNT_FIFO_COUNT_BMSK 0x7f
-#define HWIO_IPA_DPS_TX_CMDQ_COUNT_FIFO_COUNT_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_EN_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x0000042c)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_EN_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x0000042c)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_EN_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x0000042c)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_EN_RMSK 0x7
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_EN_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_EN_IN in_dword_masked( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_EN_ADDR, \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_EN_RMSK)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_EN_INM(m) in_dword_masked( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_EN_ADDR, \
-		m)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_EN_OUT(v) out_dword( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_EN_ADDR, \
-		v)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_EN_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_EN_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_EN_IN)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_EN_BITMAP_BMSK 0x7
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_EN_BITMAP_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_WR_N_RD_SEL_ADDR (IPA_DEBUG_REG_BASE + \
-						      0x00000430)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_WR_N_RD_SEL_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000430)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_WR_N_RD_SEL_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000430)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_WR_N_RD_SEL_RMSK 0x7
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_WR_N_RD_SEL_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_WR_N_RD_SEL_IN in_dword_masked( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_WR_N_RD_SEL_ADDR, \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_WR_N_RD_SEL_RMSK)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_WR_N_RD_SEL_INM(m) in_dword_masked(	\
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_WR_N_RD_SEL_ADDR, \
-		m)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_WR_N_RD_SEL_OUT(v) out_dword( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_WR_N_RD_SEL_ADDR, \
-		v)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_WR_N_RD_SEL_OUTM(m,	\
-						     v)	\
-	out_dword_masked_ns(HWIO_IPA_LOG_BUF_HW_SNIF_EL_WR_N_RD_SEL_ADDR, \
-			    m, \
-			    v, \
-			    HWIO_IPA_LOG_BUF_HW_SNIF_EL_WR_N_RD_SEL_IN)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_WR_N_RD_SEL_BITMAP_BMSK 0x7
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_WR_N_RD_SEL_BITMAP_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_CLI_MUX_ADDR (IPA_DEBUG_REG_BASE + \
-						  0x00000434)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_CLI_MUX_PHYS (IPA_DEBUG_REG_BASE_PHYS \
-						  + 0x00000434)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_CLI_MUX_OFFS (IPA_DEBUG_REG_BASE_OFFS \
-						  + 0x00000434)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_CLI_MUX_RMSK 0xfff
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_CLI_MUX_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_CLI_MUX_IN in_dword_masked(	\
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_CLI_MUX_ADDR, \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_CLI_MUX_RMSK)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_CLI_MUX_INM(m) in_dword_masked( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_CLI_MUX_ADDR, \
-		m)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_CLI_MUX_OUT(v) out_dword( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_CLI_MUX_ADDR, \
-		v)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_CLI_MUX_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_CLI_MUX_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_CLI_MUX_IN)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_CLI_MUX_ALL_CLI_MUX_CONCAT_BMSK 0xfff
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_CLI_MUX_ALL_CLI_MUX_CONCAT_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_0_CLI_n_ADDR(n) ( \
-		IPA_DEBUG_REG_BASE + 0x00000438 + 0x10 * (n))
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_0_CLI_n_PHYS(n) ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000438 + 0x10 * (n))
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_0_CLI_n_OFFS(n) ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000438 + 0x10 * (n))
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_0_CLI_n_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_0_CLI_n_MAXn 2
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_0_CLI_n_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_0_CLI_n_INI(n) \
-	in_dword_masked(HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_0_CLI_n_ADDR( \
-				n), \
-			HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_0_CLI_n_RMSK)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_0_CLI_n_INMI(n, \
-							  mask)	\
-	in_dword_masked( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_0_CLI_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_0_CLI_n_OUTI(n, \
-							  val) out_dword( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_0_CLI_n_ADDR( \
-			n), \
-		val)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_0_CLI_n_OUTMI(n, mask, \
-							   val)	\
-	out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_0_CLI_n_ADDR(n), \
-		mask, \
-		val, \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_0_CLI_n_INI(n))
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_0_CLI_n_VALUE_BMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_0_CLI_n_VALUE_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_1_CLI_n_ADDR(n) ( \
-		IPA_DEBUG_REG_BASE + 0x0000043c + 0x10 * (n))
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_1_CLI_n_PHYS(n) ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x0000043c + 0x10 * (n))
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_1_CLI_n_OFFS(n) ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x0000043c + 0x10 * (n))
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_1_CLI_n_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_1_CLI_n_MAXn 2
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_1_CLI_n_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_1_CLI_n_INI(n) \
-	in_dword_masked(HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_1_CLI_n_ADDR( \
-				n), \
-			HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_1_CLI_n_RMSK)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_1_CLI_n_INMI(n, \
-							  mask)	\
-	in_dword_masked( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_1_CLI_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_1_CLI_n_OUTI(n, \
-							  val) out_dword( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_1_CLI_n_ADDR( \
-			n), \
-		val)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_1_CLI_n_OUTMI(n, mask, \
-							   val)	\
-	out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_1_CLI_n_ADDR(n), \
-		mask, \
-		val, \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_1_CLI_n_INI(n))
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_1_CLI_n_VALUE_BMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_1_CLI_n_VALUE_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_2_CLI_n_ADDR(n) ( \
-		IPA_DEBUG_REG_BASE + 0x00000440 + 0x10 * (n))
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_2_CLI_n_PHYS(n) ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000440 + 0x10 * (n))
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_2_CLI_n_OFFS(n) ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000440 + 0x10 * (n))
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_2_CLI_n_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_2_CLI_n_MAXn 2
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_2_CLI_n_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_2_CLI_n_INI(n) \
-	in_dword_masked(HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_2_CLI_n_ADDR( \
-				n), \
-			HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_2_CLI_n_RMSK)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_2_CLI_n_INMI(n, \
-							  mask)	\
-	in_dword_masked( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_2_CLI_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_2_CLI_n_OUTI(n, \
-							  val) out_dword( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_2_CLI_n_ADDR( \
-			n), \
-		val)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_2_CLI_n_OUTMI(n, mask, \
-							   val)	\
-	out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_2_CLI_n_ADDR(n), \
-		mask, \
-		val, \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_2_CLI_n_INI(n))
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_2_CLI_n_VALUE_BMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_2_CLI_n_VALUE_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_3_CLI_n_ADDR(n) ( \
-		IPA_DEBUG_REG_BASE + 0x00000444 + 0x10 * (n))
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_3_CLI_n_PHYS(n) ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000444 + 0x10 * (n))
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_3_CLI_n_OFFS(n) ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000444 + 0x10 * (n))
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_3_CLI_n_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_3_CLI_n_MAXn 2
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_3_CLI_n_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_3_CLI_n_INI(n) \
-	in_dword_masked(HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_3_CLI_n_ADDR( \
-				n), \
-			HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_3_CLI_n_RMSK)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_3_CLI_n_INMI(n, \
-							  mask)	\
-	in_dword_masked( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_3_CLI_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_3_CLI_n_OUTI(n, \
-							  val) out_dword( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_3_CLI_n_ADDR( \
-			n), \
-		val)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_3_CLI_n_OUTMI(n, mask, \
-							   val)	\
-	out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_3_CLI_n_ADDR(n), \
-		mask, \
-		val, \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_3_CLI_n_INI(n))
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_3_CLI_n_VALUE_BMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_COMP_VAL_3_CLI_n_VALUE_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_0_CLI_n_ADDR(n) ( \
-		IPA_DEBUG_REG_BASE + 0x00000468 + 0x10 * (n))
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_0_CLI_n_PHYS(n) ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000468 + 0x10 * (n))
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_0_CLI_n_OFFS(n) ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000468 + 0x10 * (n))
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_0_CLI_n_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_0_CLI_n_MAXn 2
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_0_CLI_n_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_0_CLI_n_INI(n) \
-	in_dword_masked(HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_0_CLI_n_ADDR( \
-				n), \
-			HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_0_CLI_n_RMSK)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_0_CLI_n_INMI(n, \
-							  mask)	\
-	in_dword_masked( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_0_CLI_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_0_CLI_n_OUTI(n, \
-							  val) out_dword( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_0_CLI_n_ADDR( \
-			n), \
-		val)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_0_CLI_n_OUTMI(n, mask, \
-							   val)	\
-	out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_0_CLI_n_ADDR(n), \
-		mask, \
-		val, \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_0_CLI_n_INI(n))
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_0_CLI_n_VALUE_BMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_0_CLI_n_VALUE_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_1_CLI_n_ADDR(n) ( \
-		IPA_DEBUG_REG_BASE + 0x0000046c + 0x10 * (n))
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_1_CLI_n_PHYS(n) ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x0000046c + 0x10 * (n))
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_1_CLI_n_OFFS(n) ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x0000046c + 0x10 * (n))
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_1_CLI_n_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_1_CLI_n_MAXn 2
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_1_CLI_n_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_1_CLI_n_INI(n) \
-	in_dword_masked(HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_1_CLI_n_ADDR( \
-				n), \
-			HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_1_CLI_n_RMSK)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_1_CLI_n_INMI(n, \
-							  mask)	\
-	in_dword_masked( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_1_CLI_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_1_CLI_n_OUTI(n, \
-							  val) out_dword( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_1_CLI_n_ADDR( \
-			n), \
-		val)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_1_CLI_n_OUTMI(n, mask, \
-							   val)	\
-	out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_1_CLI_n_ADDR(n), \
-		mask, \
-		val, \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_1_CLI_n_INI(n))
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_1_CLI_n_VALUE_BMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_1_CLI_n_VALUE_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_2_CLI_n_ADDR(n) ( \
-		IPA_DEBUG_REG_BASE + 0x00000470 + 0x10 * (n))
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_2_CLI_n_PHYS(n) ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000470 + 0x10 * (n))
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_2_CLI_n_OFFS(n) ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000470 + 0x10 * (n))
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_2_CLI_n_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_2_CLI_n_MAXn 2
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_2_CLI_n_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_2_CLI_n_INI(n) \
-	in_dword_masked(HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_2_CLI_n_ADDR( \
-				n), \
-			HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_2_CLI_n_RMSK)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_2_CLI_n_INMI(n, \
-							  mask)	\
-	in_dword_masked( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_2_CLI_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_2_CLI_n_OUTI(n, \
-							  val) out_dword( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_2_CLI_n_ADDR( \
-			n), \
-		val)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_2_CLI_n_OUTMI(n, mask, \
-							   val)	\
-	out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_2_CLI_n_ADDR(n), \
-		mask, \
-		val, \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_2_CLI_n_INI(n))
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_2_CLI_n_VALUE_BMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_2_CLI_n_VALUE_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_3_CLI_n_ADDR(n) ( \
-		IPA_DEBUG_REG_BASE + 0x00000474 + 0x10 * (n))
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_3_CLI_n_PHYS(n) ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000474 + 0x10 * (n))
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_3_CLI_n_OFFS(n) ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000474 + 0x10 * (n))
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_3_CLI_n_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_3_CLI_n_MAXn 2
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_3_CLI_n_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_3_CLI_n_INI(n) \
-	in_dword_masked(HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_3_CLI_n_ADDR( \
-				n), \
-			HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_3_CLI_n_RMSK)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_3_CLI_n_INMI(n, \
-							  mask)	\
-	in_dword_masked( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_3_CLI_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_3_CLI_n_OUTI(n, \
-							  val) out_dword( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_3_CLI_n_ADDR( \
-			n), \
-		val)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_3_CLI_n_OUTMI(n, mask, \
-							   val)	\
-	out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_3_CLI_n_ADDR(n), \
-		mask, \
-		val, \
-		HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_3_CLI_n_INI(n))
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_3_CLI_n_VALUE_BMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_HW_SNIF_EL_MASK_VAL_3_CLI_n_VALUE_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_HW_SNIF_LEGACY_RX_ADDR (IPA_DEBUG_REG_BASE + \
-						 0x00000498)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_LEGACY_RX_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						 0x00000498)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_LEGACY_RX_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						 0x00000498)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_LEGACY_RX_RMSK 0x7
-#define HWIO_IPA_LOG_BUF_HW_SNIF_LEGACY_RX_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_HW_SNIF_LEGACY_RX_IN in_dword_masked( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_LEGACY_RX_ADDR, \
-		HWIO_IPA_LOG_BUF_HW_SNIF_LEGACY_RX_RMSK)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_LEGACY_RX_INM(m) in_dword_masked( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_LEGACY_RX_ADDR, \
-		m)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_LEGACY_RX_OUT(v) out_dword( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_LEGACY_RX_ADDR, \
-		v)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_LEGACY_RX_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_HW_SNIF_LEGACY_RX_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_LOG_BUF_HW_SNIF_LEGACY_RX_IN)
-#define HWIO_IPA_LOG_BUF_HW_SNIF_LEGACY_RX_SRC_GROUP_SEL_BMSK 0x7
-#define HWIO_IPA_LOG_BUF_HW_SNIF_LEGACY_RX_SRC_GROUP_SEL_SHFT 0x0
-#define HWIO_IPA_ACKMNGR_CMDQ_CMD_ADDR (IPA_DEBUG_REG_BASE + 0x000004a0)
-#define HWIO_IPA_ACKMNGR_CMDQ_CMD_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					0x000004a0)
-#define HWIO_IPA_ACKMNGR_CMDQ_CMD_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					0x000004a0)
-#define HWIO_IPA_ACKMNGR_CMDQ_CMD_RMSK 0xff
-#define HWIO_IPA_ACKMNGR_CMDQ_CMD_ATTR 0x3
-#define HWIO_IPA_ACKMNGR_CMDQ_CMD_IN in_dword_masked( \
-		HWIO_IPA_ACKMNGR_CMDQ_CMD_ADDR,	\
-		HWIO_IPA_ACKMNGR_CMDQ_CMD_RMSK)
-#define HWIO_IPA_ACKMNGR_CMDQ_CMD_INM(m) in_dword_masked( \
-		HWIO_IPA_ACKMNGR_CMDQ_CMD_ADDR,	\
-		m)
-#define HWIO_IPA_ACKMNGR_CMDQ_CMD_OUT(v) out_dword( \
-		HWIO_IPA_ACKMNGR_CMDQ_CMD_ADDR,	\
-		v)
-#define HWIO_IPA_ACKMNGR_CMDQ_CMD_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_ACKMNGR_CMDQ_CMD_ADDR,	\
-		m, \
-		v, \
-		HWIO_IPA_ACKMNGR_CMDQ_CMD_IN)
-#define HWIO_IPA_ACKMNGR_CMDQ_CMD_RD_REQ_BMSK 0x80
-#define HWIO_IPA_ACKMNGR_CMDQ_CMD_RD_REQ_SHFT 0x7
-#define HWIO_IPA_ACKMNGR_CMDQ_CMD_CMD_CLIENT_BMSK 0x7c
-#define HWIO_IPA_ACKMNGR_CMDQ_CMD_CMD_CLIENT_SHFT 0x2
-#define HWIO_IPA_ACKMNGR_CMDQ_CMD_POP_CMD_BMSK 0x2
-#define HWIO_IPA_ACKMNGR_CMDQ_CMD_POP_CMD_SHFT 0x1
-#define HWIO_IPA_ACKMNGR_CMDQ_CMD_WRITE_CMD_BMSK 0x1
-#define HWIO_IPA_ACKMNGR_CMDQ_CMD_WRITE_CMD_SHFT 0x0
-#define HWIO_IPA_ACKMNGR_CMDQ_RELEASE_WR_ADDR (IPA_DEBUG_REG_BASE + \
-					       0x000004a4)
-#define HWIO_IPA_ACKMNGR_CMDQ_RELEASE_WR_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					       0x000004a4)
-#define HWIO_IPA_ACKMNGR_CMDQ_RELEASE_WR_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					       0x000004a4)
-#define HWIO_IPA_ACKMNGR_CMDQ_RELEASE_RD_ADDR (IPA_DEBUG_REG_BASE + \
-					       0x000004a8)
-#define HWIO_IPA_ACKMNGR_CMDQ_RELEASE_RD_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					       0x000004a8)
-#define HWIO_IPA_ACKMNGR_CMDQ_RELEASE_RD_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					       0x000004a8)
-#define HWIO_IPA_ACKMNGR_CMDQ_CFG_WR_ADDR (IPA_DEBUG_REG_BASE + 0x000004ac)
-#define HWIO_IPA_ACKMNGR_CMDQ_CFG_WR_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					   0x000004ac)
-#define HWIO_IPA_ACKMNGR_CMDQ_CFG_WR_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					   0x000004ac)
-#define HWIO_IPA_ACKMNGR_CMDQ_CFG_RD_ADDR (IPA_DEBUG_REG_BASE + 0x000004b0)
-#define HWIO_IPA_ACKMNGR_CMDQ_CFG_RD_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					   0x000004b0)
-#define HWIO_IPA_ACKMNGR_CMDQ_CFG_RD_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					   0x000004b0)
-#define HWIO_IPA_ACKMNGR_CMDQ_DATA_WR_ADDR (IPA_DEBUG_REG_BASE + \
-					    0x000004b4)
-#define HWIO_IPA_ACKMNGR_CMDQ_DATA_WR_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					    0x000004b4)
-#define HWIO_IPA_ACKMNGR_CMDQ_DATA_WR_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					    0x000004b4)
-#define HWIO_IPA_ACKMNGR_CMDQ_DATA_RD_ADDR (IPA_DEBUG_REG_BASE + \
-					    0x000004b8)
-#define HWIO_IPA_ACKMNGR_CMDQ_DATA_RD_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					    0x000004b8)
-#define HWIO_IPA_ACKMNGR_CMDQ_DATA_RD_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					    0x000004b8)
-#define HWIO_IPA_ACKMNGR_CMDQ_DATA_RD_RMSK 0x7ffffff
-#define HWIO_IPA_ACKMNGR_CMDQ_DATA_RD_ATTR 0x1
-#define HWIO_IPA_ACKMNGR_CMDQ_DATA_RD_IN in_dword_masked( \
-		HWIO_IPA_ACKMNGR_CMDQ_DATA_RD_ADDR, \
-		HWIO_IPA_ACKMNGR_CMDQ_DATA_RD_RMSK)
-#define HWIO_IPA_ACKMNGR_CMDQ_DATA_RD_INM(m) in_dword_masked( \
-		HWIO_IPA_ACKMNGR_CMDQ_DATA_RD_ADDR, \
-		m)
-#define HWIO_IPA_ACKMNGR_CMDQ_DATA_RD_CMDQ_SRC_ID_VALID_BMSK 0x4000000
-#define HWIO_IPA_ACKMNGR_CMDQ_DATA_RD_CMDQ_SRC_ID_VALID_SHFT 0x1a
-#define HWIO_IPA_ACKMNGR_CMDQ_DATA_RD_CMDQ_SENT_BMSK 0x2000000
-#define HWIO_IPA_ACKMNGR_CMDQ_DATA_RD_CMDQ_SENT_SHFT 0x19
-#define HWIO_IPA_ACKMNGR_CMDQ_DATA_RD_CMDQ_ORIGIN_BMSK 0x1000000
-#define HWIO_IPA_ACKMNGR_CMDQ_DATA_RD_CMDQ_ORIGIN_SHFT 0x18
-#define HWIO_IPA_ACKMNGR_CMDQ_DATA_RD_CMDQ_LENGTH_BMSK 0xffff00
-#define HWIO_IPA_ACKMNGR_CMDQ_DATA_RD_CMDQ_LENGTH_SHFT 0x8
-#define HWIO_IPA_ACKMNGR_CMDQ_DATA_RD_CMDQ_SRC_ID_BMSK 0xff
-#define HWIO_IPA_ACKMNGR_CMDQ_DATA_RD_CMDQ_SRC_ID_SHFT 0x0
-#define HWIO_IPA_ACKMNGR_CMDQ_STATUS_ADDR (IPA_DEBUG_REG_BASE + 0x000004bc)
-#define HWIO_IPA_ACKMNGR_CMDQ_STATUS_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					   0x000004bc)
-#define HWIO_IPA_ACKMNGR_CMDQ_STATUS_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					   0x000004bc)
-#define HWIO_IPA_ACKMNGR_CMDQ_STATUS_RMSK 0x1ff
-#define HWIO_IPA_ACKMNGR_CMDQ_STATUS_ATTR 0x1
-#define HWIO_IPA_ACKMNGR_CMDQ_STATUS_IN in_dword_masked( \
-		HWIO_IPA_ACKMNGR_CMDQ_STATUS_ADDR, \
-		HWIO_IPA_ACKMNGR_CMDQ_STATUS_RMSK)
-#define HWIO_IPA_ACKMNGR_CMDQ_STATUS_INM(m) in_dword_masked( \
-		HWIO_IPA_ACKMNGR_CMDQ_STATUS_ADDR, \
-		m)
-#define HWIO_IPA_ACKMNGR_CMDQ_STATUS_CMDQ_DEPTH_BMSK 0x1fc
-#define HWIO_IPA_ACKMNGR_CMDQ_STATUS_CMDQ_DEPTH_SHFT 0x2
-#define HWIO_IPA_ACKMNGR_CMDQ_STATUS_CMDQ_FULL_BMSK 0x2
-#define HWIO_IPA_ACKMNGR_CMDQ_STATUS_CMDQ_FULL_SHFT 0x1
-#define HWIO_IPA_ACKMNGR_CMDQ_STATUS_STATUS_BMSK 0x1
-#define HWIO_IPA_ACKMNGR_CMDQ_STATUS_STATUS_SHFT 0x0
-#define HWIO_IPA_ACKMNGR_CMDQ_STATUS_EMPTY_ADDR (IPA_DEBUG_REG_BASE + \
-						 0x000004c0)
-#define HWIO_IPA_ACKMNGR_CMDQ_STATUS_EMPTY_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						 0x000004c0)
-#define HWIO_IPA_ACKMNGR_CMDQ_STATUS_EMPTY_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						 0x000004c0)
-#define HWIO_IPA_ACKMNGR_CMDQ_STATUS_EMPTY_RMSK 0x1fff
-#define HWIO_IPA_ACKMNGR_CMDQ_STATUS_EMPTY_ATTR 0x1
-#define HWIO_IPA_ACKMNGR_CMDQ_STATUS_EMPTY_IN in_dword_masked( \
-		HWIO_IPA_ACKMNGR_CMDQ_STATUS_EMPTY_ADDR, \
-		HWIO_IPA_ACKMNGR_CMDQ_STATUS_EMPTY_RMSK)
-#define HWIO_IPA_ACKMNGR_CMDQ_STATUS_EMPTY_INM(m) in_dword_masked( \
-		HWIO_IPA_ACKMNGR_CMDQ_STATUS_EMPTY_ADDR, \
-		m)
-#define HWIO_IPA_ACKMNGR_CMDQ_STATUS_EMPTY_CMDQ_EMPTY_BMSK 0x1fff
-#define HWIO_IPA_ACKMNGR_CMDQ_STATUS_EMPTY_CMDQ_EMPTY_SHFT 0x0
-#define HWIO_IPA_ACKMNGR_CMDQ_COUNT_ADDR (IPA_DEBUG_REG_BASE + 0x000004c4)
-#define HWIO_IPA_ACKMNGR_CMDQ_COUNT_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					  0x000004c4)
-#define HWIO_IPA_ACKMNGR_CMDQ_COUNT_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					  0x000004c4)
-#define HWIO_IPA_ACKMNGR_CMDQ_COUNT_RMSK 0x7f
-#define HWIO_IPA_ACKMNGR_CMDQ_COUNT_ATTR 0x1
-#define HWIO_IPA_ACKMNGR_CMDQ_COUNT_IN in_dword_masked(	\
-		HWIO_IPA_ACKMNGR_CMDQ_COUNT_ADDR, \
-		HWIO_IPA_ACKMNGR_CMDQ_COUNT_RMSK)
-#define HWIO_IPA_ACKMNGR_CMDQ_COUNT_INM(m) in_dword_masked( \
-		HWIO_IPA_ACKMNGR_CMDQ_COUNT_ADDR, \
-		m)
-#define HWIO_IPA_ACKMNGR_CMDQ_COUNT_FIFO_COUNT_BMSK 0x7f
-#define HWIO_IPA_ACKMNGR_CMDQ_COUNT_FIFO_COUNT_SHFT 0x0
-#define HWIO_IPA_GSI_FIFO_STATUS_CTRL_ADDR (IPA_DEBUG_REG_BASE + \
-					    0x000004c8)
-#define HWIO_IPA_GSI_FIFO_STATUS_CTRL_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					    0x000004c8)
-#define HWIO_IPA_GSI_FIFO_STATUS_CTRL_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					    0x000004c8)
-#define HWIO_IPA_GSI_FIFO_STATUS_CTRL_RMSK 0x3f
-#define HWIO_IPA_GSI_FIFO_STATUS_CTRL_ATTR 0x3
-#define HWIO_IPA_GSI_FIFO_STATUS_CTRL_IN in_dword_masked( \
-		HWIO_IPA_GSI_FIFO_STATUS_CTRL_ADDR, \
-		HWIO_IPA_GSI_FIFO_STATUS_CTRL_RMSK)
-#define HWIO_IPA_GSI_FIFO_STATUS_CTRL_INM(m) in_dword_masked( \
-		HWIO_IPA_GSI_FIFO_STATUS_CTRL_ADDR, \
-		m)
-#define HWIO_IPA_GSI_FIFO_STATUS_CTRL_OUT(v) out_dword(	\
-		HWIO_IPA_GSI_FIFO_STATUS_CTRL_ADDR, \
-		v)
-#define HWIO_IPA_GSI_FIFO_STATUS_CTRL_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_GSI_FIFO_STATUS_CTRL_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_GSI_FIFO_STATUS_CTRL_IN)
-#define HWIO_IPA_GSI_FIFO_STATUS_CTRL_IPA_GSI_FIFO_STATUS_EN_BMSK 0x20
-#define HWIO_IPA_GSI_FIFO_STATUS_CTRL_IPA_GSI_FIFO_STATUS_EN_SHFT 0x5
-#define HWIO_IPA_GSI_FIFO_STATUS_CTRL_IPA_GSI_FIFO_STATUS_PORT_SEL_BMSK	\
-	0x1f
-#define HWIO_IPA_GSI_FIFO_STATUS_CTRL_IPA_GSI_FIFO_STATUS_PORT_SEL_SHFT	\
-	0x0
-#define HWIO_IPA_GSI_TLV_FIFO_STATUS_ADDR (IPA_DEBUG_REG_BASE + 0x000004cc)
-#define HWIO_IPA_GSI_TLV_FIFO_STATUS_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					   0x000004cc)
-#define HWIO_IPA_GSI_TLV_FIFO_STATUS_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					   0x000004cc)
-#define HWIO_IPA_GSI_TLV_FIFO_STATUS_RMSK 0x7fffffff
-#define HWIO_IPA_GSI_TLV_FIFO_STATUS_ATTR 0x1
-#define HWIO_IPA_GSI_TLV_FIFO_STATUS_IN in_dword_masked( \
-		HWIO_IPA_GSI_TLV_FIFO_STATUS_ADDR, \
-		HWIO_IPA_GSI_TLV_FIFO_STATUS_RMSK)
-#define HWIO_IPA_GSI_TLV_FIFO_STATUS_INM(m) in_dword_masked( \
-		HWIO_IPA_GSI_TLV_FIFO_STATUS_ADDR, \
-		m)
-#define HWIO_IPA_GSI_TLV_FIFO_STATUS_FIFO_HEAD_IS_BUBBLE_BMSK 0x40000000
-#define HWIO_IPA_GSI_TLV_FIFO_STATUS_FIFO_HEAD_IS_BUBBLE_SHFT 0x1e
-#define HWIO_IPA_GSI_TLV_FIFO_STATUS_FIFO_FULL_PUB_BMSK 0x20000000
-#define HWIO_IPA_GSI_TLV_FIFO_STATUS_FIFO_FULL_PUB_SHFT 0x1d
-#define HWIO_IPA_GSI_TLV_FIFO_STATUS_FIFO_ALMOST_FULL_PUB_BMSK 0x10000000
-#define HWIO_IPA_GSI_TLV_FIFO_STATUS_FIFO_ALMOST_FULL_PUB_SHFT 0x1c
-#define HWIO_IPA_GSI_TLV_FIFO_STATUS_FIFO_FULL_BMSK 0x8000000
-#define HWIO_IPA_GSI_TLV_FIFO_STATUS_FIFO_FULL_SHFT 0x1b
-#define HWIO_IPA_GSI_TLV_FIFO_STATUS_FIFO_ALMOST_FULL_BMSK 0x4000000
-#define HWIO_IPA_GSI_TLV_FIFO_STATUS_FIFO_ALMOST_FULL_SHFT 0x1a
-#define HWIO_IPA_GSI_TLV_FIFO_STATUS_FIFO_EMPTY_PUB_BMSK 0x2000000
-#define HWIO_IPA_GSI_TLV_FIFO_STATUS_FIFO_EMPTY_PUB_SHFT 0x19
-#define HWIO_IPA_GSI_TLV_FIFO_STATUS_FIFO_EMPTY_BMSK 0x1000000
-#define HWIO_IPA_GSI_TLV_FIFO_STATUS_FIFO_EMPTY_SHFT 0x18
-#define HWIO_IPA_GSI_TLV_FIFO_STATUS_FIFO_RD_PUB_PTR_BMSK 0xff0000
-#define HWIO_IPA_GSI_TLV_FIFO_STATUS_FIFO_RD_PUB_PTR_SHFT 0x10
-#define HWIO_IPA_GSI_TLV_FIFO_STATUS_FIFO_RD_PTR_BMSK 0xff00
-#define HWIO_IPA_GSI_TLV_FIFO_STATUS_FIFO_RD_PTR_SHFT 0x8
-#define HWIO_IPA_GSI_TLV_FIFO_STATUS_FIFO_WR_PTR_BMSK 0xff
-#define HWIO_IPA_GSI_TLV_FIFO_STATUS_FIFO_WR_PTR_SHFT 0x0
-#define HWIO_IPA_GSI_AOS_FIFO_STATUS_ADDR (IPA_DEBUG_REG_BASE + 0x000004d0)
-#define HWIO_IPA_GSI_AOS_FIFO_STATUS_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					   0x000004d0)
-#define HWIO_IPA_GSI_AOS_FIFO_STATUS_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					   0x000004d0)
-#define HWIO_IPA_GSI_AOS_FIFO_STATUS_RMSK 0x7fffffff
-#define HWIO_IPA_GSI_AOS_FIFO_STATUS_ATTR 0x1
-#define HWIO_IPA_GSI_AOS_FIFO_STATUS_IN in_dword_masked( \
-		HWIO_IPA_GSI_AOS_FIFO_STATUS_ADDR, \
-		HWIO_IPA_GSI_AOS_FIFO_STATUS_RMSK)
-#define HWIO_IPA_GSI_AOS_FIFO_STATUS_INM(m) in_dword_masked( \
-		HWIO_IPA_GSI_AOS_FIFO_STATUS_ADDR, \
-		m)
-#define HWIO_IPA_GSI_AOS_FIFO_STATUS_FIFO_HEAD_IS_BUBBLE_BMSK 0x40000000
-#define HWIO_IPA_GSI_AOS_FIFO_STATUS_FIFO_HEAD_IS_BUBBLE_SHFT 0x1e
-#define HWIO_IPA_GSI_AOS_FIFO_STATUS_FIFO_FULL_PUB_BMSK 0x20000000
-#define HWIO_IPA_GSI_AOS_FIFO_STATUS_FIFO_FULL_PUB_SHFT 0x1d
-#define HWIO_IPA_GSI_AOS_FIFO_STATUS_FIFO_ALMOST_FULL_PUB_BMSK 0x10000000
-#define HWIO_IPA_GSI_AOS_FIFO_STATUS_FIFO_ALMOST_FULL_PUB_SHFT 0x1c
-#define HWIO_IPA_GSI_AOS_FIFO_STATUS_FIFO_FULL_BMSK 0x8000000
-#define HWIO_IPA_GSI_AOS_FIFO_STATUS_FIFO_FULL_SHFT 0x1b
-#define HWIO_IPA_GSI_AOS_FIFO_STATUS_FIFO_ALMOST_FULL_BMSK 0x4000000
-#define HWIO_IPA_GSI_AOS_FIFO_STATUS_FIFO_ALMOST_FULL_SHFT 0x1a
-#define HWIO_IPA_GSI_AOS_FIFO_STATUS_FIFO_EMPTY_PUB_BMSK 0x2000000
-#define HWIO_IPA_GSI_AOS_FIFO_STATUS_FIFO_EMPTY_PUB_SHFT 0x19
-#define HWIO_IPA_GSI_AOS_FIFO_STATUS_FIFO_EMPTY_BMSK 0x1000000
-#define HWIO_IPA_GSI_AOS_FIFO_STATUS_FIFO_EMPTY_SHFT 0x18
-#define HWIO_IPA_GSI_AOS_FIFO_STATUS_FIFO_RD_PUB_PTR_BMSK 0xff0000
-#define HWIO_IPA_GSI_AOS_FIFO_STATUS_FIFO_RD_PUB_PTR_SHFT 0x10
-#define HWIO_IPA_GSI_AOS_FIFO_STATUS_FIFO_RD_PTR_BMSK 0xff00
-#define HWIO_IPA_GSI_AOS_FIFO_STATUS_FIFO_RD_PTR_SHFT 0x8
-#define HWIO_IPA_GSI_AOS_FIFO_STATUS_FIFO_WR_PTR_BMSK 0xff
-#define HWIO_IPA_GSI_AOS_FIFO_STATUS_FIFO_WR_PTR_SHFT 0x0
-#define HWIO_IPA_ENDP_GSI_CONS_BYTES_TLV_ADDR (IPA_DEBUG_REG_BASE + \
-					       0x000004d4)
-#define HWIO_IPA_ENDP_GSI_CONS_BYTES_TLV_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					       0x000004d4)
-#define HWIO_IPA_ENDP_GSI_CONS_BYTES_TLV_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					       0x000004d4)
-#define HWIO_IPA_ENDP_GSI_CONS_BYTES_AOS_ADDR (IPA_DEBUG_REG_BASE + \
-					       0x00000548)
-#define HWIO_IPA_ENDP_GSI_CONS_BYTES_AOS_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					       0x00000548)
-#define HWIO_IPA_ENDP_GSI_CONS_BYTES_AOS_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					       0x00000548)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_0_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x000004d8)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_0_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x000004d8)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_0_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x000004d8)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_0_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_0_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_0_IN in_dword_masked( \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_0_ADDR, \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_0_RMSK)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_0_INM(m) in_dword_masked( \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_0_ADDR, \
-		m)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_0_OUT(v) out_dword( \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_0_ADDR, \
-		v)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_0_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_0_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_0_IN)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_0_VALUE_BMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_0_VALUE_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_1_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x000004dc)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_1_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x000004dc)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_1_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x000004dc)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_1_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_1_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_1_IN in_dword_masked( \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_1_ADDR, \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_1_RMSK)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_1_INM(m) in_dword_masked( \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_1_ADDR, \
-		m)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_1_OUT(v) out_dword( \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_1_ADDR, \
-		v)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_1_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_1_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_1_IN)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_1_VALUE_BMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_1_VALUE_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_2_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x000004e0)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_2_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x000004e0)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_2_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x000004e0)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_2_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_2_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_2_IN in_dword_masked( \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_2_ADDR, \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_2_RMSK)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_2_INM(m) in_dword_masked( \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_2_ADDR, \
-		m)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_2_OUT(v) out_dword( \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_2_ADDR, \
-		v)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_2_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_2_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_2_IN)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_2_VALUE_BMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_2_VALUE_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_3_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x000004e4)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_3_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x000004e4)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_3_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x000004e4)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_3_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_3_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_3_IN in_dword_masked( \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_3_ADDR, \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_3_RMSK)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_3_INM(m) in_dword_masked( \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_3_ADDR, \
-		m)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_3_OUT(v) out_dword( \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_3_ADDR, \
-		v)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_3_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_3_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_3_IN)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_3_VALUE_BMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_3_VALUE_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_4_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x000004e8)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_4_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x000004e8)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_4_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x000004e8)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_4_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_4_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_4_IN in_dword_masked( \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_4_ADDR, \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_4_RMSK)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_4_INM(m) in_dword_masked( \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_4_ADDR, \
-		m)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_4_OUT(v) out_dword( \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_4_ADDR, \
-		v)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_4_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_4_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_4_IN)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_4_VALUE_BMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_4_VALUE_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_5_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x000004ec)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_5_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x000004ec)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_5_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x000004ec)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_5_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_5_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_5_IN in_dword_masked( \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_5_ADDR, \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_5_RMSK)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_5_INM(m) in_dword_masked( \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_5_ADDR, \
-		m)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_5_OUT(v) out_dword( \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_5_ADDR, \
-		v)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_5_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_5_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_5_IN)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_5_VALUE_BMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_5_VALUE_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_6_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x000004f0)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_6_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x000004f0)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_6_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x000004f0)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_6_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_6_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_6_IN in_dword_masked( \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_6_ADDR, \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_6_RMSK)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_6_INM(m) in_dword_masked( \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_6_ADDR, \
-		m)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_6_OUT(v) out_dword( \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_6_ADDR, \
-		v)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_6_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_6_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_6_IN)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_6_VALUE_BMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_6_VALUE_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_7_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x000004f4)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_7_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x000004f4)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_7_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x000004f4)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_7_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_7_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_7_IN in_dword_masked( \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_7_ADDR, \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_7_RMSK)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_7_INM(m) in_dword_masked( \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_7_ADDR, \
-		m)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_7_OUT(v) out_dword( \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_7_ADDR, \
-		v)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_7_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_7_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_LOG_BUF_SW_COMP_VAL_7_IN)
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_7_VALUE_BMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_SW_COMP_VAL_7_VALUE_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_0_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x000004f8)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_0_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x000004f8)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_0_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x000004f8)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_0_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_0_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_0_IN in_dword_masked( \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_0_ADDR, \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_0_RMSK)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_0_INM(m) in_dword_masked( \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_0_ADDR, \
-		m)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_0_OUT(v) out_dword( \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_0_ADDR, \
-		v)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_0_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_0_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_0_IN)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_0_VALUE_BMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_0_VALUE_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_1_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x000004fc)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_1_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x000004fc)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_1_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x000004fc)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_1_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_1_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_1_IN in_dword_masked( \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_1_ADDR, \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_1_RMSK)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_1_INM(m) in_dword_masked( \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_1_ADDR, \
-		m)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_1_OUT(v) out_dword( \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_1_ADDR, \
-		v)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_1_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_1_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_1_IN)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_1_VALUE_BMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_1_VALUE_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_2_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x00000500)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_2_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x00000500)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_2_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x00000500)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_2_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_2_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_2_IN in_dword_masked( \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_2_ADDR, \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_2_RMSK)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_2_INM(m) in_dword_masked( \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_2_ADDR, \
-		m)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_2_OUT(v) out_dword( \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_2_ADDR, \
-		v)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_2_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_2_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_2_IN)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_2_VALUE_BMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_2_VALUE_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_3_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x00000504)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_3_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x00000504)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_3_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x00000504)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_3_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_3_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_3_IN in_dword_masked( \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_3_ADDR, \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_3_RMSK)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_3_INM(m) in_dword_masked( \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_3_ADDR, \
-		m)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_3_OUT(v) out_dword( \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_3_ADDR, \
-		v)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_3_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_3_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_3_IN)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_3_VALUE_BMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_3_VALUE_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_4_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x00000508)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_4_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x00000508)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_4_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x00000508)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_4_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_4_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_4_IN in_dword_masked( \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_4_ADDR, \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_4_RMSK)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_4_INM(m) in_dword_masked( \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_4_ADDR, \
-		m)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_4_OUT(v) out_dword( \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_4_ADDR, \
-		v)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_4_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_4_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_4_IN)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_4_VALUE_BMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_4_VALUE_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_5_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x0000050c)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_5_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x0000050c)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_5_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x0000050c)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_5_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_5_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_5_IN in_dword_masked( \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_5_ADDR, \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_5_RMSK)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_5_INM(m) in_dword_masked( \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_5_ADDR, \
-		m)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_5_OUT(v) out_dword( \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_5_ADDR, \
-		v)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_5_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_5_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_5_IN)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_5_VALUE_BMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_5_VALUE_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_6_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x00000510)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_6_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x00000510)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_6_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x00000510)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_6_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_6_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_6_IN in_dword_masked( \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_6_ADDR, \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_6_RMSK)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_6_INM(m) in_dword_masked( \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_6_ADDR, \
-		m)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_6_OUT(v) out_dword( \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_6_ADDR, \
-		v)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_6_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_6_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_6_IN)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_6_VALUE_BMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_6_VALUE_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_7_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x00000514)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_7_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x00000514)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_7_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x00000514)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_7_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_7_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_7_IN in_dword_masked( \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_7_ADDR, \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_7_RMSK)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_7_INM(m) in_dword_masked( \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_7_ADDR, \
-		m)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_7_OUT(v) out_dword( \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_7_ADDR, \
-		v)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_7_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_7_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_LOG_BUF_SW_MASK_VAL_7_IN)
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_7_VALUE_BMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_SW_MASK_VAL_7_VALUE_SHFT 0x0
-#define HWIO_IPA_UC_RX_HND_CMDQ_CMD_ADDR (IPA_DEBUG_REG_BASE + 0x00000518)
-#define HWIO_IPA_UC_RX_HND_CMDQ_CMD_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					  0x00000518)
-#define HWIO_IPA_UC_RX_HND_CMDQ_CMD_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					  0x00000518)
-#define HWIO_IPA_UC_RX_HND_CMDQ_CFG_ADDR (IPA_DEBUG_REG_BASE + 0x0000051c)
-#define HWIO_IPA_UC_RX_HND_CMDQ_CFG_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					  0x0000051c)
-#define HWIO_IPA_UC_RX_HND_CMDQ_CFG_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					  0x0000051c)
-#define HWIO_IPA_UC_RX_HND_CMDQ_DATA_WR_0_ADDR (IPA_DEBUG_REG_BASE + \
-						0x00000520)
-#define HWIO_IPA_UC_RX_HND_CMDQ_DATA_WR_0_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						0x00000520)
-#define HWIO_IPA_UC_RX_HND_CMDQ_DATA_WR_0_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						0x00000520)
-#define HWIO_IPA_UC_RX_HND_CMDQ_DATA_WR_1_ADDR (IPA_DEBUG_REG_BASE + \
-						0x00000524)
-#define HWIO_IPA_UC_RX_HND_CMDQ_DATA_WR_1_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						0x00000524)
-#define HWIO_IPA_UC_RX_HND_CMDQ_DATA_WR_1_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						0x00000524)
-#define HWIO_IPA_UC_RX_HND_CMDQ_DATA_WR_2_ADDR (IPA_DEBUG_REG_BASE + \
-						0x00000528)
-#define HWIO_IPA_UC_RX_HND_CMDQ_DATA_WR_2_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						0x00000528)
-#define HWIO_IPA_UC_RX_HND_CMDQ_DATA_WR_2_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						0x00000528)
-#define HWIO_IPA_UC_RX_HND_CMDQ_DATA_WR_3_ADDR (IPA_DEBUG_REG_BASE + \
-						0x0000052c)
-#define HWIO_IPA_UC_RX_HND_CMDQ_DATA_WR_3_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						0x0000052c)
-#define HWIO_IPA_UC_RX_HND_CMDQ_DATA_WR_3_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						0x0000052c)
-#define HWIO_IPA_UC_RX_HND_CMDQ_DATA_RD_0_ADDR (IPA_DEBUG_REG_BASE + \
-						0x00000530)
-#define HWIO_IPA_UC_RX_HND_CMDQ_DATA_RD_0_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						0x00000530)
-#define HWIO_IPA_UC_RX_HND_CMDQ_DATA_RD_0_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						0x00000530)
-#define HWIO_IPA_UC_RX_HND_CMDQ_DATA_RD_1_ADDR (IPA_DEBUG_REG_BASE + \
-						0x00000534)
-#define HWIO_IPA_UC_RX_HND_CMDQ_DATA_RD_1_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						0x00000534)
-#define HWIO_IPA_UC_RX_HND_CMDQ_DATA_RD_1_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						0x00000534)
-#define HWIO_IPA_UC_RX_HND_CMDQ_DATA_RD_2_ADDR (IPA_DEBUG_REG_BASE + \
-						0x00000538)
-#define HWIO_IPA_UC_RX_HND_CMDQ_DATA_RD_2_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						0x00000538)
-#define HWIO_IPA_UC_RX_HND_CMDQ_DATA_RD_2_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						0x00000538)
-#define HWIO_IPA_UC_RX_HND_CMDQ_DATA_RD_3_ADDR (IPA_DEBUG_REG_BASE + \
-						0x0000053c)
-#define HWIO_IPA_UC_RX_HND_CMDQ_DATA_RD_3_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						0x0000053c)
-#define HWIO_IPA_UC_RX_HND_CMDQ_DATA_RD_3_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						0x0000053c)
-#define HWIO_IPA_UC_RX_HND_CMDQ_STATUS_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x00000540)
-#define HWIO_IPA_UC_RX_HND_CMDQ_STATUS_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x00000540)
-#define HWIO_IPA_UC_RX_HND_CMDQ_STATUS_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x00000540)
-#define HWIO_IPA_RAM_HW_FIRST_ADDR (IPA_DEBUG_REG_BASE + 0x0000054c)
-#define HWIO_IPA_RAM_HW_FIRST_PHYS (IPA_DEBUG_REG_BASE_PHYS + 0x0000054c)
-#define HWIO_IPA_RAM_HW_FIRST_OFFS (IPA_DEBUG_REG_BASE_OFFS + 0x0000054c)
-#define HWIO_IPA_RAM_HW_LAST_ADDR (IPA_DEBUG_REG_BASE + 0x00000550)
-#define HWIO_IPA_RAM_HW_LAST_PHYS (IPA_DEBUG_REG_BASE_PHYS + 0x00000550)
-#define HWIO_IPA_RAM_HW_LAST_OFFS (IPA_DEBUG_REG_BASE_OFFS + 0x00000550)
-#define HWIO_IPA_RAM_SNIFFER_BASE_OFFSET_ADDR (IPA_DEBUG_REG_BASE + \
-					       0x00000554)
-#define HWIO_IPA_RAM_SNIFFER_BASE_OFFSET_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					       0x00000554)
-#define HWIO_IPA_RAM_SNIFFER_BASE_OFFSET_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					       0x00000554)
-#define HWIO_IPA_RAM_FRAG_FRST_BASE_ADDR_ADDR (IPA_DEBUG_REG_BASE + \
-					       0x00000558)
-#define HWIO_IPA_RAM_FRAG_FRST_BASE_ADDR_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					       0x00000558)
-#define HWIO_IPA_RAM_FRAG_FRST_BASE_ADDR_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					       0x00000558)
-#define HWIO_IPA_RAM_FRAG_SCND_BASE_ADDR_ADDR (IPA_DEBUG_REG_BASE + \
-					       0x0000055c)
-#define HWIO_IPA_RAM_FRAG_SCND_BASE_ADDR_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					       0x0000055c)
-#define HWIO_IPA_RAM_FRAG_SCND_BASE_ADDR_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					       0x0000055c)
-#define HWIO_IPA_RAM_GSI_TLV_BASE_ADDR_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x00000560)
-#define HWIO_IPA_RAM_GSI_TLV_BASE_ADDR_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x00000560)
-#define HWIO_IPA_RAM_GSI_TLV_BASE_ADDR_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x00000560)
-#define HWIO_IPA_RAM_DCPH_KEYS_FIRST_ADDR (IPA_DEBUG_REG_BASE + 0x00000564)
-#define HWIO_IPA_RAM_DCPH_KEYS_FIRST_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					   0x00000564)
-#define HWIO_IPA_RAM_DCPH_KEYS_FIRST_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					   0x00000564)
-#define HWIO_IPA_RAM_DCPH_KEYS_LAST_ADDR (IPA_DEBUG_REG_BASE + 0x00000568)
-#define HWIO_IPA_RAM_DCPH_KEYS_LAST_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					  0x00000568)
-#define HWIO_IPA_RAM_DCPH_KEYS_LAST_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					  0x00000568)
-#define HWIO_IPA_DPS_SEQUENCER_FIRST_ADDR (IPA_DEBUG_REG_BASE + 0x00000570)
-#define HWIO_IPA_DPS_SEQUENCER_FIRST_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					   0x00000570)
-#define HWIO_IPA_DPS_SEQUENCER_FIRST_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					   0x00000570)
-#define HWIO_IPA_DPS_SEQUENCER_LAST_ADDR (IPA_DEBUG_REG_BASE + 0x00000574)
-#define HWIO_IPA_DPS_SEQUENCER_LAST_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					  0x00000574)
-#define HWIO_IPA_DPS_SEQUENCER_LAST_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					  0x00000574)
-#define HWIO_IPA_HPS_SEQUENCER_FIRST_ADDR (IPA_DEBUG_REG_BASE + 0x00000578)
-#define HWIO_IPA_HPS_SEQUENCER_FIRST_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					   0x00000578)
-#define HWIO_IPA_HPS_SEQUENCER_FIRST_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					   0x00000578)
-#define HWIO_IPA_HPS_SEQUENCER_LAST_ADDR (IPA_DEBUG_REG_BASE + 0x0000057c)
-#define HWIO_IPA_HPS_SEQUENCER_LAST_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					  0x0000057c)
-#define HWIO_IPA_HPS_SEQUENCER_LAST_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					  0x0000057c)
-#define HWIO_IPA_RAM_PKT_CTX_BASE_ADDR_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x00000650)
-#define HWIO_IPA_RAM_PKT_CTX_BASE_ADDR_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x00000650)
-#define HWIO_IPA_RAM_PKT_CTX_BASE_ADDR_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x00000650)
-#define HWIO_IPA_RAM_SW_AREA_BASE_ADDR_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x00000654)
-#define HWIO_IPA_RAM_SW_AREA_BASE_ADDR_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x00000654)
-#define HWIO_IPA_RAM_SW_AREA_BASE_ADDR_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x00000654)
-#define HWIO_IPA_RAM_HDRI_TYPE1_BASE_ADDR_ADDR (IPA_DEBUG_REG_BASE + \
-						0x00000658)
-#define HWIO_IPA_RAM_HDRI_TYPE1_BASE_ADDR_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						0x00000658)
-#define HWIO_IPA_RAM_HDRI_TYPE1_BASE_ADDR_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						0x00000658)
-#define HWIO_IPA_RAM_AGGR_NLO_COUNTERS_BASE_ADDR_ADDR (IPA_DEBUG_REG_BASE \
-						       + 0x0000065c)
-#define HWIO_IPA_RAM_AGGR_NLO_COUNTERS_BASE_ADDR_PHYS (	\
-		IPA_DEBUG_REG_BASE_PHYS + 0x0000065c)
-#define HWIO_IPA_RAM_AGGR_NLO_COUNTERS_BASE_ADDR_OFFS (	\
-		IPA_DEBUG_REG_BASE_OFFS + 0x0000065c)
-#define HWIO_IPA_RAM_NLO_VP_CACHE_BASE_ADDR_ADDR (IPA_DEBUG_REG_BASE + \
-						  0x00000660)
-#define HWIO_IPA_RAM_NLO_VP_CACHE_BASE_ADDR_PHYS (IPA_DEBUG_REG_BASE_PHYS \
-						  + 0x00000660)
-#define HWIO_IPA_RAM_NLO_VP_CACHE_BASE_ADDR_OFFS (IPA_DEBUG_REG_BASE_OFFS \
-						  + 0x00000660)
-#define HWIO_IPA_RAM_COAL_VP_CACHE_BASE_ADDR_ADDR (IPA_DEBUG_REG_BASE +	\
-						   0x00000664)
-#define HWIO_IPA_RAM_COAL_VP_CACHE_BASE_ADDR_PHYS (IPA_DEBUG_REG_BASE_PHYS \
-						   + 0x00000664)
-#define HWIO_IPA_RAM_COAL_VP_CACHE_BASE_ADDR_OFFS (IPA_DEBUG_REG_BASE_OFFS \
-						   + 0x00000664)
-#define HWIO_IPA_RAM_COAL_VP_FIFO_BASE_ADDR_ADDR (IPA_DEBUG_REG_BASE + \
-						  0x00000668)
-#define HWIO_IPA_RAM_COAL_VP_FIFO_BASE_ADDR_PHYS (IPA_DEBUG_REG_BASE_PHYS \
-						  + 0x00000668)
-#define HWIO_IPA_RAM_COAL_VP_FIFO_BASE_ADDR_OFFS (IPA_DEBUG_REG_BASE_OFFS \
-						  + 0x00000668)
-#define HWIO_IPA_RAM_GSI_IF_CONS_ACCUMS_BASE_ADDR_ADDR (IPA_DEBUG_REG_BASE \
-							+ 0x0000066c)
-#define HWIO_IPA_RAM_GSI_IF_CONS_ACCUMS_BASE_ADDR_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x0000066c)
-#define HWIO_IPA_RAM_GSI_IF_CONS_ACCUMS_BASE_ADDR_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x0000066c)
-#define HWIO_IPA_RAM_AGGR_BASE_ADDR_ADDR (IPA_DEBUG_REG_BASE + 0x00000670)
-#define HWIO_IPA_RAM_AGGR_BASE_ADDR_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					  0x00000670)
-#define HWIO_IPA_RAM_AGGR_BASE_ADDR_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					  0x00000670)
-#define HWIO_IPA_RAM_TX_COUNTERS_BASE_ADDR_ADDR (IPA_DEBUG_REG_BASE + \
-						 0x00000674)
-#define HWIO_IPA_RAM_TX_COUNTERS_BASE_ADDR_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						 0x00000674)
-#define HWIO_IPA_RAM_TX_COUNTERS_BASE_ADDR_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						 0x00000674)
-#define HWIO_IPA_RAM_DPL_FIFO_BASE_ADDR_ADDR (IPA_DEBUG_REG_BASE + \
-					      0x00000678)
-#define HWIO_IPA_RAM_DPL_FIFO_BASE_ADDR_PHYS (IPA_DEBUG_REG_BASE_PHYS +	\
-					      0x00000678)
-#define HWIO_IPA_RAM_DPL_FIFO_BASE_ADDR_OFFS (IPA_DEBUG_REG_BASE_OFFS +	\
-					      0x00000678)
-#define HWIO_IPA_RAM_COAL_MASTER_VP_CTX_BASE_ADDR_ADDR (IPA_DEBUG_REG_BASE \
-							+ 0x0000067c)
-#define HWIO_IPA_RAM_COAL_MASTER_VP_CTX_BASE_ADDR_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x0000067c)
-#define HWIO_IPA_RAM_COAL_MASTER_VP_CTX_BASE_ADDR_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x0000067c)
-#define HWIO_IPA_RAM_COAL_MASTER_VP_AGGR_BASE_ADDR_ADDR ( \
-		IPA_DEBUG_REG_BASE + 0x00000680)
-#define HWIO_IPA_RAM_COAL_MASTER_VP_AGGR_BASE_ADDR_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000680)
-#define HWIO_IPA_RAM_COAL_MASTER_VP_AGGR_BASE_ADDR_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000680)
-#define HWIO_IPA_RAM_COAL_SLAVE_VP_CTX_BASE_ADDR_ADDR (IPA_DEBUG_REG_BASE \
-						       + 0x00000684)
-#define HWIO_IPA_RAM_COAL_SLAVE_VP_CTX_BASE_ADDR_PHYS (	\
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000684)
-#define HWIO_IPA_RAM_COAL_SLAVE_VP_CTX_BASE_ADDR_OFFS (	\
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000684)
-#define HWIO_IPA_RAM_UL_NLO_AGGR_BASE_ADDR_ADDR (IPA_DEBUG_REG_BASE + \
-						 0x00000688)
-#define HWIO_IPA_RAM_UL_NLO_AGGR_BASE_ADDR_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						 0x00000688)
-#define HWIO_IPA_RAM_UL_NLO_AGGR_BASE_ADDR_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						 0x00000688)
-#define HWIO_IPA_RAM_UC_IRAM_ADDR_BASE_ADDR_ADDR (IPA_DEBUG_REG_BASE + \
-						  0x0000069c)
-#define HWIO_IPA_RAM_UC_IRAM_ADDR_BASE_ADDR_PHYS (IPA_DEBUG_REG_BASE_PHYS \
-						  + 0x0000069c)
-#define HWIO_IPA_RAM_UC_IRAM_ADDR_BASE_ADDR_OFFS (IPA_DEBUG_REG_BASE_OFFS \
-						  + 0x0000069c)
-#define HWIO_IPA_HPS_UC2SEQ_PUSH_ADDR (IPA_DEBUG_REG_BASE + 0x00000580)
-#define HWIO_IPA_HPS_UC2SEQ_PUSH_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-				       0x00000580)
-#define HWIO_IPA_HPS_UC2SEQ_PUSH_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-				       0x00000580)
-#define HWIO_IPA_HPS_UC2SEQ_STATUS_ADDR (IPA_DEBUG_REG_BASE + 0x00000584)
-#define HWIO_IPA_HPS_UC2SEQ_STATUS_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					 0x00000584)
-#define HWIO_IPA_HPS_UC2SEQ_STATUS_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					 0x00000584)
-#define HWIO_IPA_HPS_SEQ2UC_RD_ADDR (IPA_DEBUG_REG_BASE + 0x00000588)
-#define HWIO_IPA_HPS_SEQ2UC_RD_PHYS (IPA_DEBUG_REG_BASE_PHYS + 0x00000588)
-#define HWIO_IPA_HPS_SEQ2UC_RD_OFFS (IPA_DEBUG_REG_BASE_OFFS + 0x00000588)
-#define HWIO_IPA_HPS_SEQ2UC_STATUS_ADDR (IPA_DEBUG_REG_BASE + 0x0000058c)
-#define HWIO_IPA_HPS_SEQ2UC_STATUS_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					 0x0000058c)
-#define HWIO_IPA_HPS_SEQ2UC_STATUS_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					 0x0000058c)
-#define HWIO_IPA_HPS_SEQ2UC_CMD_ADDR (IPA_DEBUG_REG_BASE + 0x00000590)
-#define HWIO_IPA_HPS_SEQ2UC_CMD_PHYS (IPA_DEBUG_REG_BASE_PHYS + 0x00000590)
-#define HWIO_IPA_HPS_SEQ2UC_CMD_OFFS (IPA_DEBUG_REG_BASE_OFFS + 0x00000590)
-#define HWIO_IPA_DPS_UC2SEQ_PUSH_ADDR (IPA_DEBUG_REG_BASE + 0x00000594)
-#define HWIO_IPA_DPS_UC2SEQ_PUSH_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-				       0x00000594)
-#define HWIO_IPA_DPS_UC2SEQ_PUSH_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-				       0x00000594)
-#define HWIO_IPA_DPS_UC2SEQ_STATUS_ADDR (IPA_DEBUG_REG_BASE + 0x00000598)
-#define HWIO_IPA_DPS_UC2SEQ_STATUS_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					 0x00000598)
-#define HWIO_IPA_DPS_UC2SEQ_STATUS_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					 0x00000598)
-#define HWIO_IPA_DPS_SEQ2UC_RD_ADDR (IPA_DEBUG_REG_BASE + 0x0000059c)
-#define HWIO_IPA_DPS_SEQ2UC_RD_PHYS (IPA_DEBUG_REG_BASE_PHYS + 0x0000059c)
-#define HWIO_IPA_DPS_SEQ2UC_RD_OFFS (IPA_DEBUG_REG_BASE_OFFS + 0x0000059c)
-#define HWIO_IPA_DPS_SEQ2UC_STATUS_ADDR (IPA_DEBUG_REG_BASE + 0x000005a0)
-#define HWIO_IPA_DPS_SEQ2UC_STATUS_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					 0x000005a0)
-#define HWIO_IPA_DPS_SEQ2UC_STATUS_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					 0x000005a0)
-#define HWIO_IPA_DPS_SEQ2UC_CMD_ADDR (IPA_DEBUG_REG_BASE + 0x000005a4)
-#define HWIO_IPA_DPS_SEQ2UC_CMD_PHYS (IPA_DEBUG_REG_BASE_PHYS + 0x000005a4)
-#define HWIO_IPA_DPS_SEQ2UC_CMD_OFFS (IPA_DEBUG_REG_BASE_OFFS + 0x000005a4)
-#define HWIO_IPA_NTF_TX_CMDQ_CMD_ADDR (IPA_DEBUG_REG_BASE + 0x00000600)
-#define HWIO_IPA_NTF_TX_CMDQ_CMD_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-				       0x00000600)
-#define HWIO_IPA_NTF_TX_CMDQ_CMD_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-				       0x00000600)
-#define HWIO_IPA_NTF_TX_CMDQ_CMD_RMSK 0xff
-#define HWIO_IPA_NTF_TX_CMDQ_CMD_ATTR 0x3
-#define HWIO_IPA_NTF_TX_CMDQ_CMD_IN in_dword_masked( \
-		HWIO_IPA_NTF_TX_CMDQ_CMD_ADDR, \
-		HWIO_IPA_NTF_TX_CMDQ_CMD_RMSK)
-#define HWIO_IPA_NTF_TX_CMDQ_CMD_INM(m) in_dword_masked( \
-		HWIO_IPA_NTF_TX_CMDQ_CMD_ADDR, \
-		m)
-#define HWIO_IPA_NTF_TX_CMDQ_CMD_OUT(v) out_dword( \
-		HWIO_IPA_NTF_TX_CMDQ_CMD_ADDR, \
-		v)
-#define HWIO_IPA_NTF_TX_CMDQ_CMD_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_NTF_TX_CMDQ_CMD_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_NTF_TX_CMDQ_CMD_IN)
-#define HWIO_IPA_NTF_TX_CMDQ_CMD_RD_REQ_BMSK 0x80
-#define HWIO_IPA_NTF_TX_CMDQ_CMD_RD_REQ_SHFT 0x7
-#define HWIO_IPA_NTF_TX_CMDQ_CMD_CMD_CLIENT_BMSK 0x7c
-#define HWIO_IPA_NTF_TX_CMDQ_CMD_CMD_CLIENT_SHFT 0x2
-#define HWIO_IPA_NTF_TX_CMDQ_CMD_POP_CMD_BMSK 0x2
-#define HWIO_IPA_NTF_TX_CMDQ_CMD_POP_CMD_SHFT 0x1
-#define HWIO_IPA_NTF_TX_CMDQ_CMD_WRITE_CMD_BMSK 0x1
-#define HWIO_IPA_NTF_TX_CMDQ_CMD_WRITE_CMD_SHFT 0x0
-#define HWIO_IPA_NTF_TX_CMDQ_RELEASE_WR_ADDR (IPA_DEBUG_REG_BASE + \
-					      0x00000604)
-#define HWIO_IPA_NTF_TX_CMDQ_RELEASE_WR_PHYS (IPA_DEBUG_REG_BASE_PHYS +	\
-					      0x00000604)
-#define HWIO_IPA_NTF_TX_CMDQ_RELEASE_WR_OFFS (IPA_DEBUG_REG_BASE_OFFS +	\
-					      0x00000604)
-#define HWIO_IPA_NTF_TX_CMDQ_RELEASE_RD_ADDR (IPA_DEBUG_REG_BASE + \
-					      0x00000608)
-#define HWIO_IPA_NTF_TX_CMDQ_RELEASE_RD_PHYS (IPA_DEBUG_REG_BASE_PHYS +	\
-					      0x00000608)
-#define HWIO_IPA_NTF_TX_CMDQ_RELEASE_RD_OFFS (IPA_DEBUG_REG_BASE_OFFS +	\
-					      0x00000608)
-#define HWIO_IPA_NTF_TX_CMDQ_CFG_WR_ADDR (IPA_DEBUG_REG_BASE + 0x0000060c)
-#define HWIO_IPA_NTF_TX_CMDQ_CFG_WR_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					  0x0000060c)
-#define HWIO_IPA_NTF_TX_CMDQ_CFG_WR_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					  0x0000060c)
-#define HWIO_IPA_NTF_TX_CMDQ_CFG_RD_ADDR (IPA_DEBUG_REG_BASE + 0x00000610)
-#define HWIO_IPA_NTF_TX_CMDQ_CFG_RD_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					  0x00000610)
-#define HWIO_IPA_NTF_TX_CMDQ_CFG_RD_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					  0x00000610)
-#define HWIO_IPA_NTF_TX_CMDQ_DATA_WR_0_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x00000614)
-#define HWIO_IPA_NTF_TX_CMDQ_DATA_WR_0_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x00000614)
-#define HWIO_IPA_NTF_TX_CMDQ_DATA_WR_0_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x00000614)
-#define HWIO_IPA_NTF_TX_CMDQ_DATA_RD_0_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x00000618)
-#define HWIO_IPA_NTF_TX_CMDQ_DATA_RD_0_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x00000618)
-#define HWIO_IPA_NTF_TX_CMDQ_DATA_RD_0_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x00000618)
-#define HWIO_IPA_NTF_TX_CMDQ_DATA_RD_0_RMSK 0xfffff
-#define HWIO_IPA_NTF_TX_CMDQ_DATA_RD_0_ATTR 0x1
-#define HWIO_IPA_NTF_TX_CMDQ_DATA_RD_0_IN in_dword_masked( \
-		HWIO_IPA_NTF_TX_CMDQ_DATA_RD_0_ADDR, \
-		HWIO_IPA_NTF_TX_CMDQ_DATA_RD_0_RMSK)
-#define HWIO_IPA_NTF_TX_CMDQ_DATA_RD_0_INM(m) in_dword_masked( \
-		HWIO_IPA_NTF_TX_CMDQ_DATA_RD_0_ADDR, \
-		m)
-#define HWIO_IPA_NTF_TX_CMDQ_DATA_RD_0_CMDQ_REP_F_BMSK 0x80000
-#define HWIO_IPA_NTF_TX_CMDQ_DATA_RD_0_CMDQ_REP_F_SHFT 0x13
-#define HWIO_IPA_NTF_TX_CMDQ_DATA_RD_0_CMDQ_OPCODE_F_BMSK 0x60000
-#define HWIO_IPA_NTF_TX_CMDQ_DATA_RD_0_CMDQ_OPCODE_F_SHFT 0x11
-#define HWIO_IPA_NTF_TX_CMDQ_DATA_RD_0_CMDQ_SRC_PIPE_F_BMSK 0x1f000
-#define HWIO_IPA_NTF_TX_CMDQ_DATA_RD_0_CMDQ_SRC_PIPE_F_SHFT 0xc
-#define HWIO_IPA_NTF_TX_CMDQ_DATA_RD_0_CMDQ_SRC_ID_F_BMSK 0xff0
-#define HWIO_IPA_NTF_TX_CMDQ_DATA_RD_0_CMDQ_SRC_ID_F_SHFT 0x4
-#define HWIO_IPA_NTF_TX_CMDQ_DATA_RD_0_CMDQ_CTX_ID_F_BMSK 0xf
-#define HWIO_IPA_NTF_TX_CMDQ_DATA_RD_0_CMDQ_CTX_ID_F_SHFT 0x0
-#define HWIO_IPA_NTF_TX_CMDQ_STATUS_ADDR (IPA_DEBUG_REG_BASE + 0x0000061c)
-#define HWIO_IPA_NTF_TX_CMDQ_STATUS_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					  0x0000061c)
-#define HWIO_IPA_NTF_TX_CMDQ_STATUS_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					  0x0000061c)
-#define HWIO_IPA_NTF_TX_CMDQ_STATUS_RMSK 0x1ff
-#define HWIO_IPA_NTF_TX_CMDQ_STATUS_ATTR 0x1
-#define HWIO_IPA_NTF_TX_CMDQ_STATUS_IN in_dword_masked(	\
-		HWIO_IPA_NTF_TX_CMDQ_STATUS_ADDR, \
-		HWIO_IPA_NTF_TX_CMDQ_STATUS_RMSK)
-#define HWIO_IPA_NTF_TX_CMDQ_STATUS_INM(m) in_dword_masked( \
-		HWIO_IPA_NTF_TX_CMDQ_STATUS_ADDR, \
-		m)
-#define HWIO_IPA_NTF_TX_CMDQ_STATUS_CMDQ_DEPTH_BMSK 0x1fc
-#define HWIO_IPA_NTF_TX_CMDQ_STATUS_CMDQ_DEPTH_SHFT 0x2
-#define HWIO_IPA_NTF_TX_CMDQ_STATUS_CMDQ_FULL_BMSK 0x2
-#define HWIO_IPA_NTF_TX_CMDQ_STATUS_CMDQ_FULL_SHFT 0x1
-#define HWIO_IPA_NTF_TX_CMDQ_STATUS_STATUS_BMSK 0x1
-#define HWIO_IPA_NTF_TX_CMDQ_STATUS_STATUS_SHFT 0x0
-#define HWIO_IPA_NTF_TX_CMDQ_STATUS_EMPTY_ADDR (IPA_DEBUG_REG_BASE + \
-						0x00000620)
-#define HWIO_IPA_NTF_TX_CMDQ_STATUS_EMPTY_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						0x00000620)
-#define HWIO_IPA_NTF_TX_CMDQ_STATUS_EMPTY_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						0x00000620)
-#define HWIO_IPA_NTF_TX_CMDQ_STATUS_EMPTY_RMSK 0x7fffffff
-#define HWIO_IPA_NTF_TX_CMDQ_STATUS_EMPTY_ATTR 0x1
-#define HWIO_IPA_NTF_TX_CMDQ_STATUS_EMPTY_IN in_dword_masked( \
-		HWIO_IPA_NTF_TX_CMDQ_STATUS_EMPTY_ADDR,	\
-		HWIO_IPA_NTF_TX_CMDQ_STATUS_EMPTY_RMSK)
-#define HWIO_IPA_NTF_TX_CMDQ_STATUS_EMPTY_INM(m) in_dword_masked( \
-		HWIO_IPA_NTF_TX_CMDQ_STATUS_EMPTY_ADDR,	\
-		m)
-#define HWIO_IPA_NTF_TX_CMDQ_STATUS_EMPTY_CMDQ_EMPTY_BMSK 0x7fffffff
-#define HWIO_IPA_NTF_TX_CMDQ_STATUS_EMPTY_CMDQ_EMPTY_SHFT 0x0
-#define HWIO_IPA_NTF_TX_SNP_ADDR (IPA_DEBUG_REG_BASE + 0x00000624)
-#define HWIO_IPA_NTF_TX_SNP_PHYS (IPA_DEBUG_REG_BASE_PHYS + 0x00000624)
-#define HWIO_IPA_NTF_TX_SNP_OFFS (IPA_DEBUG_REG_BASE_OFFS + 0x00000624)
-#define HWIO_IPA_NTF_TX_CMDQ_COUNT_ADDR (IPA_DEBUG_REG_BASE + 0x00000628)
-#define HWIO_IPA_NTF_TX_CMDQ_COUNT_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					 0x00000628)
-#define HWIO_IPA_NTF_TX_CMDQ_COUNT_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					 0x00000628)
-#define HWIO_IPA_NTF_TX_CMDQ_COUNT_RMSK 0x7f
-#define HWIO_IPA_NTF_TX_CMDQ_COUNT_ATTR 0x1
-#define HWIO_IPA_NTF_TX_CMDQ_COUNT_IN in_dword_masked( \
-		HWIO_IPA_NTF_TX_CMDQ_COUNT_ADDR, \
-		HWIO_IPA_NTF_TX_CMDQ_COUNT_RMSK)
-#define HWIO_IPA_NTF_TX_CMDQ_COUNT_INM(m) in_dword_masked( \
-		HWIO_IPA_NTF_TX_CMDQ_COUNT_ADDR, \
-		m)
-#define HWIO_IPA_NTF_TX_CMDQ_COUNT_FIFO_COUNT_BMSK 0x7f
-#define HWIO_IPA_NTF_TX_CMDQ_COUNT_FIFO_COUNT_SHFT 0x0
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_CMD_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x00000700)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_CMD_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x00000700)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_CMD_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x00000700)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_CMD_RMSK 0xff
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_CMD_ATTR 0x3
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_CMD_IN in_dword_masked( \
-		HWIO_IPA_PROD_ACKMNGR_CMDQ_CMD_ADDR, \
-		HWIO_IPA_PROD_ACKMNGR_CMDQ_CMD_RMSK)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_CMD_INM(m) in_dword_masked( \
-		HWIO_IPA_PROD_ACKMNGR_CMDQ_CMD_ADDR, \
-		m)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_CMD_OUT(v) out_dword( \
-		HWIO_IPA_PROD_ACKMNGR_CMDQ_CMD_ADDR, \
-		v)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_CMD_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_PROD_ACKMNGR_CMDQ_CMD_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_PROD_ACKMNGR_CMDQ_CMD_IN)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_CMD_RD_REQ_BMSK 0x80
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_CMD_RD_REQ_SHFT 0x7
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_CMD_CMD_CLIENT_BMSK 0x7c
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_CMD_CMD_CLIENT_SHFT 0x2
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_CMD_POP_CMD_BMSK 0x2
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_CMD_POP_CMD_SHFT 0x1
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_CMD_WRITE_CMD_BMSK 0x1
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_CMD_WRITE_CMD_SHFT 0x0
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_RELEASE_WR_ADDR (IPA_DEBUG_REG_BASE + \
-						    0x00000704)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_RELEASE_WR_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000704)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_RELEASE_WR_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000704)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_RELEASE_RD_ADDR (IPA_DEBUG_REG_BASE + \
-						    0x00000708)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_RELEASE_RD_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000708)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_RELEASE_RD_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000708)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_CFG_WR_ADDR (IPA_DEBUG_REG_BASE + \
-						0x0000070c)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_CFG_WR_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						0x0000070c)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_CFG_WR_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						0x0000070c)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_CFG_RD_ADDR (IPA_DEBUG_REG_BASE + \
-						0x00000710)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_CFG_RD_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						0x00000710)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_CFG_RD_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						0x00000710)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_DATA_WR_ADDR (IPA_DEBUG_REG_BASE + \
-						 0x00000714)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_DATA_WR_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						 0x00000714)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_DATA_WR_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						 0x00000714)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_DATA_RD_ADDR (IPA_DEBUG_REG_BASE + \
-						 0x00000718)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_DATA_RD_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						 0x00000718)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_DATA_RD_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						 0x00000718)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_DATA_RD_RMSK 0xffffffff
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_DATA_RD_ATTR 0x3
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_DATA_RD_IN in_dword_masked( \
-		HWIO_IPA_PROD_ACKMNGR_CMDQ_DATA_RD_ADDR, \
-		HWIO_IPA_PROD_ACKMNGR_CMDQ_DATA_RD_RMSK)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_DATA_RD_INM(m) in_dword_masked( \
-		HWIO_IPA_PROD_ACKMNGR_CMDQ_DATA_RD_ADDR, \
-		m)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_DATA_RD_OUT(v) out_dword( \
-		HWIO_IPA_PROD_ACKMNGR_CMDQ_DATA_RD_ADDR, \
-		v)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_DATA_RD_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_PROD_ACKMNGR_CMDQ_DATA_RD_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_PROD_ACKMNGR_CMDQ_DATA_RD_IN)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_DATA_RD_CMDQ_USERDATA_BMSK 0xf8000000
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_DATA_RD_CMDQ_USERDATA_SHFT 0x1b
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_DATA_RD_CMDQ_SRC_ID_VALID_BMSK \
-	0x4000000
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_DATA_RD_CMDQ_SRC_ID_VALID_SHFT 0x1a
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_DATA_RD_CMDQ_SENT_BMSK 0x2000000
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_DATA_RD_CMDQ_SENT_SHFT 0x19
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_DATA_RD_CMDQ_ORIGIN_BMSK 0x1000000
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_DATA_RD_CMDQ_ORIGIN_SHFT 0x18
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_DATA_RD_CMDQ_LENGTH_BMSK 0xffff00
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_DATA_RD_CMDQ_LENGTH_SHFT 0x8
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_DATA_RD_CMDQ_SRC_ID_BMSK 0xff
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_DATA_RD_CMDQ_SRC_ID_SHFT 0x0
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_STATUS_ADDR (IPA_DEBUG_REG_BASE + \
-						0x0000071c)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_STATUS_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						0x0000071c)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_STATUS_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						0x0000071c)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_STATUS_RMSK 0x1ff
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_STATUS_ATTR 0x1
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_STATUS_IN in_dword_masked( \
-		HWIO_IPA_PROD_ACKMNGR_CMDQ_STATUS_ADDR,	\
-		HWIO_IPA_PROD_ACKMNGR_CMDQ_STATUS_RMSK)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_STATUS_INM(m) in_dword_masked( \
-		HWIO_IPA_PROD_ACKMNGR_CMDQ_STATUS_ADDR,	\
-		m)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_STATUS_CMDQ_DEPTH_BMSK 0x1fc
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_STATUS_CMDQ_DEPTH_SHFT 0x2
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_STATUS_CMDQ_FULL_BMSK 0x2
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_STATUS_CMDQ_FULL_SHFT 0x1
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_STATUS_STATUS_BMSK 0x1
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_STATUS_STATUS_SHFT 0x0
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_STATUS_EMPTY_ADDR (IPA_DEBUG_REG_BASE + \
-						      0x00000720)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_STATUS_EMPTY_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000720)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_STATUS_EMPTY_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000720)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_STATUS_EMPTY_RMSK 0x7fffffff
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_STATUS_EMPTY_ATTR 0x1
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_STATUS_EMPTY_IN in_dword_masked( \
-		HWIO_IPA_PROD_ACKMNGR_CMDQ_STATUS_EMPTY_ADDR, \
-		HWIO_IPA_PROD_ACKMNGR_CMDQ_STATUS_EMPTY_RMSK)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_STATUS_EMPTY_INM(m) in_dword_masked(	\
-		HWIO_IPA_PROD_ACKMNGR_CMDQ_STATUS_EMPTY_ADDR, \
-		m)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_STATUS_EMPTY_CMDQ_EMPTY_BMSK 0x7fffffff
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_STATUS_EMPTY_CMDQ_EMPTY_SHFT 0x0
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_COUNT_ADDR (IPA_DEBUG_REG_BASE + \
-					       0x00000724)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_COUNT_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					       0x00000724)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_COUNT_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					       0x00000724)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_COUNT_RMSK 0x7f
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_COUNT_ATTR 0x1
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_COUNT_IN in_dword_masked( \
-		HWIO_IPA_PROD_ACKMNGR_CMDQ_COUNT_ADDR, \
-		HWIO_IPA_PROD_ACKMNGR_CMDQ_COUNT_RMSK)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_COUNT_INM(m) in_dword_masked( \
-		HWIO_IPA_PROD_ACKMNGR_CMDQ_COUNT_ADDR, \
-		m)
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_COUNT_FIFO_COUNT_BMSK 0x7f
-#define HWIO_IPA_PROD_ACKMNGR_CMDQ_COUNT_FIFO_COUNT_SHFT 0x0
-#define HWIO_IPA_ACKMNGR_CONTROL_CFG_0_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x00000728)
-#define HWIO_IPA_ACKMNGR_CONTROL_CFG_0_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x00000728)
-#define HWIO_IPA_ACKMNGR_CONTROL_CFG_0_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x00000728)
-#define HWIO_IPA_ACKMNGR_CONTROL_CFG_1_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x0000072c)
-#define HWIO_IPA_ACKMNGR_CONTROL_CFG_1_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x0000072c)
-#define HWIO_IPA_ACKMNGR_CONTROL_CFG_1_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x0000072c)
-#define HWIO_IPA_ACKMNGR_CONTROL_CFG_2_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x00000730)
-#define HWIO_IPA_ACKMNGR_CONTROL_CFG_2_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x00000730)
-#define HWIO_IPA_ACKMNGR_CONTROL_CFG_2_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x00000730)
-#define HWIO_IPA_ACKMNGR_CONTROL_CFG_3_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x00000734)
-#define HWIO_IPA_ACKMNGR_CONTROL_CFG_3_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x00000734)
-#define HWIO_IPA_ACKMNGR_CONTROL_CFG_3_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x00000734)
-#define HWIO_IPA_ACKMNGR_CONTROL_CFG_4_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x00000738)
-#define HWIO_IPA_ACKMNGR_CONTROL_CFG_4_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x00000738)
-#define HWIO_IPA_ACKMNGR_CONTROL_CFG_4_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x00000738)
-#define HWIO_IPA_ACKMNGR_CONTROL_CFG_5_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x0000073c)
-#define HWIO_IPA_ACKMNGR_CONTROL_CFG_5_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x0000073c)
-#define HWIO_IPA_ACKMNGR_CONTROL_CFG_5_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x0000073c)
-#define HWIO_IPA_ACKMNGR_CONTROL_CFG_6_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x00000740)
-#define HWIO_IPA_ACKMNGR_CONTROL_CFG_6_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x00000740)
-#define HWIO_IPA_ACKMNGR_CONTROL_CFG_6_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x00000740)
-#define HWIO_IPA_ACKMNGR_SW_ACCESS_ACKINJ_CFG_ADDR (IPA_DEBUG_REG_BASE + \
-						    0x00000744)
-#define HWIO_IPA_ACKMNGR_SW_ACCESS_ACKINJ_CFG_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000744)
-#define HWIO_IPA_ACKMNGR_SW_ACCESS_ACKINJ_CFG_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000744)
-#define HWIO_IPA_ACKMNGR_SW_ACCESS_ACKUPD_CFG_ADDR (IPA_DEBUG_REG_BASE + \
-						    0x00000748)
-#define HWIO_IPA_ACKMNGR_SW_ACCESS_ACKUPD_CFG_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000748)
-#define HWIO_IPA_ACKMNGR_SW_ACCESS_ACKUPD_CFG_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000748)
-#define HWIO_IPA_ACKMNGR_SW_ACCESS_CMD_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x0000074c)
-#define HWIO_IPA_ACKMNGR_SW_ACCESS_CMD_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x0000074c)
-#define HWIO_IPA_ACKMNGR_SW_ACCESS_CMD_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x0000074c)
-#define HWIO_IPA_ACKMNGR_SW_ACCESS_STATUS_ADDR (IPA_DEBUG_REG_BASE + \
-						0x00000750)
-#define HWIO_IPA_ACKMNGR_SW_ACCESS_STATUS_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						0x00000750)
-#define HWIO_IPA_ACKMNGR_SW_ACCESS_STATUS_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						0x00000750)
-#define HWIO_IPA_PROD_ACKMNGR_CONTROL_CFG_0_ADDR (IPA_DEBUG_REG_BASE + \
-						  0x00000754)
-#define HWIO_IPA_PROD_ACKMNGR_CONTROL_CFG_0_PHYS (IPA_DEBUG_REG_BASE_PHYS \
-						  + 0x00000754)
-#define HWIO_IPA_PROD_ACKMNGR_CONTROL_CFG_0_OFFS (IPA_DEBUG_REG_BASE_OFFS \
-						  + 0x00000754)
-#define HWIO_IPA_PROD_ACKMNGR_CONTROL_CFG_1_ADDR (IPA_DEBUG_REG_BASE + \
-						  0x00000758)
-#define HWIO_IPA_PROD_ACKMNGR_CONTROL_CFG_1_PHYS (IPA_DEBUG_REG_BASE_PHYS \
-						  + 0x00000758)
-#define HWIO_IPA_PROD_ACKMNGR_CONTROL_CFG_1_OFFS (IPA_DEBUG_REG_BASE_OFFS \
-						  + 0x00000758)
-#define HWIO_IPA_PROD_ACKMNGR_CONTROL_CFG_2_ADDR (IPA_DEBUG_REG_BASE + \
-						  0x0000075c)
-#define HWIO_IPA_PROD_ACKMNGR_CONTROL_CFG_2_PHYS (IPA_DEBUG_REG_BASE_PHYS \
-						  + 0x0000075c)
-#define HWIO_IPA_PROD_ACKMNGR_CONTROL_CFG_2_OFFS (IPA_DEBUG_REG_BASE_OFFS \
-						  + 0x0000075c)
-#define HWIO_IPA_PROD_ACKMNGR_CONTROL_CFG_3_ADDR (IPA_DEBUG_REG_BASE + \
-						  0x00000760)
-#define HWIO_IPA_PROD_ACKMNGR_CONTROL_CFG_3_PHYS (IPA_DEBUG_REG_BASE_PHYS \
-						  + 0x00000760)
-#define HWIO_IPA_PROD_ACKMNGR_CONTROL_CFG_3_OFFS (IPA_DEBUG_REG_BASE_OFFS \
-						  + 0x00000760)
-#define HWIO_IPA_PROD_ACKMNGR_CONTROL_CFG_4_ADDR (IPA_DEBUG_REG_BASE + \
-						  0x00000764)
-#define HWIO_IPA_PROD_ACKMNGR_CONTROL_CFG_4_PHYS (IPA_DEBUG_REG_BASE_PHYS \
-						  + 0x00000764)
-#define HWIO_IPA_PROD_ACKMNGR_CONTROL_CFG_4_OFFS (IPA_DEBUG_REG_BASE_OFFS \
-						  + 0x00000764)
-#define HWIO_IPA_PROD_ACKMNGR_CONTROL_CFG_5_ADDR (IPA_DEBUG_REG_BASE + \
-						  0x00000768)
-#define HWIO_IPA_PROD_ACKMNGR_CONTROL_CFG_5_PHYS (IPA_DEBUG_REG_BASE_PHYS \
-						  + 0x00000768)
-#define HWIO_IPA_PROD_ACKMNGR_CONTROL_CFG_5_OFFS (IPA_DEBUG_REG_BASE_OFFS \
-						  + 0x00000768)
-#define HWIO_IPA_PROD_ACKMNGR_SW_ACCESS_ACKINJ_CFG_ADDR ( \
-		IPA_DEBUG_REG_BASE + 0x0000076c)
-#define HWIO_IPA_PROD_ACKMNGR_SW_ACCESS_ACKINJ_CFG_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x0000076c)
-#define HWIO_IPA_PROD_ACKMNGR_SW_ACCESS_ACKINJ_CFG_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x0000076c)
-#define HWIO_IPA_PROD_ACKMNGR_SW_ACCESS_ACKUPD_CFG_ADDR ( \
-		IPA_DEBUG_REG_BASE + 0x00000770)
-#define HWIO_IPA_PROD_ACKMNGR_SW_ACCESS_ACKUPD_CFG_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000770)
-#define HWIO_IPA_PROD_ACKMNGR_SW_ACCESS_ACKUPD_CFG_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000770)
-#define HWIO_IPA_PROD_ACKMNGR_SW_ACCESS_CMD_ADDR (IPA_DEBUG_REG_BASE + \
-						  0x00000774)
-#define HWIO_IPA_PROD_ACKMNGR_SW_ACCESS_CMD_PHYS (IPA_DEBUG_REG_BASE_PHYS \
-						  + 0x00000774)
-#define HWIO_IPA_PROD_ACKMNGR_SW_ACCESS_CMD_OFFS (IPA_DEBUG_REG_BASE_OFFS \
-						  + 0x00000774)
-#define HWIO_IPA_PROD_ACKMNGR_SW_ACCESS_STATUS_ADDR (IPA_DEBUG_REG_BASE + \
-						     0x00000778)
-#define HWIO_IPA_PROD_ACKMNGR_SW_ACCESS_STATUS_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000778)
-#define HWIO_IPA_PROD_ACKMNGR_SW_ACCESS_STATUS_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000778)
-#define HWIO_IPA_PROD_ACKMNGR_SW_ACCESS_ACKINJ_CFG1_ADDR ( \
-		IPA_DEBUG_REG_BASE + 0x0000077c)
-#define HWIO_IPA_PROD_ACKMNGR_SW_ACCESS_ACKINJ_CFG1_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x0000077c)
-#define HWIO_IPA_PROD_ACKMNGR_SW_ACCESS_ACKINJ_CFG1_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x0000077c)
-#define HWIO_IPA_SPARE_REG_1_ADDR (IPA_DEBUG_REG_BASE + 0x00000780)
-#define HWIO_IPA_SPARE_REG_1_PHYS (IPA_DEBUG_REG_BASE_PHYS + 0x00000780)
-#define HWIO_IPA_SPARE_REG_1_OFFS (IPA_DEBUG_REG_BASE_OFFS + 0x00000780)
-#define HWIO_IPA_SPARE_REG_1_RMSK 0xffffffff
-#define HWIO_IPA_SPARE_REG_1_ATTR 0x3
-#define HWIO_IPA_SPARE_REG_1_IN in_dword_masked(HWIO_IPA_SPARE_REG_1_ADDR, \
-						HWIO_IPA_SPARE_REG_1_RMSK)
-#define HWIO_IPA_SPARE_REG_1_INM(m) in_dword_masked( \
-		HWIO_IPA_SPARE_REG_1_ADDR, \
-		m)
-#define HWIO_IPA_SPARE_REG_1_OUT(v) out_dword(HWIO_IPA_SPARE_REG_1_ADDR, v)
-#define HWIO_IPA_SPARE_REG_1_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_SPARE_REG_1_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_SPARE_REG_1_IN)
-#define HWIO_IPA_SPARE_REG_1_SPARE_BIT31_BMSK 0x80000000
-#define HWIO_IPA_SPARE_REG_1_SPARE_BIT31_SHFT 0x1f
-#define HWIO_IPA_SPARE_REG_1_SPARE_BIT30_BMSK 0x40000000
-#define HWIO_IPA_SPARE_REG_1_SPARE_BIT30_SHFT 0x1e
-#define HWIO_IPA_SPARE_REG_1_SPARE_ACKINJ_PIPE8_MASK_ENABLE_BMSK \
-	0x20000000
-#define HWIO_IPA_SPARE_REG_1_SPARE_ACKINJ_PIPE8_MASK_ENABLE_SHFT 0x1d
-#define	\
-	HWIO_IPA_SPARE_REG_1_WARB_FORCE_ARB_ROUND_FINISH_SPECIAL_DISABLE_BMSK \
-	0x10000000
-#define	\
-	HWIO_IPA_SPARE_REG_1_WARB_FORCE_ARB_ROUND_FINISH_SPECIAL_DISABLE_SHFT \
-	0x1c
-#define HWIO_IPA_SPARE_REG_1_DCPH_RAM_RD_PREFETCH_DISABLE_BMSK 0x8000000
-#define HWIO_IPA_SPARE_REG_1_DCPH_RAM_RD_PREFETCH_DISABLE_SHFT 0x1b
-#define HWIO_IPA_SPARE_REG_1_RAM_SLAVEWAY_ACCESS_PROTECTION_DISABLE_BMSK \
-	0x4000000
-#define HWIO_IPA_SPARE_REG_1_RAM_SLAVEWAY_ACCESS_PROTECTION_DISABLE_SHFT \
-	0x1a
-#define HWIO_IPA_SPARE_REG_1_SPARE_BIT25_BMSK 0x2000000
-#define HWIO_IPA_SPARE_REG_1_SPARE_BIT25_SHFT 0x19
-#define HWIO_IPA_SPARE_REG_1_SPARE_BIT24_BMSK 0x1000000
-#define HWIO_IPA_SPARE_REG_1_SPARE_BIT24_SHFT 0x18
-#define HWIO_IPA_SPARE_REG_1_SPARE_BIT23_BMSK 0x800000
-#define HWIO_IPA_SPARE_REG_1_SPARE_BIT23_SHFT 0x17
-#define HWIO_IPA_SPARE_REG_1_BAM_IDLE_IN_IPA_MISC_CGC_EN_BMSK 0x400000
-#define HWIO_IPA_SPARE_REG_1_BAM_IDLE_IN_IPA_MISC_CGC_EN_SHFT 0x16
-#define HWIO_IPA_SPARE_REG_1_GSI_IF_OUT_OF_BUF_STOP_RESET_MASK_ENABLE_BMSK \
-	0x200000
-#define HWIO_IPA_SPARE_REG_1_GSI_IF_OUT_OF_BUF_STOP_RESET_MASK_ENABLE_SHFT \
-	0x15
-#define HWIO_IPA_SPARE_REG_1_REVERT_WARB_FIX_BMSK 0x100000
-#define HWIO_IPA_SPARE_REG_1_REVERT_WARB_FIX_SHFT 0x14
-#define HWIO_IPA_SPARE_REG_1_SPARE_BIT19_BMSK 0x80000
-#define HWIO_IPA_SPARE_REG_1_SPARE_BIT19_SHFT 0x13
-#define HWIO_IPA_SPARE_REG_1_RX_STALL_ON_GEN_DEAGGR_ERROR_BMSK 0x40000
-#define HWIO_IPA_SPARE_REG_1_RX_STALL_ON_GEN_DEAGGR_ERROR_SHFT 0x12
-#define HWIO_IPA_SPARE_REG_1_RX_STALL_ON_MBIM_DEAGGR_ERROR_BMSK 0x20000
-#define HWIO_IPA_SPARE_REG_1_RX_STALL_ON_MBIM_DEAGGR_ERROR_SHFT 0x11
-#define HWIO_IPA_SPARE_REG_1_QMB_RAM_RD_CACHE_DISABLE_BMSK 0x10000
-#define HWIO_IPA_SPARE_REG_1_QMB_RAM_RD_CACHE_DISABLE_SHFT 0x10
-#define	\
-	HWIO_IPA_SPARE_REG_1_RX_CMDQ_SPLITTER_CMDQ_PENDING_MUX_DISABLE_BMSK \
-	0x8000
-#define	\
-	HWIO_IPA_SPARE_REG_1_RX_CMDQ_SPLITTER_CMDQ_PENDING_MUX_DISABLE_SHFT \
-	0xf
-#define	\
-	HWIO_IPA_SPARE_REG_1_FRAG_MNGR_FAIRNESS_EVICTION_ON_CONSTRUCTING_BMSK \
-	0x4000
-#define	\
-	HWIO_IPA_SPARE_REG_1_FRAG_MNGR_FAIRNESS_EVICTION_ON_CONSTRUCTING_SHFT \
-	0xe
-#define HWIO_IPA_SPARE_REG_1_TX_BLOCK_AGGR_QUERY_ON_HOLB_PACKET_BMSK \
-	0x2000
-#define HWIO_IPA_SPARE_REG_1_TX_BLOCK_AGGR_QUERY_ON_HOLB_PACKET_SHFT 0xd
-#define HWIO_IPA_SPARE_REG_1_SPARE_BIT12_BMSK 0x1000
-#define HWIO_IPA_SPARE_REG_1_SPARE_BIT12_SHFT 0xc
-#define HWIO_IPA_SPARE_REG_1_TX_GIVES_SSPND_ACK_ON_OPEN_AGGR_FRAME_BMSK	\
-	0x800
-#define HWIO_IPA_SPARE_REG_1_TX_GIVES_SSPND_ACK_ON_OPEN_AGGR_FRAME_SHFT	\
-	0xb
-#define HWIO_IPA_SPARE_REG_1_ACL_DISPATCHER_PKT_CHECK_DISABLE_BMSK 0x400
-#define HWIO_IPA_SPARE_REG_1_ACL_DISPATCHER_PKT_CHECK_DISABLE_SHFT 0xa
-#define HWIO_IPA_SPARE_REG_1_SPARE_BIT8_BMSK 0x100
-#define HWIO_IPA_SPARE_REG_1_SPARE_BIT8_SHFT 0x8
-#define HWIO_IPA_SPARE_REG_1_ACL_DISPATCHER_FRAG_NOTIF_CHECK_DISABLE_BMSK \
-	0x40
-#define HWIO_IPA_SPARE_REG_1_ACL_DISPATCHER_FRAG_NOTIF_CHECK_DISABLE_SHFT \
-	0x6
-#define HWIO_IPA_SPARE_REG_1_ACL_INORDER_MULTI_DISABLE_BMSK 0x20
-#define HWIO_IPA_SPARE_REG_1_ACL_INORDER_MULTI_DISABLE_SHFT 0x5
-#define HWIO_IPA_SPARE_REG_1_SPARE_BIT4_BMSK 0x10
-#define HWIO_IPA_SPARE_REG_1_SPARE_BIT4_SHFT 0x4
-#define HWIO_IPA_SPARE_REG_1_SPARE_BIT3_BMSK 0x8
-#define HWIO_IPA_SPARE_REG_1_SPARE_BIT3_SHFT 0x3
-#define HWIO_IPA_SPARE_REG_1_GENQMB_AOOOWR_BMSK 0x4
-#define HWIO_IPA_SPARE_REG_1_GENQMB_AOOOWR_SHFT 0x2
-#define HWIO_IPA_SPARE_REG_1_SPARE_BIT1_BMSK 0x2
-#define HWIO_IPA_SPARE_REG_1_SPARE_BIT1_SHFT 0x1
-#define HWIO_IPA_SPARE_REG_1_SPARE_BIT0_BMSK 0x1
-#define HWIO_IPA_SPARE_REG_1_SPARE_BIT0_SHFT 0x0
-#define HWIO_IPA_SPARE_REG_2_ADDR (IPA_DEBUG_REG_BASE + 0x00000784)
-#define HWIO_IPA_SPARE_REG_2_PHYS (IPA_DEBUG_REG_BASE_PHYS + 0x00000784)
-#define HWIO_IPA_SPARE_REG_2_OFFS (IPA_DEBUG_REG_BASE_OFFS + 0x00000784)
-#define HWIO_IPA_SPARE_REG_2_RMSK 0xffffffff
-#define HWIO_IPA_SPARE_REG_2_ATTR 0x3
-#define HWIO_IPA_SPARE_REG_2_IN in_dword_masked(HWIO_IPA_SPARE_REG_2_ADDR, \
-						HWIO_IPA_SPARE_REG_2_RMSK)
-#define HWIO_IPA_SPARE_REG_2_INM(m) in_dword_masked( \
-		HWIO_IPA_SPARE_REG_2_ADDR, \
-		m)
-#define HWIO_IPA_SPARE_REG_2_OUT(v) out_dword(HWIO_IPA_SPARE_REG_2_ADDR, v)
-#define HWIO_IPA_SPARE_REG_2_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_SPARE_REG_2_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_SPARE_REG_2_IN)
-#define HWIO_IPA_SPARE_REG_2_SPARE_BITS_BMSK 0xfffffffc
-#define HWIO_IPA_SPARE_REG_2_SPARE_BITS_SHFT 0x2
-#define	\
-	HWIO_IPA_SPARE_REG_2_CMDQ_SPLIT_NOT_WAIT_DATA_DESC_PRIOR_HDR_PUSH_BMSK \
-	0x2
-#define	\
-	HWIO_IPA_SPARE_REG_2_CMDQ_SPLIT_NOT_WAIT_DATA_DESC_PRIOR_HDR_PUSH_SHFT \
-	0x1
-#define HWIO_IPA_SPARE_REG_2_TX_BRESP_INJ_WITH_FLOP_BMSK 0x1
-#define HWIO_IPA_SPARE_REG_2_TX_BRESP_INJ_WITH_FLOP_SHFT 0x0
-#define HWIO_IPA_ENDP_GSI_CFG1_n_ADDR(n) (IPA_DEBUG_REG_BASE + \
-					  0x00000794 + 0x4 * (n))
-#define HWIO_IPA_ENDP_GSI_CFG1_n_PHYS(n) (IPA_DEBUG_REG_BASE_PHYS + \
-					  0x00000794 + 0x4 * (n))
-#define HWIO_IPA_ENDP_GSI_CFG1_n_OFFS(n) (IPA_DEBUG_REG_BASE_OFFS + \
-					  0x00000794 + 0x4 * (n))
-#define HWIO_IPA_ENDP_GSI_CFG1_n_RMSK 0x80010000
-#define HWIO_IPA_ENDP_GSI_CFG1_n_MAXn 30
-#define HWIO_IPA_ENDP_GSI_CFG1_n_ATTR 0x3
-#define HWIO_IPA_ENDP_GSI_CFG1_n_INI(n) in_dword_masked( \
-		HWIO_IPA_ENDP_GSI_CFG1_n_ADDR(n), \
-		HWIO_IPA_ENDP_GSI_CFG1_n_RMSK)
-#define HWIO_IPA_ENDP_GSI_CFG1_n_INMI(n, mask) in_dword_masked(	\
-		HWIO_IPA_ENDP_GSI_CFG1_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_ENDP_GSI_CFG1_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_ENDP_GSI_CFG1_n_ADDR(n), \
-		val)
-#define HWIO_IPA_ENDP_GSI_CFG1_n_OUTMI(n, mask, val) out_dword_masked_ns( \
-		HWIO_IPA_ENDP_GSI_CFG1_n_ADDR(n), \
-		mask, \
-		val, \
-		HWIO_IPA_ENDP_GSI_CFG1_n_INI(n))
-#define HWIO_IPA_ENDP_GSI_CFG1_n_INIT_ENDP_BMSK 0x80000000
-#define HWIO_IPA_ENDP_GSI_CFG1_n_INIT_ENDP_SHFT 0x1f
-#define HWIO_IPA_ENDP_GSI_CFG1_n_ENDP_EN_BMSK 0x10000
-#define HWIO_IPA_ENDP_GSI_CFG1_n_ENDP_EN_SHFT 0x10
-#define HWIO_IPA_GSI_IPA_IF_TLV_OUT_GENERATOR_1_ADDR (IPA_DEBUG_REG_BASE + \
-						      0x00000908)
-#define HWIO_IPA_GSI_IPA_IF_TLV_OUT_GENERATOR_1_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000908)
-#define HWIO_IPA_GSI_IPA_IF_TLV_OUT_GENERATOR_1_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000908)
-#define HWIO_IPA_GSI_IPA_IF_TLV_OUT_GENERATOR_2_ADDR (IPA_DEBUG_REG_BASE + \
-						      0x0000090c)
-#define HWIO_IPA_GSI_IPA_IF_TLV_OUT_GENERATOR_2_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x0000090c)
-#define HWIO_IPA_GSI_IPA_IF_TLV_OUT_GENERATOR_2_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x0000090c)
-#define HWIO_IPA_GSI_IPA_IF_TLV_OUT_GENERATOR_3_ADDR (IPA_DEBUG_REG_BASE + \
-						      0x00000910)
-#define HWIO_IPA_GSI_IPA_IF_TLV_OUT_GENERATOR_3_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000910)
-#define HWIO_IPA_GSI_IPA_IF_TLV_OUT_GENERATOR_3_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000910)
-#define HWIO_IPA_GSI_IPA_IF_TLV_OUT_GENERATOR_CTRL_ADDR ( \
-		IPA_DEBUG_REG_BASE + 0x00000914)
-#define HWIO_IPA_GSI_IPA_IF_TLV_OUT_GENERATOR_CTRL_PHYS ( \
-		IPA_DEBUG_REG_BASE_PHYS + 0x00000914)
-#define HWIO_IPA_GSI_IPA_IF_TLV_OUT_GENERATOR_CTRL_OFFS ( \
-		IPA_DEBUG_REG_BASE_OFFS + 0x00000914)
-#define HWIO_IPA_GSI_IPA_IF_TLV_IN_RDY_ADDR (IPA_DEBUG_REG_BASE + \
-					     0x00000918)
-#define HWIO_IPA_GSI_IPA_IF_TLV_IN_RDY_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x00000918)
-#define HWIO_IPA_GSI_IPA_IF_TLV_IN_RDY_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x00000918)
-#define HWIO_IPA_GSI_IPA_IF_TLV_IN_DATA_1_ADDR (IPA_DEBUG_REG_BASE + \
-						0x0000091c)
-#define HWIO_IPA_GSI_IPA_IF_TLV_IN_DATA_1_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						0x0000091c)
-#define HWIO_IPA_GSI_IPA_IF_TLV_IN_DATA_1_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						0x0000091c)
-#define HWIO_IPA_GSI_IPA_IF_TLV_IN_DATA_2_ADDR (IPA_DEBUG_REG_BASE + \
-						0x00000920)
-#define HWIO_IPA_GSI_IPA_IF_TLV_IN_DATA_2_PHYS (IPA_DEBUG_REG_BASE_PHYS + \
-						0x00000920)
-#define HWIO_IPA_GSI_IPA_IF_TLV_IN_DATA_2_OFFS (IPA_DEBUG_REG_BASE_OFFS + \
-						0x00000920)
-#define HWIO_IPA_ENDP_GSI_CFG_TLV_n_ADDR(n) (IPA_DEBUG_REG_BASE + \
-					     0x00000924 + 0x4 * (n))
-#define HWIO_IPA_ENDP_GSI_CFG_TLV_n_PHYS(n) (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x00000924 + 0x4 * (n))
-#define HWIO_IPA_ENDP_GSI_CFG_TLV_n_OFFS(n) (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x00000924 + 0x4 * (n))
-#define HWIO_IPA_ENDP_GSI_CFG_TLV_n_RMSK 0xffffff
-#define HWIO_IPA_ENDP_GSI_CFG_TLV_n_MAXn 30
-#define HWIO_IPA_ENDP_GSI_CFG_TLV_n_ATTR 0x3
-#define HWIO_IPA_ENDP_GSI_CFG_TLV_n_INI(n) in_dword_masked( \
-		HWIO_IPA_ENDP_GSI_CFG_TLV_n_ADDR(n), \
-		HWIO_IPA_ENDP_GSI_CFG_TLV_n_RMSK)
-#define HWIO_IPA_ENDP_GSI_CFG_TLV_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_ENDP_GSI_CFG_TLV_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_ENDP_GSI_CFG_TLV_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_ENDP_GSI_CFG_TLV_n_ADDR(n), \
-		val)
-#define HWIO_IPA_ENDP_GSI_CFG_TLV_n_OUTMI(n, mask, \
-					  val) out_dword_masked_ns( \
-		HWIO_IPA_ENDP_GSI_CFG_TLV_n_ADDR( \
-			n), \
-		mask, \
-		val, \
-		HWIO_IPA_ENDP_GSI_CFG_TLV_n_INI(n))
-#define HWIO_IPA_ENDP_GSI_CFG_TLV_n_FIFO_SIZE_BMSK 0xff0000
-#define HWIO_IPA_ENDP_GSI_CFG_TLV_n_FIFO_SIZE_SHFT 0x10
-#define HWIO_IPA_ENDP_GSI_CFG_TLV_n_FIFO_BASE_ADDR_BMSK 0xffff
-#define HWIO_IPA_ENDP_GSI_CFG_TLV_n_FIFO_BASE_ADDR_SHFT 0x0
-#define HWIO_IPA_ENDP_GSI_CFG_AOS_n_ADDR(n) (IPA_DEBUG_REG_BASE + \
-					     0x000009a8 + 0x4 * (n))
-#define HWIO_IPA_ENDP_GSI_CFG_AOS_n_PHYS(n) (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x000009a8 + 0x4 * (n))
-#define HWIO_IPA_ENDP_GSI_CFG_AOS_n_OFFS(n) (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x000009a8 + 0x4 * (n))
-#define HWIO_IPA_ENDP_GSI_CFG_AOS_n_RMSK 0xffffff
-#define HWIO_IPA_ENDP_GSI_CFG_AOS_n_MAXn 30
-#define HWIO_IPA_ENDP_GSI_CFG_AOS_n_ATTR 0x3
-#define HWIO_IPA_ENDP_GSI_CFG_AOS_n_INI(n) in_dword_masked( \
-		HWIO_IPA_ENDP_GSI_CFG_AOS_n_ADDR(n), \
-		HWIO_IPA_ENDP_GSI_CFG_AOS_n_RMSK)
-#define HWIO_IPA_ENDP_GSI_CFG_AOS_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_ENDP_GSI_CFG_AOS_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_ENDP_GSI_CFG_AOS_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_ENDP_GSI_CFG_AOS_n_ADDR(n), \
-		val)
-#define HWIO_IPA_ENDP_GSI_CFG_AOS_n_OUTMI(n, mask, \
-					  val) out_dword_masked_ns( \
-		HWIO_IPA_ENDP_GSI_CFG_AOS_n_ADDR( \
-			n), \
-		mask, \
-		val, \
-		HWIO_IPA_ENDP_GSI_CFG_AOS_n_INI(n))
-#define HWIO_IPA_ENDP_GSI_CFG_AOS_n_FIFO_SIZE_BMSK 0xff0000
-#define HWIO_IPA_ENDP_GSI_CFG_AOS_n_FIFO_SIZE_SHFT 0x10
-#define HWIO_IPA_ENDP_GSI_CFG_AOS_n_FIFO_BASE_ADDR_BMSK 0xffff
-#define HWIO_IPA_ENDP_GSI_CFG_AOS_n_FIFO_BASE_ADDR_SHFT 0x0
-#define HWIO_IPA_COAL_VP_AOS_FIFO_n_ADDR(n) (IPA_DEBUG_REG_BASE + \
-					     0x00000a60 + 0x4 * (n))
-#define HWIO_IPA_COAL_VP_AOS_FIFO_n_PHYS(n) (IPA_DEBUG_REG_BASE_PHYS + \
-					     0x00000a60 + 0x4 * (n))
-#define HWIO_IPA_COAL_VP_AOS_FIFO_n_OFFS(n) (IPA_DEBUG_REG_BASE_OFFS + \
-					     0x00000a60 + 0x4 * (n))
-#define HWIO_IPA_CTXH_CTRL_ADDR (IPA_DEBUG_REG_BASE + 0x00000afc)
-#define HWIO_IPA_CTXH_CTRL_PHYS (IPA_DEBUG_REG_BASE_PHYS + 0x00000afc)
-#define HWIO_IPA_CTXH_CTRL_OFFS (IPA_DEBUG_REG_BASE_OFFS + 0x00000afc)
-#define HWIO_IPA_CTXH_CTRL_RMSK 0x8000000f
-#define HWIO_IPA_CTXH_CTRL_ATTR 0x3
-#define HWIO_IPA_CTXH_CTRL_IN in_dword_masked(HWIO_IPA_CTXH_CTRL_ADDR, \
-					      HWIO_IPA_CTXH_CTRL_RMSK)
-#define HWIO_IPA_CTXH_CTRL_INM(m) in_dword_masked(HWIO_IPA_CTXH_CTRL_ADDR, \
-						  m)
-#define HWIO_IPA_CTXH_CTRL_OUT(v) out_dword(HWIO_IPA_CTXH_CTRL_ADDR, v)
-#define HWIO_IPA_CTXH_CTRL_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_CTXH_CTRL_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_CTXH_CTRL_IN)
-#define HWIO_IPA_CTXH_CTRL_CTXH_LOCK_BMSK 0x80000000
-#define HWIO_IPA_CTXH_CTRL_CTXH_LOCK_SHFT 0x1f
-#define HWIO_IPA_CTXH_CTRL_CTXH_LOCK_ID_BMSK 0xf
-#define HWIO_IPA_CTXH_CTRL_CTXH_LOCK_ID_SHFT 0x0
-#define HWIO_IPA_CTX_ID_m_CTX_NUM_n_ADDR(m, n) (IPA_DEBUG_REG_BASE + \
-						0x00000b00 + 0x80 * (m) + \
-						0x4 * (n))
-#define HWIO_IPA_CTX_ID_m_CTX_NUM_n_PHYS(m, n) (IPA_DEBUG_REG_BASE_PHYS + \
-						0x00000b00 + 0x80 * (m) + \
-						0x4 * (n))
-#define HWIO_IPA_CTX_ID_m_CTX_NUM_n_OFFS(m, n) (IPA_DEBUG_REG_BASE_OFFS + \
-						0x00000b00 + 0x80 * (m) + \
-						0x4 * (n))
-#define IPA_EE_REG_BASE (IPA_0_IPA_WRAPPER_BASE + 0x00043000)
-#define IPA_EE_REG_BASE_PHYS (IPA_0_IPA_WRAPPER_BASE_PHYS + 0x00043000)
-#define IPA_EE_REG_BASE_OFFS 0x00043000
-#define HWIO_IPA_IRQ_STTS_EE_n_ADDR(n) (IPA_EE_REG_BASE + 0x00000008 + \
-					0x1000 * (n))
-#define HWIO_IPA_IRQ_STTS_EE_n_PHYS(n) (IPA_EE_REG_BASE_PHYS + \
-					0x00000008 + 0x1000 * (n))
-#define HWIO_IPA_IRQ_STTS_EE_n_OFFS(n) (IPA_EE_REG_BASE_OFFS + \
-					0x00000008 + 0x1000 * (n))
-#define HWIO_IPA_IRQ_EN_EE_n_ADDR(n) (IPA_EE_REG_BASE + 0x0000000c + \
-				      0x1000 * (n))
-#define HWIO_IPA_IRQ_EN_EE_n_PHYS(n) (IPA_EE_REG_BASE_PHYS + 0x0000000c + \
-				      0x1000 * (n))
-#define HWIO_IPA_IRQ_EN_EE_n_OFFS(n) (IPA_EE_REG_BASE_OFFS + 0x0000000c + \
-				      0x1000 * (n))
-#define HWIO_IPA_IRQ_EN_EE_n_RMSK 0x7bffffd
-#define HWIO_IPA_IRQ_EN_EE_n_MAXn 3
-#define HWIO_IPA_IRQ_EN_EE_n_ATTR 0x3
-#define HWIO_IPA_IRQ_EN_EE_n_INI(n) in_dword_masked( \
-		HWIO_IPA_IRQ_EN_EE_n_ADDR(n), \
-		HWIO_IPA_IRQ_EN_EE_n_RMSK)
-#define HWIO_IPA_IRQ_EN_EE_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_IRQ_EN_EE_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_IRQ_EN_EE_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_IRQ_EN_EE_n_ADDR(n), \
-		val)
-#define HWIO_IPA_IRQ_EN_EE_n_OUTMI(n, mask, val) out_dword_masked_ns( \
-		HWIO_IPA_IRQ_EN_EE_n_ADDR(n), \
-		mask, \
-		val, \
-		HWIO_IPA_IRQ_EN_EE_n_INI(n))
-#define HWIO_IPA_IRQ_EN_EE_n_TLV_LEN_MIN_DSM_IRQ_EN_BMSK 0x4000000
-#define HWIO_IPA_IRQ_EN_EE_n_TLV_LEN_MIN_DSM_IRQ_EN_SHFT 0x1a
-#define HWIO_IPA_IRQ_EN_EE_n_GSI_UC_IRQ_EN_BMSK 0x2000000
-#define HWIO_IPA_IRQ_EN_EE_n_GSI_UC_IRQ_EN_SHFT 0x19
-#define HWIO_IPA_IRQ_EN_EE_n_GSI_IPA_IF_TLV_RCVD_IRQ_EN_BMSK 0x1000000
-#define HWIO_IPA_IRQ_EN_EE_n_GSI_IPA_IF_TLV_RCVD_IRQ_EN_SHFT 0x18
-#define HWIO_IPA_IRQ_EN_EE_n_GSI_EE_IRQ_EN_BMSK 0x800000
-#define HWIO_IPA_IRQ_EN_EE_n_GSI_EE_IRQ_EN_SHFT 0x17
-#define HWIO_IPA_IRQ_EN_EE_n_UCP_IRQ_EN_BMSK 0x200000
-#define HWIO_IPA_IRQ_EN_EE_n_UCP_IRQ_EN_SHFT 0x15
-#define HWIO_IPA_IRQ_EN_EE_n_PIPE_RED_MARKER_ABOVE_IRQ_EN_BMSK 0x100000
-#define HWIO_IPA_IRQ_EN_EE_n_PIPE_RED_MARKER_ABOVE_IRQ_EN_SHFT 0x14
-#define HWIO_IPA_IRQ_EN_EE_n_PIPE_YELLOW_MARKER_ABOVE_IRQ_EN_BMSK 0x80000
-#define HWIO_IPA_IRQ_EN_EE_n_PIPE_YELLOW_MARKER_ABOVE_IRQ_EN_SHFT 0x13
-#define HWIO_IPA_IRQ_EN_EE_n_PIPE_RED_MARKER_BELOW_IRQ_EN_BMSK 0x40000
-#define HWIO_IPA_IRQ_EN_EE_n_PIPE_RED_MARKER_BELOW_IRQ_EN_SHFT 0x12
-#define HWIO_IPA_IRQ_EN_EE_n_PIPE_YELLOW_MARKER_BELOW_IRQ_EN_BMSK 0x20000
-#define HWIO_IPA_IRQ_EN_EE_n_PIPE_YELLOW_MARKER_BELOW_IRQ_EN_SHFT 0x11
-#define HWIO_IPA_IRQ_EN_EE_n_BAM_GSI_IDLE_IRQ_EN_BMSK 0x10000
-#define HWIO_IPA_IRQ_EN_EE_n_BAM_GSI_IDLE_IRQ_EN_SHFT 0x10
-#define HWIO_IPA_IRQ_EN_EE_n_TX_HOLB_DROP_IRQ_EN_BMSK 0x8000
-#define HWIO_IPA_IRQ_EN_EE_n_TX_HOLB_DROP_IRQ_EN_SHFT 0xf
-#define HWIO_IPA_IRQ_EN_EE_n_TX_SUSPEND_IRQ_EN_BMSK 0x4000
-#define HWIO_IPA_IRQ_EN_EE_n_TX_SUSPEND_IRQ_EN_SHFT 0xe
-#define HWIO_IPA_IRQ_EN_EE_n_PROC_ERR_IRQ_EN_BMSK 0x2000
-#define HWIO_IPA_IRQ_EN_EE_n_PROC_ERR_IRQ_EN_SHFT 0xd
-#define HWIO_IPA_IRQ_EN_EE_n_STEP_MODE_IRQ_EN_BMSK 0x1000
-#define HWIO_IPA_IRQ_EN_EE_n_STEP_MODE_IRQ_EN_SHFT 0xc
-#define HWIO_IPA_IRQ_EN_EE_n_TX_ERR_IRQ_EN_BMSK 0x800
-#define HWIO_IPA_IRQ_EN_EE_n_TX_ERR_IRQ_EN_SHFT 0xb
-#define HWIO_IPA_IRQ_EN_EE_n_DEAGGR_ERR_IRQ_EN_BMSK 0x400
-#define HWIO_IPA_IRQ_EN_EE_n_DEAGGR_ERR_IRQ_EN_SHFT 0xa
-#define HWIO_IPA_IRQ_EN_EE_n_RX_ERR_IRQ_EN_BMSK 0x200
-#define HWIO_IPA_IRQ_EN_EE_n_RX_ERR_IRQ_EN_SHFT 0x9
-#define HWIO_IPA_IRQ_EN_EE_n_PROC_TO_UC_ACK_Q_NOT_EMPTY_IRQ_EN_BMSK 0x100
-#define HWIO_IPA_IRQ_EN_EE_n_PROC_TO_UC_ACK_Q_NOT_EMPTY_IRQ_EN_SHFT 0x8
-#define HWIO_IPA_IRQ_EN_EE_n_UC_RX_CMD_Q_NOT_FULL_IRQ_EN_BMSK 0x80
-#define HWIO_IPA_IRQ_EN_EE_n_UC_RX_CMD_Q_NOT_FULL_IRQ_EN_SHFT 0x7
-#define HWIO_IPA_IRQ_EN_EE_n_UC_IN_Q_NOT_EMPTY_IRQ_EN_BMSK 0x40
-#define HWIO_IPA_IRQ_EN_EE_n_UC_IN_Q_NOT_EMPTY_IRQ_EN_SHFT 0x6
-#define HWIO_IPA_IRQ_EN_EE_n_UC_IRQ_3_IRQ_EN_BMSK 0x20
-#define HWIO_IPA_IRQ_EN_EE_n_UC_IRQ_3_IRQ_EN_SHFT 0x5
-#define HWIO_IPA_IRQ_EN_EE_n_UC_IRQ_2_IRQ_EN_BMSK 0x10
-#define HWIO_IPA_IRQ_EN_EE_n_UC_IRQ_2_IRQ_EN_SHFT 0x4
-#define HWIO_IPA_IRQ_EN_EE_n_UC_IRQ_1_IRQ_EN_BMSK 0x8
-#define HWIO_IPA_IRQ_EN_EE_n_UC_IRQ_1_IRQ_EN_SHFT 0x3
-#define HWIO_IPA_IRQ_EN_EE_n_UC_IRQ_0_IRQ_EN_BMSK 0x4
-#define HWIO_IPA_IRQ_EN_EE_n_UC_IRQ_0_IRQ_EN_SHFT 0x2
-#define HWIO_IPA_IRQ_EN_EE_n_BAD_SNOC_ACCESS_IRQ_EN_BMSK 0x1
-#define HWIO_IPA_IRQ_EN_EE_n_BAD_SNOC_ACCESS_IRQ_EN_SHFT 0x0
-#define HWIO_IPA_IRQ_CLR_EE_n_ADDR(n) (IPA_EE_REG_BASE + 0x00000010 + \
-				       0x1000 * (n))
-#define HWIO_IPA_IRQ_CLR_EE_n_PHYS(n) (IPA_EE_REG_BASE_PHYS + 0x00000010 + \
-				       0x1000 * (n))
-#define HWIO_IPA_IRQ_CLR_EE_n_OFFS(n) (IPA_EE_REG_BASE_OFFS + 0x00000010 + \
-				       0x1000 * (n))
-#define HWIO_IPA_SNOC_FEC_EE_n_ADDR(n) (IPA_EE_REG_BASE + 0x00000018 + \
-					0x1000 * (n))
-#define HWIO_IPA_SNOC_FEC_EE_n_PHYS(n) (IPA_EE_REG_BASE_PHYS + \
-					0x00000018 + 0x1000 * (n))
-#define HWIO_IPA_SNOC_FEC_EE_n_OFFS(n) (IPA_EE_REG_BASE_OFFS + \
-					0x00000018 + 0x1000 * (n))
-#define HWIO_IPA_SNOC_FEC_EE_n_RMSK 0x8000f1ff
-#define HWIO_IPA_SNOC_FEC_EE_n_MAXn 3
-#define HWIO_IPA_SNOC_FEC_EE_n_ATTR 0x1
-#define HWIO_IPA_SNOC_FEC_EE_n_INI(n) in_dword_masked( \
-		HWIO_IPA_SNOC_FEC_EE_n_ADDR(n),	\
-		HWIO_IPA_SNOC_FEC_EE_n_RMSK)
-#define HWIO_IPA_SNOC_FEC_EE_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_SNOC_FEC_EE_n_ADDR(n),	\
-		mask)
-#define HWIO_IPA_SNOC_FEC_EE_n_READ_NOT_WRITE_BMSK 0x80000000
-#define HWIO_IPA_SNOC_FEC_EE_n_READ_NOT_WRITE_SHFT 0x1f
-#define HWIO_IPA_SNOC_FEC_EE_n_TID_BMSK 0xf000
-#define HWIO_IPA_SNOC_FEC_EE_n_TID_SHFT 0xc
-#define HWIO_IPA_SNOC_FEC_EE_n_QMB_INDEX_BMSK 0x100
-#define HWIO_IPA_SNOC_FEC_EE_n_QMB_INDEX_SHFT 0x8
-#define HWIO_IPA_SNOC_FEC_EE_n_CLIENT_BMSK 0xff
-#define HWIO_IPA_SNOC_FEC_EE_n_CLIENT_SHFT 0x0
-#define HWIO_IPA_IRQ_EE_UC_n_ADDR(n) (IPA_EE_REG_BASE + 0x0000001c + \
-				      0x1000 * (n))
-#define HWIO_IPA_IRQ_EE_UC_n_PHYS(n) (IPA_EE_REG_BASE_PHYS + 0x0000001c + \
-				      0x1000 * (n))
-#define HWIO_IPA_IRQ_EE_UC_n_OFFS(n) (IPA_EE_REG_BASE_OFFS + 0x0000001c + \
-				      0x1000 * (n))
-#define HWIO_IPA_FEC_ADDR_EE_n_ADDR(n) (IPA_EE_REG_BASE + 0x00000020 + \
-					0x1000 * (n))
-#define HWIO_IPA_FEC_ADDR_EE_n_PHYS(n) (IPA_EE_REG_BASE_PHYS + \
-					0x00000020 + 0x1000 * (n))
-#define HWIO_IPA_FEC_ADDR_EE_n_OFFS(n) (IPA_EE_REG_BASE_OFFS + \
-					0x00000020 + 0x1000 * (n))
-#define HWIO_IPA_FEC_ADDR_EE_n_RMSK 0xffffffff
-#define HWIO_IPA_FEC_ADDR_EE_n_MAXn 3
-#define HWIO_IPA_FEC_ADDR_EE_n_ATTR 0x1
-#define HWIO_IPA_FEC_ADDR_EE_n_INI(n) in_dword_masked( \
-		HWIO_IPA_FEC_ADDR_EE_n_ADDR(n),	\
-		HWIO_IPA_FEC_ADDR_EE_n_RMSK)
-#define HWIO_IPA_FEC_ADDR_EE_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_FEC_ADDR_EE_n_ADDR(n),	\
-		mask)
-#define HWIO_IPA_FEC_ADDR_EE_n_ADDR_BMSK 0xffffffff
-#define HWIO_IPA_FEC_ADDR_EE_n_ADDR_SHFT 0x0
-#define HWIO_IPA_FEC_ADDR_MSB_EE_n_ADDR(n) (IPA_EE_REG_BASE + 0x00000024 + \
-					    0x1000 * (n))
-#define HWIO_IPA_FEC_ADDR_MSB_EE_n_PHYS(n) (IPA_EE_REG_BASE_PHYS + \
-					    0x00000024 + 0x1000 * (n))
-#define HWIO_IPA_FEC_ADDR_MSB_EE_n_OFFS(n) (IPA_EE_REG_BASE_OFFS + \
-					    0x00000024 + 0x1000 * (n))
-#define HWIO_IPA_FEC_ATTR_EE_n_ADDR(n) (IPA_EE_REG_BASE + 0x00000028 + \
-					0x1000 * (n))
-#define HWIO_IPA_FEC_ATTR_EE_n_PHYS(n) (IPA_EE_REG_BASE_PHYS + \
-					0x00000028 + 0x1000 * (n))
-#define HWIO_IPA_FEC_ATTR_EE_n_OFFS(n) (IPA_EE_REG_BASE_OFFS + \
-					0x00000028 + 0x1000 * (n))
-#define HWIO_IPA_FEC_ATTR_EE_n_RMSK 0xffffffff
-#define HWIO_IPA_FEC_ATTR_EE_n_MAXn 3
-#define HWIO_IPA_FEC_ATTR_EE_n_ATTR 0x1
-#define HWIO_IPA_FEC_ATTR_EE_n_INI(n) in_dword_masked( \
-		HWIO_IPA_FEC_ATTR_EE_n_ADDR(n),	\
-		HWIO_IPA_FEC_ATTR_EE_n_RMSK)
-#define HWIO_IPA_FEC_ATTR_EE_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_FEC_ATTR_EE_n_ADDR(n),	\
-		mask)
-#define HWIO_IPA_FEC_ATTR_EE_n_ERROR_INFO_BMSK 0xffffffc0
-#define HWIO_IPA_FEC_ATTR_EE_n_ERROR_INFO_SHFT 0x6
-#define HWIO_IPA_FEC_ATTR_EE_n_OPCODE_BMSK 0x3f
-#define HWIO_IPA_FEC_ATTR_EE_n_OPCODE_SHFT 0x0
-#define HWIO_IPA_SUSPEND_IRQ_INFO_EE_n_ADDR(n) (IPA_EE_REG_BASE + \
-						0x00000030 + 0x1000 * (n))
-#define HWIO_IPA_SUSPEND_IRQ_INFO_EE_n_PHYS(n) (IPA_EE_REG_BASE_PHYS + \
-						0x00000030 + 0x1000 * (n))
-#define HWIO_IPA_SUSPEND_IRQ_INFO_EE_n_OFFS(n) (IPA_EE_REG_BASE_OFFS + \
-						0x00000030 + 0x1000 * (n))
-#define HWIO_IPA_SUSPEND_IRQ_INFO_EE_n_RMSK 0x7fffffff
-#define HWIO_IPA_SUSPEND_IRQ_INFO_EE_n_MAXn 3
-#define HWIO_IPA_SUSPEND_IRQ_INFO_EE_n_ATTR 0x1
-#define HWIO_IPA_SUSPEND_IRQ_INFO_EE_n_INI(n) in_dword_masked( \
-		HWIO_IPA_SUSPEND_IRQ_INFO_EE_n_ADDR(n),	\
-		HWIO_IPA_SUSPEND_IRQ_INFO_EE_n_RMSK)
-#define HWIO_IPA_SUSPEND_IRQ_INFO_EE_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_SUSPEND_IRQ_INFO_EE_n_ADDR(n),	\
-		mask)
-#define HWIO_IPA_SUSPEND_IRQ_INFO_EE_n_ENDPOINTS_BMSK 0x7fffffff
-#define HWIO_IPA_SUSPEND_IRQ_INFO_EE_n_ENDPOINTS_SHFT 0x0
-#define HWIO_IPA_SUSPEND_IRQ_EN_EE_n_ADDR(n) (IPA_EE_REG_BASE +	\
-					      0x00000034 + 0x1000 * (n))
-#define HWIO_IPA_SUSPEND_IRQ_EN_EE_n_PHYS(n) (IPA_EE_REG_BASE_PHYS + \
-					      0x00000034 + 0x1000 * (n))
-#define HWIO_IPA_SUSPEND_IRQ_EN_EE_n_OFFS(n) (IPA_EE_REG_BASE_OFFS + \
-					      0x00000034 + 0x1000 * (n))
-#define HWIO_IPA_SUSPEND_IRQ_EN_EE_n_RMSK 0x7fffffff
-#define HWIO_IPA_SUSPEND_IRQ_EN_EE_n_MAXn 3
-#define HWIO_IPA_SUSPEND_IRQ_EN_EE_n_ATTR 0x3
-#define HWIO_IPA_SUSPEND_IRQ_EN_EE_n_INI(n) in_dword_masked( \
-		HWIO_IPA_SUSPEND_IRQ_EN_EE_n_ADDR(n), \
-		HWIO_IPA_SUSPEND_IRQ_EN_EE_n_RMSK)
-#define HWIO_IPA_SUSPEND_IRQ_EN_EE_n_INMI(n, mask) in_dword_masked( \
-		HWIO_IPA_SUSPEND_IRQ_EN_EE_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_SUSPEND_IRQ_EN_EE_n_OUTI(n, val) out_dword( \
-		HWIO_IPA_SUSPEND_IRQ_EN_EE_n_ADDR(n), \
-		val)
-#define HWIO_IPA_SUSPEND_IRQ_EN_EE_n_OUTMI(n, mask, \
-					   val) out_dword_masked_ns( \
-		HWIO_IPA_SUSPEND_IRQ_EN_EE_n_ADDR( \
-			n), \
-		mask, \
-		val, \
-		HWIO_IPA_SUSPEND_IRQ_EN_EE_n_INI(n))
-#define HWIO_IPA_SUSPEND_IRQ_EN_EE_n_ENDPOINTS_BMSK 0x7fffffff
-#define HWIO_IPA_SUSPEND_IRQ_EN_EE_n_ENDPOINTS_SHFT 0x0
-#define HWIO_IPA_SUSPEND_IRQ_CLR_EE_n_ADDR(n) (IPA_EE_REG_BASE + \
-					       0x00000038 + 0x1000 * (n))
-#define HWIO_IPA_SUSPEND_IRQ_CLR_EE_n_PHYS(n) (IPA_EE_REG_BASE_PHYS + \
-					       0x00000038 + 0x1000 * (n))
-#define HWIO_IPA_SUSPEND_IRQ_CLR_EE_n_OFFS(n) (IPA_EE_REG_BASE_OFFS + \
-					       0x00000038 + 0x1000 * (n))
-#define HWIO_IPA_HOLB_DROP_IRQ_INFO_EE_n_ADDR(n) (IPA_EE_REG_BASE + \
-						  0x0000003c + 0x1000 * (n))
-#define HWIO_IPA_HOLB_DROP_IRQ_INFO_EE_n_PHYS(n) (IPA_EE_REG_BASE_PHYS + \
-						  0x0000003c + 0x1000 * (n))
-#define HWIO_IPA_HOLB_DROP_IRQ_INFO_EE_n_OFFS(n) (IPA_EE_REG_BASE_OFFS + \
-						  0x0000003c + 0x1000 * (n))
-#define HWIO_IPA_HOLB_DROP_IRQ_INFO_EE_n_RMSK 0x7fffe000
-#define HWIO_IPA_HOLB_DROP_IRQ_INFO_EE_n_MAXn 3
-#define HWIO_IPA_HOLB_DROP_IRQ_INFO_EE_n_ATTR 0x1
-#define HWIO_IPA_HOLB_DROP_IRQ_INFO_EE_n_INI(n) in_dword_masked( \
-		HWIO_IPA_HOLB_DROP_IRQ_INFO_EE_n_ADDR(n), \
-		HWIO_IPA_HOLB_DROP_IRQ_INFO_EE_n_RMSK)
-#define HWIO_IPA_HOLB_DROP_IRQ_INFO_EE_n_INMI(n, mask) in_dword_masked(	\
-		HWIO_IPA_HOLB_DROP_IRQ_INFO_EE_n_ADDR(n), \
-		mask)
-#define HWIO_IPA_HOLB_DROP_IRQ_INFO_EE_n_ENDPOINTS_BMSK 0x7fffe000
-#define HWIO_IPA_HOLB_DROP_IRQ_INFO_EE_n_ENDPOINTS_SHFT 0xd
-#define HWIO_IPA_HOLB_DROP_IRQ_EN_EE_n_ADDR(n) (IPA_EE_REG_BASE + \
-						0x00000040 + 0x1000 * (n))
-#define HWIO_IPA_HOLB_DROP_IRQ_EN_EE_n_PHYS(n) (IPA_EE_REG_BASE_PHYS + \
-						0x00000040 + 0x1000 * (n))
-#define HWIO_IPA_HOLB_DROP_IRQ_EN_EE_n_OFFS(n) (IPA_EE_REG_BASE_OFFS + \
-						0x00000040 + 0x1000 * (n))
-#define HWIO_IPA_HOLB_DROP_IRQ_CLR_EE_n_ADDR(n) (IPA_EE_REG_BASE + \
-						 0x00000044 + 0x1000 * (n))
-#define HWIO_IPA_HOLB_DROP_IRQ_CLR_EE_n_PHYS(n) (IPA_EE_REG_BASE_PHYS +	\
-						 0x00000044 + 0x1000 * (n))
-#define HWIO_IPA_HOLB_DROP_IRQ_CLR_EE_n_OFFS(n) (IPA_EE_REG_BASE_OFFS +	\
-						 0x00000044 + 0x1000 * (n))
-#define HWIO_IPA_LOG_BUF_STATUS_ADDR_ADDR (IPA_EE_REG_BASE + 0x000010a0)
-#define HWIO_IPA_LOG_BUF_STATUS_ADDR_PHYS (IPA_EE_REG_BASE_PHYS + \
-					   0x000010a0)
-#define HWIO_IPA_LOG_BUF_STATUS_ADDR_OFFS (IPA_EE_REG_BASE_OFFS + \
-					   0x000010a0)
-#define HWIO_IPA_LOG_BUF_STATUS_ADDR_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_STATUS_ADDR_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_STATUS_ADDR_IN in_dword_masked( \
-		HWIO_IPA_LOG_BUF_STATUS_ADDR_ADDR, \
-		HWIO_IPA_LOG_BUF_STATUS_ADDR_RMSK)
-#define HWIO_IPA_LOG_BUF_STATUS_ADDR_INM(m) in_dword_masked( \
-		HWIO_IPA_LOG_BUF_STATUS_ADDR_ADDR, \
-		m)
-#define HWIO_IPA_LOG_BUF_STATUS_ADDR_OUT(v) out_dword( \
-		HWIO_IPA_LOG_BUF_STATUS_ADDR_ADDR, \
-		v)
-#define HWIO_IPA_LOG_BUF_STATUS_ADDR_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_STATUS_ADDR_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_LOG_BUF_STATUS_ADDR_IN)
-#define HWIO_IPA_LOG_BUF_STATUS_ADDR_START_ADDR_BMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_STATUS_ADDR_START_ADDR_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_STATUS_ADDR_MSB_ADDR (IPA_EE_REG_BASE + \
-					       0x000010a4)
-#define HWIO_IPA_LOG_BUF_STATUS_ADDR_MSB_PHYS (IPA_EE_REG_BASE_PHYS + \
-					       0x000010a4)
-#define HWIO_IPA_LOG_BUF_STATUS_ADDR_MSB_OFFS (IPA_EE_REG_BASE_OFFS + \
-					       0x000010a4)
-#define HWIO_IPA_LOG_BUF_STATUS_ADDR_MSB_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_STATUS_ADDR_MSB_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_STATUS_ADDR_MSB_IN in_dword_masked( \
-		HWIO_IPA_LOG_BUF_STATUS_ADDR_MSB_ADDR, \
-		HWIO_IPA_LOG_BUF_STATUS_ADDR_MSB_RMSK)
-#define HWIO_IPA_LOG_BUF_STATUS_ADDR_MSB_INM(m) in_dword_masked( \
-		HWIO_IPA_LOG_BUF_STATUS_ADDR_MSB_ADDR, \
-		m)
-#define HWIO_IPA_LOG_BUF_STATUS_ADDR_MSB_OUT(v) out_dword( \
-		HWIO_IPA_LOG_BUF_STATUS_ADDR_MSB_ADDR, \
-		v)
-#define HWIO_IPA_LOG_BUF_STATUS_ADDR_MSB_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_STATUS_ADDR_MSB_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_LOG_BUF_STATUS_ADDR_MSB_IN)
-#define HWIO_IPA_LOG_BUF_STATUS_ADDR_MSB_START_ADDR_BMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_STATUS_ADDR_MSB_START_ADDR_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_STATUS_WRITE_PTR_ADDR (IPA_EE_REG_BASE + \
-						0x000010a8)
-#define HWIO_IPA_LOG_BUF_STATUS_WRITE_PTR_PHYS (IPA_EE_REG_BASE_PHYS + \
-						0x000010a8)
-#define HWIO_IPA_LOG_BUF_STATUS_WRITE_PTR_OFFS (IPA_EE_REG_BASE_OFFS + \
-						0x000010a8)
-#define HWIO_IPA_LOG_BUF_STATUS_WRITE_PTR_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_STATUS_WRITE_PTR_ATTR 0x1
-#define HWIO_IPA_LOG_BUF_STATUS_WRITE_PTR_IN in_dword_masked( \
-		HWIO_IPA_LOG_BUF_STATUS_WRITE_PTR_ADDR,	\
-		HWIO_IPA_LOG_BUF_STATUS_WRITE_PTR_RMSK)
-#define HWIO_IPA_LOG_BUF_STATUS_WRITE_PTR_INM(m) in_dword_masked( \
-		HWIO_IPA_LOG_BUF_STATUS_WRITE_PTR_ADDR,	\
-		m)
-#define HWIO_IPA_LOG_BUF_STATUS_WRITE_PTR_WRITE_ADDR_BMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_STATUS_WRITE_PTR_WRITE_ADDR_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_STATUS_WRITE_PTR_MSB_ADDR (IPA_EE_REG_BASE + \
-						    0x000010ac)
-#define HWIO_IPA_LOG_BUF_STATUS_WRITE_PTR_MSB_PHYS (IPA_EE_REG_BASE_PHYS + \
-						    0x000010ac)
-#define HWIO_IPA_LOG_BUF_STATUS_WRITE_PTR_MSB_OFFS (IPA_EE_REG_BASE_OFFS + \
-						    0x000010ac)
-#define HWIO_IPA_LOG_BUF_STATUS_WRITE_PTR_MSB_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_STATUS_WRITE_PTR_MSB_ATTR 0x1
-#define HWIO_IPA_LOG_BUF_STATUS_WRITE_PTR_MSB_IN in_dword_masked( \
-		HWIO_IPA_LOG_BUF_STATUS_WRITE_PTR_MSB_ADDR, \
-		HWIO_IPA_LOG_BUF_STATUS_WRITE_PTR_MSB_RMSK)
-#define HWIO_IPA_LOG_BUF_STATUS_WRITE_PTR_MSB_INM(m) in_dword_masked( \
-		HWIO_IPA_LOG_BUF_STATUS_WRITE_PTR_MSB_ADDR, \
-		m)
-#define HWIO_IPA_LOG_BUF_STATUS_WRITE_PTR_MSB_WRITE_ADDR_BMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_STATUS_WRITE_PTR_MSB_WRITE_ADDR_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_STATUS_CFG_ADDR (IPA_EE_REG_BASE + 0x000010b0)
-#define HWIO_IPA_LOG_BUF_STATUS_CFG_PHYS (IPA_EE_REG_BASE_PHYS + \
-					  0x000010b0)
-#define HWIO_IPA_LOG_BUF_STATUS_CFG_OFFS (IPA_EE_REG_BASE_OFFS + \
-					  0x000010b0)
-#define HWIO_IPA_LOG_BUF_STATUS_CFG_RMSK 0x1ffff
-#define HWIO_IPA_LOG_BUF_STATUS_CFG_ATTR 0x3
-#define HWIO_IPA_LOG_BUF_STATUS_CFG_IN in_dword_masked(	\
-		HWIO_IPA_LOG_BUF_STATUS_CFG_ADDR, \
-		HWIO_IPA_LOG_BUF_STATUS_CFG_RMSK)
-#define HWIO_IPA_LOG_BUF_STATUS_CFG_INM(m) in_dword_masked( \
-		HWIO_IPA_LOG_BUF_STATUS_CFG_ADDR, \
-		m)
-#define HWIO_IPA_LOG_BUF_STATUS_CFG_OUT(v) out_dword( \
-		HWIO_IPA_LOG_BUF_STATUS_CFG_ADDR, \
-		v)
-#define HWIO_IPA_LOG_BUF_STATUS_CFG_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_LOG_BUF_STATUS_CFG_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_LOG_BUF_STATUS_CFG_IN)
-#define HWIO_IPA_LOG_BUF_STATUS_CFG_ENABLE_BMSK 0x10000
-#define HWIO_IPA_LOG_BUF_STATUS_CFG_ENABLE_SHFT 0x10
-#define HWIO_IPA_LOG_BUF_STATUS_CFG_SIZE_BMSK 0xffff
-#define HWIO_IPA_LOG_BUF_STATUS_CFG_SIZE_SHFT 0x0
-#define HWIO_IPA_LOG_BUF_STATUS_RAM_PTR_ADDR (IPA_EE_REG_BASE + 0x000010b4)
-#define HWIO_IPA_LOG_BUF_STATUS_RAM_PTR_PHYS (IPA_EE_REG_BASE_PHYS + \
-					      0x000010b4)
-#define HWIO_IPA_LOG_BUF_STATUS_RAM_PTR_OFFS (IPA_EE_REG_BASE_OFFS + \
-					      0x000010b4)
-#define HWIO_IPA_LOG_BUF_STATUS_RAM_PTR_RMSK 0xffffffff
-#define HWIO_IPA_LOG_BUF_STATUS_RAM_PTR_ATTR 0x1
-#define HWIO_IPA_LOG_BUF_STATUS_RAM_PTR_IN in_dword_masked( \
-		HWIO_IPA_LOG_BUF_STATUS_RAM_PTR_ADDR, \
-		HWIO_IPA_LOG_BUF_STATUS_RAM_PTR_RMSK)
-#define HWIO_IPA_LOG_BUF_STATUS_RAM_PTR_INM(m) in_dword_masked(	\
-		HWIO_IPA_LOG_BUF_STATUS_RAM_PTR_ADDR, \
-		m)
-#define HWIO_IPA_LOG_BUF_STATUS_RAM_PTR_WRITE_PTR_BMSK 0xffff0000
-#define HWIO_IPA_LOG_BUF_STATUS_RAM_PTR_WRITE_PTR_SHFT 0x10
-#define HWIO_IPA_LOG_BUF_STATUS_RAM_PTR_READ_PTR_BMSK 0xffff
-#define HWIO_IPA_LOG_BUF_STATUS_RAM_PTR_READ_PTR_SHFT 0x0
-#define HWIO_IPA_LTE_DECIPH_INIT_VALUES_0_ADDR (IPA_EE_REG_BASE + \
-						0x000010c0)
-#define HWIO_IPA_LTE_DECIPH_INIT_VALUES_0_PHYS (IPA_EE_REG_BASE_PHYS + \
-						0x000010c0)
-#define HWIO_IPA_LTE_DECIPH_INIT_VALUES_0_OFFS (IPA_EE_REG_BASE_OFFS + \
-						0x000010c0)
-#define HWIO_IPA_LTE_DECIPH_INIT_VALUES_1_ADDR (IPA_EE_REG_BASE + \
-						0x000010c4)
-#define HWIO_IPA_LTE_DECIPH_INIT_VALUES_1_PHYS (IPA_EE_REG_BASE_PHYS + \
-						0x000010c4)
-#define HWIO_IPA_LTE_DECIPH_INIT_VALUES_1_OFFS (IPA_EE_REG_BASE_OFFS + \
-						0x000010c4)
-#define HWIO_IPA_LTE_DECIPH_CONFIG_VALUES_0_ADDR (IPA_EE_REG_BASE + \
-						  0x000010c8)
-#define HWIO_IPA_LTE_DECIPH_CONFIG_VALUES_0_PHYS (IPA_EE_REG_BASE_PHYS + \
-						  0x000010c8)
-#define HWIO_IPA_LTE_DECIPH_CONFIG_VALUES_0_OFFS (IPA_EE_REG_BASE_OFFS + \
-						  0x000010c8)
-#define HWIO_IPA_LTE_DECIPH_CONFIG_VALUES_1_ADDR (IPA_EE_REG_BASE + \
-						  0x000010cc)
-#define HWIO_IPA_LTE_DECIPH_CONFIG_VALUES_1_PHYS (IPA_EE_REG_BASE_PHYS + \
-						  0x000010cc)
-#define HWIO_IPA_LTE_DECIPH_CONFIG_VALUES_1_OFFS (IPA_EE_REG_BASE_OFFS + \
-						  0x000010cc)
-#define HWIO_IPA_SECURED_PIPES_ADDR (IPA_EE_REG_BASE + 0x000010d0)
-#define HWIO_IPA_SECURED_PIPES_PHYS (IPA_EE_REG_BASE_PHYS + 0x000010d0)
-#define HWIO_IPA_SECURED_PIPES_OFFS (IPA_EE_REG_BASE_OFFS + 0x000010d0)
-#define HWIO_IPA_LTE_DECIPH_INIT_VALUES_CFG_ADDR (IPA_EE_REG_BASE + \
-						  0x000010d4)
-#define HWIO_IPA_LTE_DECIPH_INIT_VALUES_CFG_PHYS (IPA_EE_REG_BASE_PHYS + \
-						  0x000010d4)
-#define HWIO_IPA_LTE_DECIPH_INIT_VALUES_CFG_OFFS (IPA_EE_REG_BASE_OFFS + \
-						  0x000010d4)
-#define HWIO_IPA_UC_REGS_INSIDE_IPA__CONTROL_ADDR (IPA_EE_REG_BASE + \
-						   0x00001200)
-#define HWIO_IPA_UC_REGS_INSIDE_IPA__CONTROL_PHYS (IPA_EE_REG_BASE_PHYS + \
-						   0x00001200)
-#define HWIO_IPA_UC_REGS_INSIDE_IPA__CONTROL_OFFS (IPA_EE_REG_BASE_OFFS + \
-						   0x00001200)
-#define HWIO_IPA_UC_REGS_INSIDE_IPA__NMI_ADDR (IPA_EE_REG_BASE + \
-					       0x00001204)
-#define HWIO_IPA_UC_REGS_INSIDE_IPA__NMI_PHYS (IPA_EE_REG_BASE_PHYS + \
-					       0x00001204)
-#define HWIO_IPA_UC_REGS_INSIDE_IPA__NMI_OFFS (IPA_EE_REG_BASE_OFFS + \
-					       0x00001204)
-#define HWIO_IPA_SET_UC_IRQ_EE_n_ADDR(n) (IPA_EE_REG_BASE + 0x00002048 + \
-					  0x4 * (n))
-#define HWIO_IPA_SET_UC_IRQ_EE_n_PHYS(n) (IPA_EE_REG_BASE_PHYS + \
-					  0x00002048 + 0x4 * (n))
-#define HWIO_IPA_SET_UC_IRQ_EE_n_OFFS(n) (IPA_EE_REG_BASE_OFFS + \
-					  0x00002048 + 0x4 * (n))
-#define HWIO_IPA_SET_UC_IRQ_ALL_EES_ADDR (IPA_EE_REG_BASE + 0x00002058)
-#define HWIO_IPA_SET_UC_IRQ_ALL_EES_PHYS (IPA_EE_REG_BASE_PHYS + \
-					  0x00002058)
-#define HWIO_IPA_SET_UC_IRQ_ALL_EES_OFFS (IPA_EE_REG_BASE_OFFS + \
-					  0x00002058)
-#define HWIO_IPA_UCP_RESUME_ADDR (IPA_EE_REG_BASE + 0x000030a0)
-#define HWIO_IPA_UCP_RESUME_PHYS (IPA_EE_REG_BASE_PHYS + 0x000030a0)
-#define HWIO_IPA_UCP_RESUME_OFFS (IPA_EE_REG_BASE_OFFS + 0x000030a0)
-#define HWIO_IPA_PROC_UCP_CFG_ADDR (IPA_EE_REG_BASE + 0x000030a4)
-#define HWIO_IPA_PROC_UCP_CFG_PHYS (IPA_EE_REG_BASE_PHYS + 0x000030a4)
-#define HWIO_IPA_PROC_UCP_CFG_OFFS (IPA_EE_REG_BASE_OFFS + 0x000030a4)
-#define HWIO_IPA_UC_PKT_PROCESS_BASE_ADDR_0_ADDR (IPA_EE_REG_BASE + \
-						  0x000030a8)
-#define HWIO_IPA_UC_PKT_PROCESS_BASE_ADDR_0_PHYS (IPA_EE_REG_BASE_PHYS + \
-						  0x000030a8)
-#define HWIO_IPA_UC_PKT_PROCESS_BASE_ADDR_0_OFFS (IPA_EE_REG_BASE_OFFS + \
-						  0x000030a8)
-#define HWIO_IPA_UC_PKT_PROCESS_BASE_ADDR_1_ADDR (IPA_EE_REG_BASE + \
-						  0x000030ac)
-#define HWIO_IPA_UC_PKT_PROCESS_BASE_ADDR_1_PHYS (IPA_EE_REG_BASE_PHYS + \
-						  0x000030ac)
-#define HWIO_IPA_UC_PKT_PROCESS_BASE_ADDR_1_OFFS (IPA_EE_REG_BASE_OFFS + \
-						  0x000030ac)
-#define HWIO_IPA_UC_PKT_PROCESS_BASE_ADDR_2_ADDR (IPA_EE_REG_BASE + \
-						  0x000030b0)
-#define HWIO_IPA_UC_PKT_PROCESS_BASE_ADDR_2_PHYS (IPA_EE_REG_BASE_PHYS + \
-						  0x000030b0)
-#define HWIO_IPA_UC_PKT_PROCESS_BASE_ADDR_2_OFFS (IPA_EE_REG_BASE_OFFS + \
-						  0x000030b0)
-#define IPA_UC_IPA_UC_PER_REG_BASE (IPA_0_IPA_WRAPPER_BASE + 0x000c0000)
-#define IPA_UC_IPA_UC_PER_REG_BASE_PHYS (IPA_0_IPA_WRAPPER_BASE_PHYS + \
-					 0x000c0000)
-#define IPA_UC_IPA_UC_PER_REG_BASE_OFFS 0x000c0000
-#define HWIO_IPA_UC_STATUS_ADDR (IPA_UC_IPA_UC_PER_REG_BASE + 0x00000000)
-#define HWIO_IPA_UC_STATUS_PHYS (IPA_UC_IPA_UC_PER_REG_BASE_PHYS + \
-				 0x00000000)
-#define HWIO_IPA_UC_STATUS_OFFS (IPA_UC_IPA_UC_PER_REG_BASE_OFFS + \
-				 0x00000000)
-#define HWIO_IPA_UC_CONTROL_ADDR (IPA_UC_IPA_UC_PER_REG_BASE + 0x00000004)
-#define HWIO_IPA_UC_CONTROL_PHYS (IPA_UC_IPA_UC_PER_REG_BASE_PHYS + \
-				  0x00000004)
-#define HWIO_IPA_UC_CONTROL_OFFS (IPA_UC_IPA_UC_PER_REG_BASE_OFFS + \
-				  0x00000004)
-#define HWIO_IPA_UC_BASE_ADDR_ADDR (IPA_UC_IPA_UC_PER_REG_BASE + \
-				    0x00000008)
-#define HWIO_IPA_UC_BASE_ADDR_PHYS (IPA_UC_IPA_UC_PER_REG_BASE_PHYS + \
-				    0x00000008)
-#define HWIO_IPA_UC_BASE_ADDR_OFFS (IPA_UC_IPA_UC_PER_REG_BASE_OFFS + \
-				    0x00000008)
-#define HWIO_IPA_UC_BASE_ADDR_MSB_ADDR (IPA_UC_IPA_UC_PER_REG_BASE + \
-					0x0000000c)
-#define HWIO_IPA_UC_BASE_ADDR_MSB_PHYS (IPA_UC_IPA_UC_PER_REG_BASE_PHYS + \
-					0x0000000c)
-#define HWIO_IPA_UC_BASE_ADDR_MSB_OFFS (IPA_UC_IPA_UC_PER_REG_BASE_OFFS + \
-					0x0000000c)
-#define HWIO_IPA_UC_SYS_BUS_ATTRIB_ADDR (IPA_UC_IPA_UC_PER_REG_BASE + \
-					 0x00000010)
-#define HWIO_IPA_UC_SYS_BUS_ATTRIB_PHYS (IPA_UC_IPA_UC_PER_REG_BASE_PHYS + \
-					 0x00000010)
-#define HWIO_IPA_UC_SYS_BUS_ATTRIB_OFFS (IPA_UC_IPA_UC_PER_REG_BASE_OFFS + \
-					 0x00000010)
-#define HWIO_IPA_UC_PEND_IRQ_ADDR (IPA_UC_IPA_UC_PER_REG_BASE + 0x00000014)
-#define HWIO_IPA_UC_PEND_IRQ_PHYS (IPA_UC_IPA_UC_PER_REG_BASE_PHYS + \
-				   0x00000014)
-#define HWIO_IPA_UC_PEND_IRQ_OFFS (IPA_UC_IPA_UC_PER_REG_BASE_OFFS + \
-				   0x00000014)
-#define HWIO_IPA_UC_TRACE_BUFFER_ADDR (IPA_UC_IPA_UC_PER_REG_BASE + \
-				       0x00000018)
-#define HWIO_IPA_UC_TRACE_BUFFER_PHYS (IPA_UC_IPA_UC_PER_REG_BASE_PHYS + \
-				       0x00000018)
-#define HWIO_IPA_UC_TRACE_BUFFER_OFFS (IPA_UC_IPA_UC_PER_REG_BASE_OFFS + \
-				       0x00000018)
-#define HWIO_IPA_UC_PC_ADDR (IPA_UC_IPA_UC_PER_REG_BASE + 0x0000001c)
-#define HWIO_IPA_UC_PC_PHYS (IPA_UC_IPA_UC_PER_REG_BASE_PHYS + 0x0000001c)
-#define HWIO_IPA_UC_PC_OFFS (IPA_UC_IPA_UC_PER_REG_BASE_OFFS + 0x0000001c)
-#define HWIO_IPA_UC_VUIC_INT_ADDRESS_LSB_ADDR (IPA_UC_IPA_UC_PER_REG_BASE \
-					       + 0x00000024)
-#define HWIO_IPA_UC_VUIC_INT_ADDRESS_LSB_PHYS (	\
-		IPA_UC_IPA_UC_PER_REG_BASE_PHYS + 0x00000024)
-#define HWIO_IPA_UC_VUIC_INT_ADDRESS_LSB_OFFS (	\
-		IPA_UC_IPA_UC_PER_REG_BASE_OFFS + 0x00000024)
-#define HWIO_IPA_UC_VUIC_INT_ADDRESS_MSB_ADDR (IPA_UC_IPA_UC_PER_REG_BASE \
-					       + 0x00000028)
-#define HWIO_IPA_UC_VUIC_INT_ADDRESS_MSB_PHYS (	\
-		IPA_UC_IPA_UC_PER_REG_BASE_PHYS + 0x00000028)
-#define HWIO_IPA_UC_VUIC_INT_ADDRESS_MSB_OFFS (	\
-		IPA_UC_IPA_UC_PER_REG_BASE_OFFS + 0x00000028)
-#define HWIO_IPA_UC_QMB_SYS_ADDR_ADDR (IPA_UC_IPA_UC_PER_REG_BASE + \
-				       0x00000100)
-#define HWIO_IPA_UC_QMB_SYS_ADDR_PHYS (IPA_UC_IPA_UC_PER_REG_BASE_PHYS + \
-				       0x00000100)
-#define HWIO_IPA_UC_QMB_SYS_ADDR_OFFS (IPA_UC_IPA_UC_PER_REG_BASE_OFFS + \
-				       0x00000100)
-#define HWIO_IPA_UC_QMB_SYS_ADDR_RMSK 0xffffffff
-#define HWIO_IPA_UC_QMB_SYS_ADDR_ATTR 0x3
-#define HWIO_IPA_UC_QMB_SYS_ADDR_IN in_dword_masked( \
-		HWIO_IPA_UC_QMB_SYS_ADDR_ADDR, \
-		HWIO_IPA_UC_QMB_SYS_ADDR_RMSK)
-#define HWIO_IPA_UC_QMB_SYS_ADDR_INM(m) in_dword_masked( \
-		HWIO_IPA_UC_QMB_SYS_ADDR_ADDR, \
-		m)
-#define HWIO_IPA_UC_QMB_SYS_ADDR_OUT(v) out_dword( \
-		HWIO_IPA_UC_QMB_SYS_ADDR_ADDR, \
-		v)
-#define HWIO_IPA_UC_QMB_SYS_ADDR_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_UC_QMB_SYS_ADDR_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_UC_QMB_SYS_ADDR_IN)
-#define HWIO_IPA_UC_QMB_SYS_ADDR_ADDR_BMSK 0xffffffff
-#define HWIO_IPA_UC_QMB_SYS_ADDR_ADDR_SHFT 0x0
-#define HWIO_IPA_UC_QMB_SYS_ADDR_MSB_ADDR (IPA_UC_IPA_UC_PER_REG_BASE +	\
-					   0x00000104)
-#define HWIO_IPA_UC_QMB_SYS_ADDR_MSB_PHYS (IPA_UC_IPA_UC_PER_REG_BASE_PHYS \
-					   + 0x00000104)
-#define HWIO_IPA_UC_QMB_SYS_ADDR_MSB_OFFS (IPA_UC_IPA_UC_PER_REG_BASE_OFFS \
-					   + 0x00000104)
-#define HWIO_IPA_UC_QMB_SYS_ADDR_MSB_RMSK 0xffffffff
-#define HWIO_IPA_UC_QMB_SYS_ADDR_MSB_ATTR 0x3
-#define HWIO_IPA_UC_QMB_SYS_ADDR_MSB_IN in_dword_masked( \
-		HWIO_IPA_UC_QMB_SYS_ADDR_MSB_ADDR, \
-		HWIO_IPA_UC_QMB_SYS_ADDR_MSB_RMSK)
-#define HWIO_IPA_UC_QMB_SYS_ADDR_MSB_INM(m) in_dword_masked( \
-		HWIO_IPA_UC_QMB_SYS_ADDR_MSB_ADDR, \
-		m)
-#define HWIO_IPA_UC_QMB_SYS_ADDR_MSB_OUT(v) out_dword( \
-		HWIO_IPA_UC_QMB_SYS_ADDR_MSB_ADDR, \
-		v)
-#define HWIO_IPA_UC_QMB_SYS_ADDR_MSB_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_UC_QMB_SYS_ADDR_MSB_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_UC_QMB_SYS_ADDR_MSB_IN)
-#define HWIO_IPA_UC_QMB_SYS_ADDR_MSB_ADDR_MSB_BMSK 0xffffffff
-#define HWIO_IPA_UC_QMB_SYS_ADDR_MSB_ADDR_MSB_SHFT 0x0
-#define HWIO_IPA_UC_QMB_LOCAL_ADDR_ADDR (IPA_UC_IPA_UC_PER_REG_BASE + \
-					 0x00000108)
-#define HWIO_IPA_UC_QMB_LOCAL_ADDR_PHYS (IPA_UC_IPA_UC_PER_REG_BASE_PHYS + \
-					 0x00000108)
-#define HWIO_IPA_UC_QMB_LOCAL_ADDR_OFFS (IPA_UC_IPA_UC_PER_REG_BASE_OFFS + \
-					 0x00000108)
-#define HWIO_IPA_UC_QMB_LOCAL_ADDR_RMSK 0x3ffff
-#define HWIO_IPA_UC_QMB_LOCAL_ADDR_ATTR 0x3
-#define HWIO_IPA_UC_QMB_LOCAL_ADDR_IN in_dword_masked( \
-		HWIO_IPA_UC_QMB_LOCAL_ADDR_ADDR, \
-		HWIO_IPA_UC_QMB_LOCAL_ADDR_RMSK)
-#define HWIO_IPA_UC_QMB_LOCAL_ADDR_INM(m) in_dword_masked( \
-		HWIO_IPA_UC_QMB_LOCAL_ADDR_ADDR, \
-		m)
-#define HWIO_IPA_UC_QMB_LOCAL_ADDR_OUT(v) out_dword( \
-		HWIO_IPA_UC_QMB_LOCAL_ADDR_ADDR, \
-		v)
-#define HWIO_IPA_UC_QMB_LOCAL_ADDR_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_UC_QMB_LOCAL_ADDR_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_UC_QMB_LOCAL_ADDR_IN)
-#define HWIO_IPA_UC_QMB_LOCAL_ADDR_ADDR_BMSK 0x3ffff
-#define HWIO_IPA_UC_QMB_LOCAL_ADDR_ADDR_SHFT 0x0
-#define HWIO_IPA_UC_QMB_LENGTH_ADDR (IPA_UC_IPA_UC_PER_REG_BASE + \
-				     0x0000010c)
-#define HWIO_IPA_UC_QMB_LENGTH_PHYS (IPA_UC_IPA_UC_PER_REG_BASE_PHYS + \
-				     0x0000010c)
-#define HWIO_IPA_UC_QMB_LENGTH_OFFS (IPA_UC_IPA_UC_PER_REG_BASE_OFFS + \
-				     0x0000010c)
-#define HWIO_IPA_UC_QMB_LENGTH_RMSK 0x7f
-#define HWIO_IPA_UC_QMB_LENGTH_ATTR 0x3
-#define HWIO_IPA_UC_QMB_LENGTH_IN in_dword_masked( \
-		HWIO_IPA_UC_QMB_LENGTH_ADDR, \
-		HWIO_IPA_UC_QMB_LENGTH_RMSK)
-#define HWIO_IPA_UC_QMB_LENGTH_INM(m) in_dword_masked( \
-		HWIO_IPA_UC_QMB_LENGTH_ADDR, \
-		m)
-#define HWIO_IPA_UC_QMB_LENGTH_OUT(v) out_dword( \
-		HWIO_IPA_UC_QMB_LENGTH_ADDR, \
-		v)
-#define HWIO_IPA_UC_QMB_LENGTH_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_UC_QMB_LENGTH_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_UC_QMB_LENGTH_IN)
-#define HWIO_IPA_UC_QMB_LENGTH_LENGTH_BMSK 0x7f
-#define HWIO_IPA_UC_QMB_LENGTH_LENGTH_SHFT 0x0
-#define HWIO_IPA_UC_QMB_TRIGGER_ADDR (IPA_UC_IPA_UC_PER_REG_BASE + \
-				      0x00000110)
-#define HWIO_IPA_UC_QMB_TRIGGER_PHYS (IPA_UC_IPA_UC_PER_REG_BASE_PHYS +	\
-				      0x00000110)
-#define HWIO_IPA_UC_QMB_TRIGGER_OFFS (IPA_UC_IPA_UC_PER_REG_BASE_OFFS +	\
-				      0x00000110)
-#define HWIO_IPA_UC_QMB_TRIGGER_RMSK 0x31
-#define HWIO_IPA_UC_QMB_TRIGGER_ATTR 0x3
-#define HWIO_IPA_UC_QMB_TRIGGER_IN in_dword_masked( \
-		HWIO_IPA_UC_QMB_TRIGGER_ADDR, \
-		HWIO_IPA_UC_QMB_TRIGGER_RMSK)
-#define HWIO_IPA_UC_QMB_TRIGGER_INM(m) in_dword_masked(	\
-		HWIO_IPA_UC_QMB_TRIGGER_ADDR, \
-		m)
-#define HWIO_IPA_UC_QMB_TRIGGER_OUT(v) out_dword( \
-		HWIO_IPA_UC_QMB_TRIGGER_ADDR, \
-		v)
-#define HWIO_IPA_UC_QMB_TRIGGER_OUTM(m, v) out_dword_masked_ns(	\
-		HWIO_IPA_UC_QMB_TRIGGER_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_UC_QMB_TRIGGER_IN)
-#define HWIO_IPA_UC_QMB_TRIGGER_POSTING_BMSK 0x30
-#define HWIO_IPA_UC_QMB_TRIGGER_POSTING_SHFT 0x4
-#define HWIO_IPA_UC_QMB_TRIGGER_POSTING_DATA_POSTED_FVAL 0x0
-#define HWIO_IPA_UC_QMB_TRIGGER_POSTING_RESP_POSTED_FVAL 0x1
-#define HWIO_IPA_UC_QMB_TRIGGER_POSTING_DATA_COMPLETE_FVAL 0x2
-#define HWIO_IPA_UC_QMB_TRIGGER_POSTING_RESP_COMPLETE_FVAL 0x3
-#define HWIO_IPA_UC_QMB_TRIGGER_DIRECTION_BMSK 0x1
-#define HWIO_IPA_UC_QMB_TRIGGER_DIRECTION_SHFT 0x0
-#define HWIO_IPA_UC_QMB_TRIGGER_DIRECTION_READ_FVAL 0x0
-#define HWIO_IPA_UC_QMB_TRIGGER_DIRECTION_WRITE_FVAL 0x1
-#define HWIO_IPA_UC_QMB_PENDING_TID_ADDR (IPA_UC_IPA_UC_PER_REG_BASE + \
-					  0x00000114)
-#define HWIO_IPA_UC_QMB_PENDING_TID_PHYS (IPA_UC_IPA_UC_PER_REG_BASE_PHYS \
-					  + 0x00000114)
-#define HWIO_IPA_UC_QMB_PENDING_TID_OFFS (IPA_UC_IPA_UC_PER_REG_BASE_OFFS \
-					  + 0x00000114)
-#define HWIO_IPA_UC_QMB_PENDING_TID_RMSK 0x11113f
-#define HWIO_IPA_UC_QMB_PENDING_TID_ATTR 0x1
-#define HWIO_IPA_UC_QMB_PENDING_TID_IN in_dword_masked(	\
-		HWIO_IPA_UC_QMB_PENDING_TID_ADDR, \
-		HWIO_IPA_UC_QMB_PENDING_TID_RMSK)
-#define HWIO_IPA_UC_QMB_PENDING_TID_INM(m) in_dword_masked( \
-		HWIO_IPA_UC_QMB_PENDING_TID_ADDR, \
-		m)
-#define HWIO_IPA_UC_QMB_PENDING_TID_ERROR_SECURITY_BMSK 0x100000
-#define HWIO_IPA_UC_QMB_PENDING_TID_ERROR_SECURITY_SHFT 0x14
-#define HWIO_IPA_UC_QMB_PENDING_TID_ERROR_MAX_COMP_BMSK 0x10000
-#define HWIO_IPA_UC_QMB_PENDING_TID_ERROR_MAX_COMP_SHFT 0x10
-#define HWIO_IPA_UC_QMB_PENDING_TID_ERROR_MAX_OS_BMSK 0x1000
-#define HWIO_IPA_UC_QMB_PENDING_TID_ERROR_MAX_OS_SHFT 0xc
-#define HWIO_IPA_UC_QMB_PENDING_TID_ERROR_BUS_BMSK 0x100
-#define HWIO_IPA_UC_QMB_PENDING_TID_ERROR_BUS_SHFT 0x8
-#define HWIO_IPA_UC_QMB_PENDING_TID_TID_BMSK 0x3f
-#define HWIO_IPA_UC_QMB_PENDING_TID_TID_SHFT 0x0
-#define HWIO_IPA_UC_QMB_COMPLETED_RD_FIFO_ADDR (IPA_UC_IPA_UC_PER_REG_BASE \
-						+ 0x00000118)
-#define HWIO_IPA_UC_QMB_COMPLETED_RD_FIFO_PHYS ( \
-		IPA_UC_IPA_UC_PER_REG_BASE_PHYS + 0x00000118)
-#define HWIO_IPA_UC_QMB_COMPLETED_RD_FIFO_OFFS ( \
-		IPA_UC_IPA_UC_PER_REG_BASE_OFFS + 0x00000118)
-#define HWIO_IPA_UC_QMB_COMPLETED_RD_FIFO_PEEK_ADDR ( \
-		IPA_UC_IPA_UC_PER_REG_BASE + 0x0000011c)
-#define HWIO_IPA_UC_QMB_COMPLETED_RD_FIFO_PEEK_PHYS ( \
-		IPA_UC_IPA_UC_PER_REG_BASE_PHYS + 0x0000011c)
-#define HWIO_IPA_UC_QMB_COMPLETED_RD_FIFO_PEEK_OFFS ( \
-		IPA_UC_IPA_UC_PER_REG_BASE_OFFS + 0x0000011c)
-#define HWIO_IPA_UC_QMB_COMPLETED_RD_FIFO_PEEK_RMSK 0x113f
-#define HWIO_IPA_UC_QMB_COMPLETED_RD_FIFO_PEEK_ATTR 0x1
-#define HWIO_IPA_UC_QMB_COMPLETED_RD_FIFO_PEEK_IN in_dword_masked( \
-		HWIO_IPA_UC_QMB_COMPLETED_RD_FIFO_PEEK_ADDR, \
-		HWIO_IPA_UC_QMB_COMPLETED_RD_FIFO_PEEK_RMSK)
-#define HWIO_IPA_UC_QMB_COMPLETED_RD_FIFO_PEEK_INM(m) in_dword_masked( \
-		HWIO_IPA_UC_QMB_COMPLETED_RD_FIFO_PEEK_ADDR, \
-		m)
-#define HWIO_IPA_UC_QMB_COMPLETED_RD_FIFO_PEEK_VALID_BMSK 0x1000
-#define HWIO_IPA_UC_QMB_COMPLETED_RD_FIFO_PEEK_VALID_SHFT 0xc
-#define HWIO_IPA_UC_QMB_COMPLETED_RD_FIFO_PEEK_ERROR_BMSK 0x100
-#define HWIO_IPA_UC_QMB_COMPLETED_RD_FIFO_PEEK_ERROR_SHFT 0x8
-#define HWIO_IPA_UC_QMB_COMPLETED_RD_FIFO_PEEK_TID_BMSK 0x3f
-#define HWIO_IPA_UC_QMB_COMPLETED_RD_FIFO_PEEK_TID_SHFT 0x0
-#define HWIO_IPA_UC_QMB_COMPLETED_WR_FIFO_ADDR (IPA_UC_IPA_UC_PER_REG_BASE \
-						+ 0x00000120)
-#define HWIO_IPA_UC_QMB_COMPLETED_WR_FIFO_PHYS ( \
-		IPA_UC_IPA_UC_PER_REG_BASE_PHYS + 0x00000120)
-#define HWIO_IPA_UC_QMB_COMPLETED_WR_FIFO_OFFS ( \
-		IPA_UC_IPA_UC_PER_REG_BASE_OFFS + 0x00000120)
-#define HWIO_IPA_UC_QMB_COMPLETED_WR_FIFO_PEEK_ADDR ( \
-		IPA_UC_IPA_UC_PER_REG_BASE + 0x00000124)
-#define HWIO_IPA_UC_QMB_COMPLETED_WR_FIFO_PEEK_PHYS ( \
-		IPA_UC_IPA_UC_PER_REG_BASE_PHYS + 0x00000124)
-#define HWIO_IPA_UC_QMB_COMPLETED_WR_FIFO_PEEK_OFFS ( \
-		IPA_UC_IPA_UC_PER_REG_BASE_OFFS + 0x00000124)
-#define HWIO_IPA_UC_QMB_COMPLETED_WR_FIFO_PEEK_RMSK 0x113f
-#define HWIO_IPA_UC_QMB_COMPLETED_WR_FIFO_PEEK_ATTR 0x1
-#define HWIO_IPA_UC_QMB_COMPLETED_WR_FIFO_PEEK_IN in_dword_masked( \
-		HWIO_IPA_UC_QMB_COMPLETED_WR_FIFO_PEEK_ADDR, \
-		HWIO_IPA_UC_QMB_COMPLETED_WR_FIFO_PEEK_RMSK)
-#define HWIO_IPA_UC_QMB_COMPLETED_WR_FIFO_PEEK_INM(m) in_dword_masked( \
-		HWIO_IPA_UC_QMB_COMPLETED_WR_FIFO_PEEK_ADDR, \
-		m)
-#define HWIO_IPA_UC_QMB_COMPLETED_WR_FIFO_PEEK_VALID_BMSK 0x1000
-#define HWIO_IPA_UC_QMB_COMPLETED_WR_FIFO_PEEK_VALID_SHFT 0xc
-#define HWIO_IPA_UC_QMB_COMPLETED_WR_FIFO_PEEK_ERROR_BMSK 0x100
-#define HWIO_IPA_UC_QMB_COMPLETED_WR_FIFO_PEEK_ERROR_SHFT 0x8
-#define HWIO_IPA_UC_QMB_COMPLETED_WR_FIFO_PEEK_TID_BMSK 0x3f
-#define HWIO_IPA_UC_QMB_COMPLETED_WR_FIFO_PEEK_TID_SHFT 0x0
-#define HWIO_IPA_UC_QMB_MISC_ADDR (IPA_UC_IPA_UC_PER_REG_BASE + 0x00000128)
-#define HWIO_IPA_UC_QMB_MISC_PHYS (IPA_UC_IPA_UC_PER_REG_BASE_PHYS + \
-				   0x00000128)
-#define HWIO_IPA_UC_QMB_MISC_OFFS (IPA_UC_IPA_UC_PER_REG_BASE_OFFS + \
-				   0x00000128)
-#define HWIO_IPA_UC_QMB_MISC_RMSK 0xf11333ff
-#define HWIO_IPA_UC_QMB_MISC_ATTR 0x3
-#define HWIO_IPA_UC_QMB_MISC_IN in_dword_masked(HWIO_IPA_UC_QMB_MISC_ADDR, \
-						HWIO_IPA_UC_QMB_MISC_RMSK)
-#define HWIO_IPA_UC_QMB_MISC_INM(m) in_dword_masked( \
-		HWIO_IPA_UC_QMB_MISC_ADDR, \
-		m)
-#define HWIO_IPA_UC_QMB_MISC_OUT(v) out_dword(HWIO_IPA_UC_QMB_MISC_ADDR, v)
-#define HWIO_IPA_UC_QMB_MISC_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_UC_QMB_MISC_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_UC_QMB_MISC_IN)
-#define HWIO_IPA_UC_QMB_MISC_QMB_HREADY_BCR_BMSK 0x80000000
-#define HWIO_IPA_UC_QMB_MISC_QMB_HREADY_BCR_SHFT 0x1f
-#define HWIO_IPA_UC_QMB_MISC_POSTED_STALL_BMSK 0x40000000
-#define HWIO_IPA_UC_QMB_MISC_POSTED_STALL_SHFT 0x1e
-#define HWIO_IPA_UC_QMB_MISC_IRQ_COAL_BMSK 0x20000000
-#define HWIO_IPA_UC_QMB_MISC_IRQ_COAL_SHFT 0x1d
-#define HWIO_IPA_UC_QMB_MISC_SWAP_BMSK 0x10000000
-#define HWIO_IPA_UC_QMB_MISC_SWAP_SHFT 0x1c
-#define HWIO_IPA_UC_QMB_MISC_OOOWR_BMSK 0x1000000
-#define HWIO_IPA_UC_QMB_MISC_OOOWR_SHFT 0x18
-#define HWIO_IPA_UC_QMB_MISC_OOORD_BMSK 0x100000
-#define HWIO_IPA_UC_QMB_MISC_OOORD_SHFT 0x14
-#define HWIO_IPA_UC_QMB_MISC_WR_PRIORITY_BMSK 0x30000
-#define HWIO_IPA_UC_QMB_MISC_WR_PRIORITY_SHFT 0x10
-#define HWIO_IPA_UC_QMB_MISC_RD_PRIORITY_BMSK 0x3000
-#define HWIO_IPA_UC_QMB_MISC_RD_PRIORITY_SHFT 0xc
-#define HWIO_IPA_UC_QMB_MISC_USER_BMSK 0x3ff
-#define HWIO_IPA_UC_QMB_MISC_USER_SHFT 0x0
-#define HWIO_IPA_UC_QMB_STATUS_ADDR (IPA_UC_IPA_UC_PER_REG_BASE + \
-				     0x0000012c)
-#define HWIO_IPA_UC_QMB_STATUS_PHYS (IPA_UC_IPA_UC_PER_REG_BASE_PHYS + \
-				     0x0000012c)
-#define HWIO_IPA_UC_QMB_STATUS_OFFS (IPA_UC_IPA_UC_PER_REG_BASE_OFFS + \
-				     0x0000012c)
-#define HWIO_IPA_UC_QMB_STATUS_RMSK 0x1fff1fff
-#define HWIO_IPA_UC_QMB_STATUS_ATTR 0x1
-#define HWIO_IPA_UC_QMB_STATUS_IN in_dword_masked( \
-		HWIO_IPA_UC_QMB_STATUS_ADDR, \
-		HWIO_IPA_UC_QMB_STATUS_RMSK)
-#define HWIO_IPA_UC_QMB_STATUS_INM(m) in_dword_masked( \
-		HWIO_IPA_UC_QMB_STATUS_ADDR, \
-		m)
-#define HWIO_IPA_UC_QMB_STATUS_COMPLETED_WR_FIFO_FULL_BMSK 0x10000000
-#define HWIO_IPA_UC_QMB_STATUS_COMPLETED_WR_FIFO_FULL_SHFT 0x1c
-#define HWIO_IPA_UC_QMB_STATUS_COMPLETED_WR_CNT_BMSK 0xf000000
-#define HWIO_IPA_UC_QMB_STATUS_COMPLETED_WR_CNT_SHFT 0x18
-#define HWIO_IPA_UC_QMB_STATUS_OUTSTANDING_WR_CNT_BMSK 0xf00000
-#define HWIO_IPA_UC_QMB_STATUS_OUTSTANDING_WR_CNT_SHFT 0x14
-#define HWIO_IPA_UC_QMB_STATUS_MAX_OUTSTANDING_WR_BMSK 0xf0000
-#define HWIO_IPA_UC_QMB_STATUS_MAX_OUTSTANDING_WR_SHFT 0x10
-#define HWIO_IPA_UC_QMB_STATUS_COMPLETED_RD_FIFO_FULL_BMSK 0x1000
-#define HWIO_IPA_UC_QMB_STATUS_COMPLETED_RD_FIFO_FULL_SHFT 0xc
-#define HWIO_IPA_UC_QMB_STATUS_COMPLETED_RD_CNT_BMSK 0xf00
-#define HWIO_IPA_UC_QMB_STATUS_COMPLETED_RD_CNT_SHFT 0x8
-#define HWIO_IPA_UC_QMB_STATUS_OUTSTANDING_RD_CNT_BMSK 0xf0
-#define HWIO_IPA_UC_QMB_STATUS_OUTSTANDING_RD_CNT_SHFT 0x4
-#define HWIO_IPA_UC_QMB_STATUS_MAX_OUTSTANDING_RD_BMSK 0xf
-#define HWIO_IPA_UC_QMB_STATUS_MAX_OUTSTANDING_RD_SHFT 0x0
-#define HWIO_IPA_UC_QMB_BUS_ATTRIB_ADDR (IPA_UC_IPA_UC_PER_REG_BASE + \
-					 0x00000130)
-#define HWIO_IPA_UC_QMB_BUS_ATTRIB_PHYS (IPA_UC_IPA_UC_PER_REG_BASE_PHYS + \
-					 0x00000130)
-#define HWIO_IPA_UC_QMB_BUS_ATTRIB_OFFS (IPA_UC_IPA_UC_PER_REG_BASE_OFFS + \
-					 0x00000130)
-#define HWIO_IPA_UC_QMB_BUS_ATTRIB_RMSK 0x1117
-#define HWIO_IPA_UC_QMB_BUS_ATTRIB_ATTR 0x3
-#define HWIO_IPA_UC_QMB_BUS_ATTRIB_IN in_dword_masked( \
-		HWIO_IPA_UC_QMB_BUS_ATTRIB_ADDR, \
-		HWIO_IPA_UC_QMB_BUS_ATTRIB_RMSK)
-#define HWIO_IPA_UC_QMB_BUS_ATTRIB_INM(m) in_dword_masked( \
-		HWIO_IPA_UC_QMB_BUS_ATTRIB_ADDR, \
-		m)
-#define HWIO_IPA_UC_QMB_BUS_ATTRIB_OUT(v) out_dword( \
-		HWIO_IPA_UC_QMB_BUS_ATTRIB_ADDR, \
-		v)
-#define HWIO_IPA_UC_QMB_BUS_ATTRIB_OUTM(m, v) out_dword_masked_ns( \
-		HWIO_IPA_UC_QMB_BUS_ATTRIB_ADDR, \
-		m, \
-		v, \
-		HWIO_IPA_UC_QMB_BUS_ATTRIB_IN)
-#define HWIO_IPA_UC_QMB_BUS_ATTRIB_SHARED_BMSK 0x1000
-#define HWIO_IPA_UC_QMB_BUS_ATTRIB_SHARED_SHFT 0xc
-#define HWIO_IPA_UC_QMB_BUS_ATTRIB_INNERSHARED_BMSK 0x100
-#define HWIO_IPA_UC_QMB_BUS_ATTRIB_INNERSHARED_SHFT 0x8
-#define HWIO_IPA_UC_QMB_BUS_ATTRIB_NOALLOCATE_BMSK 0x10
-#define HWIO_IPA_UC_QMB_BUS_ATTRIB_NOALLOCATE_SHFT 0x4
-#define HWIO_IPA_UC_QMB_BUS_ATTRIB_MEMTYPE_BMSK 0x7
-#define HWIO_IPA_UC_QMB_BUS_ATTRIB_MEMTYPE_SHFT 0x0
-#define HWIO_IPA_UC_QMB_BUS_ATTRIB_MEMTYPE_STRONGLY_ORDERED_FVAL 0x0
-#define HWIO_IPA_UC_QMB_BUS_ATTRIB_MEMTYPE_DEVICE_FVAL 0x1
-#define HWIO_IPA_UC_QMB_BUS_ATTRIB_MEMTYPE_NON_CACHEABLE_FVAL 0x2
-#define HWIO_IPA_UC_QMB_BUS_ATTRIB_MEMTYPE_COPYBACK_WRITEALLOCATE_FVAL 0x3
-#define HWIO_IPA_UC_QMB_BUS_ATTRIB_MEMTYPE_WRITETHROUGH_NOALLOCATE_FVAL	\
-	0x6
-#define HWIO_IPA_UC_QMB_BUS_ATTRIB_MEMTYPE_COPYBACK_NOALLOCATE_FVAL 0x7
-#define HWIO_IPA_UC_MBOX_INT_STTS_n_ADDR(n) (IPA_UC_IPA_UC_PER_REG_BASE + \
-					     0x00000200 + 0x10 * (n))
-#define HWIO_IPA_UC_MBOX_INT_STTS_n_PHYS(n) ( \
-		IPA_UC_IPA_UC_PER_REG_BASE_PHYS + 0x00000200 + 0x10 * (n))
-#define HWIO_IPA_UC_MBOX_INT_STTS_n_OFFS(n) ( \
-		IPA_UC_IPA_UC_PER_REG_BASE_OFFS + 0x00000200 + 0x10 * (n))
-#define HWIO_IPA_UC_MBOX_INT_EN_n_ADDR(n) (IPA_UC_IPA_UC_PER_REG_BASE +	\
-					   0x00000204 + 0x10 * (n))
-#define HWIO_IPA_UC_MBOX_INT_EN_n_PHYS(n) (IPA_UC_IPA_UC_PER_REG_BASE_PHYS \
-					   + 0x00000204 + 0x10 * (n))
-#define HWIO_IPA_UC_MBOX_INT_EN_n_OFFS(n) (IPA_UC_IPA_UC_PER_REG_BASE_OFFS \
-					   + 0x00000204 + 0x10 * (n))
-#define HWIO_IPA_UC_MBOX_INT_CLR_n_ADDR(n) (IPA_UC_IPA_UC_PER_REG_BASE + \
-					    0x00000208 + 0x10 * (n))
-#define HWIO_IPA_UC_MBOX_INT_CLR_n_PHYS(n) ( \
-		IPA_UC_IPA_UC_PER_REG_BASE_PHYS + 0x00000208 + 0x10 * (n))
-#define HWIO_IPA_UC_MBOX_INT_CLR_n_OFFS(n) ( \
-		IPA_UC_IPA_UC_PER_REG_BASE_OFFS + 0x00000208 + 0x10 * (n))
-#define HWIO_IPA_UC_IPA_INT_STTS_n_ADDR(n) (IPA_UC_IPA_UC_PER_REG_BASE + \
-					    0x00000300 + 0x10 * (n))
-#define HWIO_IPA_UC_IPA_INT_STTS_n_PHYS(n) ( \
-		IPA_UC_IPA_UC_PER_REG_BASE_PHYS + 0x00000300 + 0x10 * (n))
-#define HWIO_IPA_UC_IPA_INT_STTS_n_OFFS(n) ( \
-		IPA_UC_IPA_UC_PER_REG_BASE_OFFS + 0x00000300 + 0x10 * (n))
-#define HWIO_IPA_UC_IPA_INT_EN_n_ADDR(n) (IPA_UC_IPA_UC_PER_REG_BASE + \
-					  0x00000304 + 0x10 * (n))
-#define HWIO_IPA_UC_IPA_INT_EN_n_PHYS(n) (IPA_UC_IPA_UC_PER_REG_BASE_PHYS \
-					  + 0x00000304 + 0x10 * (n))
-#define HWIO_IPA_UC_IPA_INT_EN_n_OFFS(n) (IPA_UC_IPA_UC_PER_REG_BASE_OFFS \
-					  + 0x00000304 + 0x10 * (n))
-#define HWIO_IPA_UC_IPA_INT_CLR_n_ADDR(n) (IPA_UC_IPA_UC_PER_REG_BASE +	\
-					   0x00000308 + 0x10 * (n))
-#define HWIO_IPA_UC_IPA_INT_CLR_n_PHYS(n) (IPA_UC_IPA_UC_PER_REG_BASE_PHYS \
-					   + 0x00000308 + 0x10 * (n))
-#define HWIO_IPA_UC_IPA_INT_CLR_n_OFFS(n) (IPA_UC_IPA_UC_PER_REG_BASE_OFFS \
-					   + 0x00000308 + 0x10 * (n))
-#define HWIO_IPA_UC_HWEV_INT_STTS_ADDR (IPA_UC_IPA_UC_PER_REG_BASE + \
-					0x00000400)
-#define HWIO_IPA_UC_HWEV_INT_STTS_PHYS (IPA_UC_IPA_UC_PER_REG_BASE_PHYS + \
-					0x00000400)
-#define HWIO_IPA_UC_HWEV_INT_STTS_OFFS (IPA_UC_IPA_UC_PER_REG_BASE_OFFS + \
-					0x00000400)
-#define HWIO_IPA_UC_HWEV_INT_EN_ADDR (IPA_UC_IPA_UC_PER_REG_BASE + \
-				      0x00000404)
-#define HWIO_IPA_UC_HWEV_INT_EN_PHYS (IPA_UC_IPA_UC_PER_REG_BASE_PHYS +	\
-				      0x00000404)
-#define HWIO_IPA_UC_HWEV_INT_EN_OFFS (IPA_UC_IPA_UC_PER_REG_BASE_OFFS +	\
-				      0x00000404)
-#define HWIO_IPA_UC_HWEV_INT_CLR_ADDR (IPA_UC_IPA_UC_PER_REG_BASE + \
-				       0x00000408)
-#define HWIO_IPA_UC_HWEV_INT_CLR_PHYS (IPA_UC_IPA_UC_PER_REG_BASE_PHYS + \
-				       0x00000408)
-#define HWIO_IPA_UC_HWEV_INT_CLR_OFFS (IPA_UC_IPA_UC_PER_REG_BASE_OFFS + \
-				       0x00000408)
-#define HWIO_IPA_UC_SWEV_INT_STTS_ADDR (IPA_UC_IPA_UC_PER_REG_BASE + \
-					0x00000410)
-#define HWIO_IPA_UC_SWEV_INT_STTS_PHYS (IPA_UC_IPA_UC_PER_REG_BASE_PHYS + \
-					0x00000410)
-#define HWIO_IPA_UC_SWEV_INT_STTS_OFFS (IPA_UC_IPA_UC_PER_REG_BASE_OFFS + \
-					0x00000410)
-#define HWIO_IPA_UC_SWEV_INT_EN_ADDR (IPA_UC_IPA_UC_PER_REG_BASE + \
-				      0x00000414)
-#define HWIO_IPA_UC_SWEV_INT_EN_PHYS (IPA_UC_IPA_UC_PER_REG_BASE_PHYS +	\
-				      0x00000414)
-#define HWIO_IPA_UC_SWEV_INT_EN_OFFS (IPA_UC_IPA_UC_PER_REG_BASE_OFFS +	\
-				      0x00000414)
-#define HWIO_IPA_UC_SWEV_INT_CLR_ADDR (IPA_UC_IPA_UC_PER_REG_BASE + \
-				       0x00000418)
-#define HWIO_IPA_UC_SWEV_INT_CLR_PHYS (IPA_UC_IPA_UC_PER_REG_BASE_PHYS + \
-				       0x00000418)
-#define HWIO_IPA_UC_SWEV_INT_CLR_OFFS (IPA_UC_IPA_UC_PER_REG_BASE_OFFS + \
-				       0x00000418)
-#define HWIO_IPA_UC_VUIC_INT_STTS_ADDR (IPA_UC_IPA_UC_PER_REG_BASE + \
-					0x0000041c)
-#define HWIO_IPA_UC_VUIC_INT_STTS_PHYS (IPA_UC_IPA_UC_PER_REG_BASE_PHYS + \
-					0x0000041c)
-#define HWIO_IPA_UC_VUIC_INT_STTS_OFFS (IPA_UC_IPA_UC_PER_REG_BASE_OFFS + \
-					0x0000041c)
-#define HWIO_IPA_UC_VUIC_INT_CLR_ADDR (IPA_UC_IPA_UC_PER_REG_BASE + \
-				       0x00000420)
-#define HWIO_IPA_UC_VUIC_INT_CLR_PHYS (IPA_UC_IPA_UC_PER_REG_BASE_PHYS + \
-				       0x00000420)
-#define HWIO_IPA_UC_VUIC_INT_CLR_OFFS (IPA_UC_IPA_UC_PER_REG_BASE_OFFS + \
-				       0x00000420)
-#define HWIO_IPA_UC_TIMER_CTRL_n_ADDR(n) (IPA_UC_IPA_UC_PER_REG_BASE + \
-					  0x00000500 + 0x10 * (n))
-#define HWIO_IPA_UC_TIMER_CTRL_n_PHYS(n) (IPA_UC_IPA_UC_PER_REG_BASE_PHYS \
-					  + 0x00000500 + 0x10 * (n))
-#define HWIO_IPA_UC_TIMER_CTRL_n_OFFS(n) (IPA_UC_IPA_UC_PER_REG_BASE_OFFS \
-					  + 0x00000500 + 0x10 * (n))
-#define HWIO_IPA_UC_TIMER_STATUS_n_ADDR(n) (IPA_UC_IPA_UC_PER_REG_BASE + \
-					    0x00000508 + 0x10 * (n))
-#define HWIO_IPA_UC_TIMER_STATUS_n_PHYS(n) ( \
-		IPA_UC_IPA_UC_PER_REG_BASE_PHYS + 0x00000508 + 0x10 * (n))
-#define HWIO_IPA_UC_TIMER_STATUS_n_OFFS(n) ( \
-		IPA_UC_IPA_UC_PER_REG_BASE_OFFS + 0x00000508 + 0x10 * (n))
-#define HWIO_IPA_UC_EVENTS_ADDR (IPA_UC_IPA_UC_PER_REG_BASE + 0x00000600)
-#define HWIO_IPA_UC_EVENTS_PHYS (IPA_UC_IPA_UC_PER_REG_BASE_PHYS + \
-				 0x00000600)
-#define HWIO_IPA_UC_EVENTS_OFFS (IPA_UC_IPA_UC_PER_REG_BASE_OFFS + \
-				 0x00000600)
-#define HWIO_IPA_UC_VUIC_BUS_ADDR_TRANSLATE_EN_ADDR ( \
-		IPA_UC_IPA_UC_PER_REG_BASE + 0x00000710)
-#define HWIO_IPA_UC_VUIC_BUS_ADDR_TRANSLATE_EN_PHYS ( \
-		IPA_UC_IPA_UC_PER_REG_BASE_PHYS + 0x00000710)
-#define HWIO_IPA_UC_VUIC_BUS_ADDR_TRANSLATE_EN_OFFS ( \
-		IPA_UC_IPA_UC_PER_REG_BASE_OFFS + 0x00000710)
-#define HWIO_IPA_UC_SYS_ADDR_MSB_ADDR (IPA_UC_IPA_UC_PER_REG_BASE + \
-				       0x00000714)
-#define HWIO_IPA_UC_SYS_ADDR_MSB_PHYS (IPA_UC_IPA_UC_PER_REG_BASE_PHYS + \
-				       0x00000714)
-#define HWIO_IPA_UC_SYS_ADDR_MSB_OFFS (IPA_UC_IPA_UC_PER_REG_BASE_OFFS + \
-				       0x00000714)
-#define HWIO_IPA_UC_PC_RESTORE_WR_ADDR (IPA_UC_IPA_UC_PER_REG_BASE + \
-					0x00000718)
-#define HWIO_IPA_UC_PC_RESTORE_WR_PHYS (IPA_UC_IPA_UC_PER_REG_BASE_PHYS + \
-					0x00000718)
-#define HWIO_IPA_UC_PC_RESTORE_WR_OFFS (IPA_UC_IPA_UC_PER_REG_BASE_OFFS + \
-					0x00000718)
-#define HWIO_IPA_UC_PC_RESTORE_RD_ADDR (IPA_UC_IPA_UC_PER_REG_BASE + \
-					0x0000071c)
-#define HWIO_IPA_UC_PC_RESTORE_RD_PHYS (IPA_UC_IPA_UC_PER_REG_BASE_PHYS + \
-					0x0000071c)
-#define HWIO_IPA_UC_PC_RESTORE_RD_OFFS (IPA_UC_IPA_UC_PER_REG_BASE_OFFS + \
-					0x0000071c)
-#define HWIO_IPA_UC_SPARE_ADDR (IPA_UC_IPA_UC_PER_REG_BASE + 0x00001ffc)
-#define HWIO_IPA_UC_SPARE_PHYS (IPA_UC_IPA_UC_PER_REG_BASE_PHYS + \
-				0x00001ffc)
-#define HWIO_IPA_UC_SPARE_OFFS (IPA_UC_IPA_UC_PER_REG_BASE_OFFS + \
-				0x00001ffc)
-#endif

ipa/ipa_v3/dump/ipa4.5/ipa_hwio_def.h

@@ -1,2963 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2019, The Linux Foundation. All rights reserved.
- */
-#if !defined(_IPA_HWIO_DEF_H_)
-#define _IPA_HWIO_DEF_H_
-struct ipa_hwio_def_ipa_gsi_top_gsi_cfg_s {
-	u32	gsi_enable : 1;
-	u32	mcs_enable : 1;
-	u32	double_mcs_clk_freq : 1;
-	u32	uc_is_mcs : 1;
-	u32	gsi_pwr_clps : 1;
-	u32	bp_mtrix_disable : 1;
-	u32	reserved0 : 2;
-	u32	sleep_clk_div : 4;
-	u32	reserved1 : 20;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_cfg_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_cfg_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_ree_cfg_s {
-	u32	move_to_esc_clr_mode_trsh : 1;
-	u32	channel_empty_int_enable : 1;
-	u32	reserved0 : 6;
-	u32	max_burst_size : 8;
-	u32	reserved1 : 16;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_ree_cfg_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_ree_cfg_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_manager_ee_qos_n_s {
-	u32	ee_prio : 2;
-	u32	reserved0 : 6;
-	u32	max_ch_alloc : 5;
-	u32	reserved1 : 3;
-	u32	max_ev_alloc : 5;
-	u32	reserved2 : 11;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_manager_ee_qos_n_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_manager_ee_qos_n_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_shram_ptr_ch_cntxt_base_addr_s {
-	u32	shram_ptr : 16;
-	u32	reserved0 : 16;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_shram_ptr_ch_cntxt_base_addr_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_shram_ptr_ch_cntxt_base_addr_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_shram_ptr_ev_cntxt_base_addr_s {
-	u32	shram_ptr : 16;
-	u32	reserved0 : 16;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_shram_ptr_ev_cntxt_base_addr_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_shram_ptr_ev_cntxt_base_addr_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_shram_ptr_re_storage_base_addr_s {
-	u32	shram_ptr : 16;
-	u32	reserved0 : 16;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_shram_ptr_re_storage_base_addr_u {
-	struct
-	ipa_hwio_def_ipa_gsi_top_gsi_shram_ptr_re_storage_base_addr_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_shram_ptr_re_esc_buf_base_addr_s {
-	u32	shram_ptr : 16;
-	u32	reserved0 : 16;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_shram_ptr_re_esc_buf_base_addr_u {
-	struct
-	ipa_hwio_def_ipa_gsi_top_gsi_shram_ptr_re_esc_buf_base_addr_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_shram_ptr_ee_scrach_base_addr_s {
-	u32	shram_ptr : 16;
-	u32	reserved0 : 16;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_shram_ptr_ee_scrach_base_addr_u {
-	struct
-	ipa_hwio_def_ipa_gsi_top_gsi_shram_ptr_ee_scrach_base_addr_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_shram_ptr_func_stack_base_addr_s {
-	u32	shram_ptr : 16;
-	u32	reserved0 : 16;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_shram_ptr_func_stack_base_addr_u {
-	struct
-	ipa_hwio_def_ipa_gsi_top_gsi_shram_ptr_func_stack_base_addr_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_ch_cmd_s {
-	u32	iram_ptr : 12;
-	u32	reserved0 : 20;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_ch_cmd_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_ch_cmd_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_ee_generic_cmd_s {
-	u32	iram_ptr : 12;
-	u32	reserved0 : 20;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_ee_generic_cmd_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_ee_generic_cmd_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_ch_db_s {
-	u32	iram_ptr : 12;
-	u32	reserved0 : 20;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_ch_db_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_ch_db_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_ev_db_s {
-	u32	iram_ptr : 12;
-	u32	reserved0 : 20;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_ev_db_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_ev_db_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_new_re_s {
-	u32	iram_ptr : 12;
-	u32	reserved0 : 20;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_new_re_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_new_re_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_ch_dis_comp_s {
-	u32	iram_ptr : 12;
-	u32	reserved0 : 20;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_ch_dis_comp_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_ch_dis_comp_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_ch_empty_s {
-	u32	iram_ptr : 12;
-	u32	reserved0 : 20;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_ch_empty_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_ch_empty_s def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_event_gen_comp_s {
-	u32	iram_ptr : 12;
-	u32	reserved0 : 20;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_event_gen_comp_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_event_gen_comp_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_timer_expired_s {
-	u32	iram_ptr : 12;
-	u32	reserved0 : 20;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_timer_expired_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_timer_expired_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_write_eng_comp_s {
-	u32	iram_ptr : 12;
-	u32	reserved0 : 20;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_write_eng_comp_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_write_eng_comp_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_read_eng_comp_s {
-	u32	iram_ptr : 12;
-	u32	reserved0 : 20;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_read_eng_comp_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_read_eng_comp_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_uc_gp_int_s {
-	u32	iram_ptr : 12;
-	u32	reserved0 : 20;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_uc_gp_int_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_uc_gp_int_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_int_mod_stopped_s {
-	u32	iram_ptr : 12;
-	u32	reserved0 : 20;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_int_mod_stopped_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_int_mod_stopped_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_inst_ram_n_s {
-	u32 inst_byte_0 : 8;
-	u32 inst_byte_1 : 8;
-	u32 inst_byte_2 : 8;
-	u32 inst_byte_3 : 8;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_inst_ram_n_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_inst_ram_n_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_shram_n_s {
-	u32 shram : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_shram_n_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_shram_n_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_map_ee_n_ch_k_vp_table_s {
-	u32	phy_ch : 5;
-	u32	valid : 1;
-	u32	reserved0 : 26;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_map_ee_n_ch_k_vp_table_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_map_ee_n_ch_k_vp_table_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_test_bus_sel_s {
-	u32	gsi_testbus_sel : 8;
-	u32	reserved0 : 8;
-	u32	gsi_hw_events_sel : 4;
-	u32	reserved1 : 12;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_test_bus_sel_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_test_bus_sel_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_test_bus_reg_s {
-	u32 gsi_testbus_reg : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_test_bus_reg_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_test_bus_reg_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_debug_busy_reg_s {
-	u32	csr_busy : 1;
-	u32	ree_busy : 1;
-	u32	mcs_busy : 1;
-	u32	timer_busy : 1;
-	u32	rd_wr_busy : 1;
-	u32	ev_eng_busy : 1;
-	u32	int_eng_busy : 1;
-	u32	ree_pwr_clps_busy : 1;
-	u32	db_eng_busy : 1;
-	u32	dbg_cnt_busy : 1;
-	u32	uc_busy : 1;
-	u32	ic_busy : 1;
-	u32	sdma_busy : 1;
-	u32	reserved0 : 19;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_debug_busy_reg_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_debug_busy_reg_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_debug_event_pending_s {
-	u32 chid_bit_map : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_debug_event_pending_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_debug_event_pending_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_debug_timer_pending_s {
-	u32 chid_bit_map : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_debug_timer_pending_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_debug_timer_pending_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_debug_rd_wr_pending_s {
-	u32 chid_bit_map : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_debug_rd_wr_pending_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_debug_rd_wr_pending_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_debug_countern_s {
-	u32	counter_value : 16;
-	u32	reserved0 : 16;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_debug_countern_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_debug_countern_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_debug_pc_from_sw_s {
-	u32	iram_ptr : 12;
-	u32	reserved0 : 20;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_debug_pc_from_sw_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_debug_pc_from_sw_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_debug_sw_stall_s {
-	u32	mcs_stall : 1;
-	u32	reserved0 : 31;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_debug_sw_stall_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_debug_sw_stall_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_debug_pc_for_debug_s {
-	u32	iram_ptr : 12;
-	u32	reserved0 : 20;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_debug_pc_for_debug_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_debug_pc_for_debug_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_debug_qsb_log_err_trns_id_s {
-	u32	err_write : 1;
-	u32	reserved0 : 7;
-	u32	err_tid : 8;
-	u32	err_mid : 8;
-	u32	err_saved : 1;
-	u32	reserved1 : 7;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_debug_qsb_log_err_trns_id_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_debug_qsb_log_err_trns_id_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_debug_sw_rf_n_read_s {
-	u32 rf_reg : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_debug_sw_rf_n_read_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_debug_sw_rf_n_read_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_gsi_debug_ee_n_ev_k_vp_table_s {
-	u32	phy_ev_ch : 5;
-	u32	valid : 1;
-	u32	reserved0 : 26;
-};
-union ipa_hwio_def_ipa_gsi_top_gsi_debug_ee_n_ev_k_vp_table_u {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_debug_ee_n_ev_k_vp_table_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_cntxt_0_s {
-	u32	chtype_protocol : 3;
-	u32	chtype_dir : 1;
-	u32	ee : 4;
-	u32	chid : 5;
-	u32	chtype_protocol_msb : 1;
-	u32	erindex : 5;
-	u32	reserved0 : 1;
-	u32	chstate : 4;
-	u32	element_size : 8;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_cntxt_0_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_cntxt_0_s def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_cntxt_1_s {
-	u32	r_length : 16;
-	u32	reserved0 : 16;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_cntxt_1_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_cntxt_1_s def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_cntxt_2_s {
-	u32 r_base_addr_lsbs : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_cntxt_2_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_cntxt_2_s def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_cntxt_3_s {
-	u32 r_base_addr_msbs : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_cntxt_3_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_cntxt_3_s def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_cntxt_4_s {
-	u32 read_ptr_lsb : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_cntxt_4_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_cntxt_4_s def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_cntxt_5_s {
-	u32 read_ptr_msb : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_cntxt_5_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_cntxt_5_s def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_cntxt_6_s {
-	u32 write_ptr_lsb : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_cntxt_6_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_cntxt_6_s def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_cntxt_7_s {
-	u32 write_ptr_msb : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_cntxt_7_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_cntxt_7_s def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_re_fetch_read_ptr_s {
-	u32	read_ptr : 16;
-	u32	reserved0 : 16;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_re_fetch_read_ptr_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_re_fetch_read_ptr_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_re_fetch_write_ptr_s {
-	u32	re_intr_db : 16;
-	u32	reserved0 : 16;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_re_fetch_write_ptr_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_re_fetch_write_ptr_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_qos_s {
-	u32	wrr_weight : 4;
-	u32	reserved0 : 4;
-	u32	max_prefetch : 1;
-	u32	use_db_eng : 1;
-	u32	prefetch_mode : 4;
-	u32	reserved1 : 2;
-	u32	empty_lvl_thrshold : 8;
-	u32	reserved2 : 8;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_qos_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_qos_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_scratch_0_s {
-	u32 scratch : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_scratch_0_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_scratch_0_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_scratch_1_s {
-	u32 scratch : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_scratch_1_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_scratch_1_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_scratch_2_s {
-	u32 scratch : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_scratch_2_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_scratch_2_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_scratch_3_s {
-	u32 scratch : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_scratch_3_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_gsi_ch_k_scratch_3_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_0_s {
-	u32	chtype : 4;
-	u32	ee : 4;
-	u32	evchid : 8;
-	u32	intype : 1;
-	u32	reserved0 : 3;
-	u32	chstate : 4;
-	u32	element_size : 8;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_0_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_0_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_1_s {
-	u32	r_length : 16;
-	u32	reserved0 : 16;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_1_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_1_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_2_s {
-	u32 r_base_addr_lsbs : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_2_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_2_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_3_s {
-	u32 r_base_addr_msbs : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_3_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_3_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_4_s {
-	u32 read_ptr_lsb : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_4_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_4_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_5_s {
-	u32 read_ptr_msb : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_5_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_5_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_6_s {
-	u32 write_ptr_lsb : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_6_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_6_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_7_s {
-	u32 write_ptr_msb : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_7_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_7_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_8_s {
-	u32	int_modt : 16;
-	u32	int_modc : 8;
-	u32	int_mod_cnt : 8;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_8_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_8_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_9_s {
-	u32 intvec : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_9_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_9_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_10_s {
-	u32 msi_addr_lsb : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_10_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_10_s def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_11_s {
-	u32 msi_addr_msb : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_11_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_11_s def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_12_s {
-	u32 rp_update_addr_lsb : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_12_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_12_s def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_13_s {
-	u32 rp_update_addr_msb : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_13_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_cntxt_13_s def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_scratch_0_s {
-	u32 scratch : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_scratch_0_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_scratch_0_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_scratch_1_s {
-	u32 scratch : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_scratch_1_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_ev_ch_k_scratch_1_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_gsi_status_s {
-	u32	enabled : 1;
-	u32	reserved0 : 31;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_gsi_status_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_gsi_status_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_type_irq_s {
-	u32	ch_ctrl : 1;
-	u32	ev_ctrl : 1;
-	u32	glob_ee : 1;
-	u32	ieob : 1;
-	u32	inter_ee_ch_ctrl : 1;
-	u32	inter_ee_ev_ctrl : 1;
-	u32	general : 1;
-	u32	reserved0 : 25;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_type_irq_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_type_irq_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_type_irq_msk_s {
-	u32	ch_ctrl : 1;
-	u32	ev_ctrl : 1;
-	u32	glob_ee : 1;
-	u32	ieob : 1;
-	u32	inter_ee_ch_ctrl : 1;
-	u32	inter_ee_ev_ctrl : 1;
-	u32	general : 1;
-	u32	reserved0 : 25;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_type_irq_msk_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_type_irq_msk_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_src_gsi_ch_irq_s {
-	u32 gsi_ch_bit_map : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_src_gsi_ch_irq_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_src_gsi_ch_irq_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_src_ev_ch_irq_s {
-	u32 ev_ch_bit_map : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_src_ev_ch_irq_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_src_ev_ch_irq_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_src_gsi_ch_irq_msk_s {
-	u32	gsi_ch_bit_map_msk : 23;
-	u32	reserved0 : 9;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_src_gsi_ch_irq_msk_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_src_gsi_ch_irq_msk_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_src_ev_ch_irq_msk_s {
-	u32	ev_ch_bit_map_msk : 20;
-	u32	reserved0 : 12;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_src_ev_ch_irq_msk_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_src_ev_ch_irq_msk_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_src_gsi_ch_irq_clr_s {
-	u32 gsi_ch_bit_map : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_src_gsi_ch_irq_clr_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_src_gsi_ch_irq_clr_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_src_ev_ch_irq_clr_s {
-	u32 ev_ch_bit_map : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_src_ev_ch_irq_clr_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_src_ev_ch_irq_clr_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_src_ieob_irq_s {
-	u32 ev_ch_bit_map : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_src_ieob_irq_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_src_ieob_irq_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_src_ieob_irq_msk_s {
-	u32	ev_ch_bit_map_msk : 20;
-	u32	reserved0 : 12;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_src_ieob_irq_msk_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_src_ieob_irq_msk_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_src_ieob_irq_clr_s {
-	u32 ev_ch_bit_map : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_src_ieob_irq_clr_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_src_ieob_irq_clr_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_glob_irq_stts_s {
-	u32	error_int : 1;
-	u32	gp_int1 : 1;
-	u32	gp_int2 : 1;
-	u32	gp_int3 : 1;
-	u32	reserved0 : 28;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_glob_irq_stts_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_glob_irq_stts_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_gsi_irq_stts_s {
-	u32	gsi_break_point : 1;
-	u32	gsi_bus_error : 1;
-	u32	gsi_cmd_fifo_ovrflow : 1;
-	u32	gsi_mcs_stack_ovrflow : 1;
-	u32	reserved0 : 28;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_gsi_irq_stts_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_gsi_irq_stts_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_intset_s {
-	u32	intype : 1;
-	u32	reserved0 : 31;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_intset_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_intset_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_msi_base_lsb_s {
-	u32 msi_addr_lsb : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_msi_base_lsb_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_msi_base_lsb_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_msi_base_msb_s {
-	u32 msi_addr_msb : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_msi_base_msb_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_msi_base_msb_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_error_log_s {
-	u32 error_log : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_error_log_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_error_log_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_error_log_clr_s {
-	u32 error_log_clr : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_error_log_clr_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_error_log_clr_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_scratch_0_s {
-	u32 scratch : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_scratch_0_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_scratch_0_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_scratch_1_s {
-	u32 scratch : 32;
-};
-union ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_scratch_1_u {
-	struct ipa_hwio_def_ipa_gsi_top_ee_n_cntxt_scratch_1_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_comp_hw_version_s {
-	u32	step : 16;
-	u32	minor : 12;
-	u32	major : 4;
-};
-union ipa_hwio_def_ipa_comp_hw_version_u {
-	struct ipa_hwio_def_ipa_comp_hw_version_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_comp_cfg_s {
-	u32	reserved0 : 1;
-	u32	gsi_snoc_bypass_dis : 1;
-	u32	gen_qmb_0_snoc_bypass_dis : 1;
-	u32	gen_qmb_1_snoc_bypass_dis : 1;
-	u32	reserved1 : 1;
-	u32	ipa_qmb_select_by_address_cons_en : 1;
-	u32	ipa_qmb_select_by_address_prod_en : 1;
-	u32	gsi_multi_inorder_rd_dis : 1;
-	u32	gsi_multi_inorder_wr_dis : 1;
-	u32	gen_qmb_0_multi_inorder_rd_dis : 1;
-	u32	gen_qmb_1_multi_inorder_rd_dis : 1;
-	u32	gen_qmb_0_multi_inorder_wr_dis : 1;
-	u32	gen_qmb_1_multi_inorder_wr_dis : 1;
-	u32	gen_qmb_0_snoc_cnoc_loop_protection_disable : 1;
-	u32	gsi_snoc_cnoc_loop_protection_disable : 1;
-	u32	gsi_multi_axi_masters_dis : 1;
-	u32	ipa_qmb_select_by_address_global_en : 1;
-	u32	ipa_atomic_fetcher_arb_lock_dis : 4;
-	u32	ipa_full_flush_wait_rsc_closure_en : 1;
-	u32	reserved2 : 10;
-};
-union ipa_hwio_def_ipa_comp_cfg_u {
-	struct ipa_hwio_def_ipa_comp_cfg_s	def;
-	u32					value;
-};
-struct ipa_hwio_def_ipa_route_s {
-	u32	route_dis : 1;
-	u32	route_def_pipe : 5;
-	u32	route_def_hdr_table : 1;
-	u32	route_def_hdr_ofst : 10;
-	u32	route_frag_def_pipe : 5;
-	u32	reserved0 : 2;
-	u32	route_def_retain_hdr : 1;
-	u32	reserved1 : 7;
-};
-union ipa_hwio_def_ipa_route_u {
-	struct ipa_hwio_def_ipa_route_s def;
-	u32				value;
-};
-struct ipa_hwio_def_ipa_proc_iph_cfg_s {
-	u32	iph_threshold : 2;
-	u32	iph_pipelining_disable : 1;
-	u32	reserved0 : 1;
-	u32	status_from_iph_frst_always : 1;
-	u32	iph_nat_blind_invalidate_tport_offset_disable : 1;
-	u32	pipestage_overlap_disable : 1;
-	u32	ftch_dcph_overlap_enable : 1;
-	u32	iph_pkt_parser_protocol_stop_enable : 1;
-	u32	iph_pkt_parser_protocol_stop_hop : 1;
-	u32	iph_pkt_parser_protocol_stop_dest : 1;
-	u32	iph_pkt_parser_ihl_to_2nd_frag_en : 1;
-	u32	reserved1 : 4;
-	u32	iph_pkt_parser_protocol_stop_value : 8;
-	u32	d_dcph_multi_engine_disable : 1;
-	u32	reserved2 : 7;
-};
-union ipa_hwio_def_ipa_proc_iph_cfg_u {
-	struct ipa_hwio_def_ipa_proc_iph_cfg_s	def;
-	u32					value;
-};
-struct ipa_hwio_def_ipa_dpl_timer_lsb_s {
-	u32 tod_lsb : 32;
-};
-union ipa_hwio_def_ipa_dpl_timer_lsb_u {
-	struct ipa_hwio_def_ipa_dpl_timer_lsb_s def;
-	u32					value;
-};
-struct ipa_hwio_def_ipa_dpl_timer_msb_s {
-	u32	tod_msb : 16;
-	u32	reserved0 : 15;
-	u32	timer_en : 1;
-};
-union ipa_hwio_def_ipa_dpl_timer_msb_u {
-	struct ipa_hwio_def_ipa_dpl_timer_msb_s def;
-	u32					value;
-};
-struct ipa_hwio_def_ipa_state_tx_wrapper_s {
-	u32	tx0_idle : 1;
-	u32	tx1_idle : 1;
-	u32	ipa_prod_ackmngr_db_empty : 1;
-	u32	ipa_prod_ackmngr_state_idle : 1;
-	u32	ipa_prod_bresp_empty : 1;
-	u32	ipa_prod_bresp_toggle_idle : 1;
-	u32	ipa_mbim_pkt_fms_idle : 1;
-	u32	mbim_direct_dma : 2;
-	u32	trnseq_force_valid : 1;
-	u32	pkt_drop_cnt_idle : 1;
-	u32	nlo_direct_dma : 2;
-	u32	coal_direct_dma : 2;
-	u32	coal_slave_idle : 1;
-	u32	coal_slave_ctx_idle : 1;
-	u32	reserved0 : 8;
-	u32	coal_slave_open_frame : 4;
-	u32	reserved1 : 3;
-};
-union ipa_hwio_def_ipa_state_tx_wrapper_u {
-	struct ipa_hwio_def_ipa_state_tx_wrapper_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_state_tx1_s {
-	u32	flopped_arbit_type : 3;
-	u32	arbit_type : 3;
-	u32	pa_idle : 1;
-	u32	pa_ctx_idle : 1;
-	u32	pa_rst_idle : 1;
-	u32	pa_pub_cnt_empty : 1;
-	u32	tx_cmd_main_idle : 1;
-	u32	tx_cmd_trnseq_idle : 1;
-	u32	tx_cmd_snif_idle : 1;
-	u32	tx_cmd_bresp_aloc_idle : 1;
-	u32	tx_cmd_bresp_inj_idle : 1;
-	u32	ar_idle : 1;
-	u32	dmaw_idle : 1;
-	u32	dmaw_last_outsd_idle : 1;
-	u32	pf_idle : 1;
-	u32	pf_empty : 1;
-	u32	aligner_empty : 1;
-	u32	holb_idle : 1;
-	u32	holb_mask_idle : 1;
-	u32	rsrcrel_idle : 1;
-	u32	suspend_empty : 1;
-	u32	cs_snif_idle : 1;
-	u32	last_cmd_pipe : 5;
-	u32	suspend_req_empty : 1;
-};
-union ipa_hwio_def_ipa_state_tx1_u {
-	struct ipa_hwio_def_ipa_state_tx1_s	def;
-	u32					value;
-};
-struct ipa_hwio_def_ipa_state_fetcher_s {
-	u32	ipa_hps_ftch_state_idle : 1;
-	u32	ipa_hps_ftch_alloc_state_idle : 1;
-	u32	ipa_hps_ftch_pkt_state_idle : 1;
-	u32	ipa_hps_ftch_imm_state_idle : 1;
-	u32	ipa_hps_ftch_cmplt_state_idle : 1;
-	u32	ipa_hps_dmar_state_idle : 7;
-	u32	ipa_hps_dmar_slot_state_idle : 7;
-	u32	ipa_hps_imm_cmd_exec_state_idle : 1;
-	u32	reserved0 : 12;
-};
-union ipa_hwio_def_ipa_state_fetcher_u {
-	struct ipa_hwio_def_ipa_state_fetcher_s def;
-	u32					value;
-};
-struct ipa_hwio_def_ipa_state_fetcher_mask_0_s {
-	u32	mask_queue_dmar_uses_queue : 8;
-	u32	mask_queue_imm_exec : 8;
-	u32	mask_queue_no_resources_context : 8;
-	u32	mask_queue_no_resources_hps_dmar : 8;
-};
-union ipa_hwio_def_ipa_state_fetcher_mask_0_u {
-	struct ipa_hwio_def_ipa_state_fetcher_mask_0_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_state_fetcher_mask_1_s {
-	u32	mask_queue_no_resources_ack_entry : 8;
-	u32	mask_queue_arb_lock : 8;
-	u32	mask_queue_step_mode : 8;
-	u32	mask_queue_no_space_dpl_fifo : 8;
-};
-union ipa_hwio_def_ipa_state_fetcher_mask_1_u {
-	struct ipa_hwio_def_ipa_state_fetcher_mask_1_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_state_dpl_fifo_s {
-	u32	pop_fsm_state : 3;
-	u32	reserved0 : 29;
-};
-union ipa_hwio_def_ipa_state_dpl_fifo_u {
-	struct ipa_hwio_def_ipa_state_dpl_fifo_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_state_coal_master_s {
-	u32	vp_vld : 4;
-	u32	main_fsm_state : 4;
-	u32	find_open_fsm_state : 4;
-	u32	hash_calc_fsm_state : 4;
-	u32	check_fit_fsm_state : 4;
-	u32	init_vp_fsm_state : 4;
-	u32	lru_vp : 4;
-	u32	vp_timer_expired : 4;
-};
-union ipa_hwio_def_ipa_state_coal_master_u {
-	struct ipa_hwio_def_ipa_state_coal_master_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_state_dfetcher_s {
-	u32	ipa_dps_ftch_pkt_state_idle : 1;
-	u32	ipa_dps_ftch_cmplt_state_idle : 1;
-	u32	reserved0 : 2;
-	u32	ipa_dps_dmar_state_idle : 6;
-	u32	reserved1 : 2;
-	u32	ipa_dps_dmar_slot_state_idle : 6;
-	u32	reserved2 : 14;
-};
-union ipa_hwio_def_ipa_state_dfetcher_u {
-	struct ipa_hwio_def_ipa_state_dfetcher_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_state_acl_s {
-	u32	ipa_hps_h_dcph_empty : 1;
-	u32	ipa_hps_h_dcph_active : 1;
-	u32	ipa_hps_pkt_parser_empty : 1;
-	u32	ipa_hps_pkt_parser_active : 1;
-	u32	ipa_hps_filter_nat_empty : 1;
-	u32	ipa_hps_filter_nat_active : 1;
-	u32	ipa_hps_router_empty : 1;
-	u32	ipa_hps_router_active : 1;
-	u32	ipa_hps_hdri_empty : 1;
-	u32	ipa_hps_hdri_active : 1;
-	u32	ipa_hps_ucp_empty : 1;
-	u32	ipa_hps_ucp_active : 1;
-	u32	ipa_hps_enqueuer_empty : 1;
-	u32	ipa_hps_enqueuer_active : 1;
-	u32	ipa_dps_d_dcph_empty : 1;
-	u32	ipa_dps_d_dcph_active : 1;
-	u32	reserved0 : 2;
-	u32	ipa_dps_dispatcher_empty : 1;
-	u32	ipa_dps_dispatcher_active : 1;
-	u32	ipa_dps_d_dcph_2_empty : 1;
-	u32	ipa_dps_d_dcph_2_active : 1;
-	u32	ipa_hps_sequencer_idle : 1;
-	u32	ipa_dps_sequencer_idle : 1;
-	u32	ipa_dps_d_dcph_2nd_empty : 1;
-	u32	ipa_dps_d_dcph_2nd_active : 1;
-	u32	ipa_hps_coal_master_empty : 1;
-	u32	ipa_hps_coal_master_active : 1;
-	u32	reserved1 : 4;
-};
-union ipa_hwio_def_ipa_state_acl_u {
-	struct ipa_hwio_def_ipa_state_acl_s	def;
-	u32					value;
-};
-struct ipa_hwio_def_ipa_state_gsi_tlv_s {
-	u32	ipa_gsi_toggle_fsm_idle : 1;
-	u32	reserved0 : 31;
-};
-union ipa_hwio_def_ipa_state_gsi_tlv_u {
-	struct ipa_hwio_def_ipa_state_gsi_tlv_s def;
-	u32					value;
-};
-struct ipa_hwio_def_ipa_state_gsi_aos_s {
-	u32	ipa_gsi_aos_fsm_idle : 1;
-	u32	reserved0 : 31;
-};
-union ipa_hwio_def_ipa_state_gsi_aos_u {
-	struct ipa_hwio_def_ipa_state_gsi_aos_s def;
-	u32					value;
-};
-struct ipa_hwio_def_ipa_state_gsi_if_s {
-	u32	ipa_gsi_prod_fsm_tx_0 : 4;
-	u32	ipa_gsi_prod_fsm_tx_1 : 4;
-	u32	reserved0 : 24;
-};
-union ipa_hwio_def_ipa_state_gsi_if_u {
-	struct ipa_hwio_def_ipa_state_gsi_if_s	def;
-	u32					value;
-};
-struct ipa_hwio_def_ipa_state_gsi_skip_s {
-	u32	ipa_gsi_skip_fsm : 2;
-	u32	reserved0 : 30;
-};
-union ipa_hwio_def_ipa_state_gsi_skip_u {
-	struct ipa_hwio_def_ipa_state_gsi_skip_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_state_gsi_if_cons_s {
-	u32	state : 1;
-	u32	cache_vld : 6;
-	u32	rx_req : 10;
-	u32	rx_req_no_zero : 10;
-	u32	reserved0 : 5;
-};
-union ipa_hwio_def_ipa_state_gsi_if_cons_u {
-	struct ipa_hwio_def_ipa_state_gsi_if_cons_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_state_s {
-	u32	rx_wait : 1;
-	u32	rx_idle : 1;
-	u32	tx_idle : 1;
-	u32	dpl_fifo_idle : 1;
-	u32	bam_gsi_idle : 1;
-	u32	ipa_status_sniffer_idle : 1;
-	u32	ipa_noc_idle : 1;
-	u32	aggr_idle : 1;
-	u32	mbim_aggr_idle : 1;
-	u32	ipa_rsrc_mngr_db_empty : 1;
-	u32	ipa_rsrc_state_idle : 1;
-	u32	ipa_ackmngr_db_empty : 1;
-	u32	ipa_ackmngr_state_idle : 1;
-	u32	ipa_tx_ackq_full : 1;
-	u32	ipa_prod_ackmngr_db_empty : 1;
-	u32	ipa_prod_ackmngr_state_idle : 1;
-	u32	ipa_prod_bresp_idle : 1;
-	u32	ipa_full_idle : 1;
-	u32	ipa_ntf_tx_empty : 1;
-	u32	ipa_tx_ackq_empty : 1;
-	u32	ipa_uc_ackq_empty : 1;
-	u32	ipa_rx_ackq_empty : 1;
-	u32	ipa_tx_commander_cmdq_empty : 1;
-	u32	ipa_rx_splt_cmdq_empty : 4;
-	u32	reserved0 : 1;
-	u32	ipa_rx_hps_empty : 1;
-	u32	ipa_hps_dps_empty : 1;
-	u32	ipa_dps_tx_empty : 1;
-	u32	ipa_uc_rx_hnd_cmdq_empty : 1;
-};
-union ipa_hwio_def_ipa_state_u {
-	struct ipa_hwio_def_ipa_state_s def;
-	u32				value;
-};
-struct ipa_hwio_def_ipa_state_rx_active_s {
-	u32	endpoints : 13;
-	u32	reserved0 : 19;
-};
-union ipa_hwio_def_ipa_state_rx_active_u {
-	struct ipa_hwio_def_ipa_state_rx_active_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_state_tx0_s {
-	u32	last_arbit_type : 2;
-	u32	next_arbit_type : 2;
-	u32	pa_idle : 1;
-	u32	pa_ctx_idle : 1;
-	u32	pa_pub_cnt_empty : 1;
-	u32	tx_cmd_main_idle : 1;
-	u32	tx_cmd_trnseq_idle : 1;
-	u32	tx_cmd_snif_idle : 1;
-	u32	tx_cmd_bresp_aloc_idle : 1;
-	u32	tx_cmd_bresp_inj_idle : 1;
-	u32	ar_idle : 1;
-	u32	dmaw_idle : 1;
-	u32	dmaw_last_outsd_idle : 1;
-	u32	pf_idle : 1;
-	u32	pf_empty : 1;
-	u32	aligner_empty : 1;
-	u32	holb_idle : 1;
-	u32	holb_mask_idle : 1;
-	u32	rsrcrel_idle : 1;
-	u32	suspend_empty : 1;
-	u32	cs_snif_idle : 1;
-	u32	last_cmd_pipe : 5;
-	u32	reserved0 : 4;
-};
-union ipa_hwio_def_ipa_state_tx0_u {
-	struct ipa_hwio_def_ipa_state_tx0_s	def;
-	u32					value;
-};
-struct ipa_hwio_def_ipa_state_aggr_active_s {
-	u32	endpoints : 31;
-	u32	reserved0 : 1;
-};
-union ipa_hwio_def_ipa_state_aggr_active_u {
-	struct ipa_hwio_def_ipa_state_aggr_active_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_state_nlo_aggr_s {
-	u32 nlo_aggr_state : 32;
-};
-union ipa_hwio_def_ipa_state_nlo_aggr_u {
-	struct ipa_hwio_def_ipa_state_nlo_aggr_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_state_coal_master_1_s {
-	u32	init_vp_wr_ctx_line : 6;
-	u32	init_vp_rd_pkt_line : 6;
-	u32	init_vp_fsm_state : 4;
-	u32	check_fit_rd_ctx_line : 6;
-	u32	check_fit_fsm_state : 4;
-	u32	arbiter_state : 4;
-	u32	reserved0 : 2;
-};
-union ipa_hwio_def_ipa_state_coal_master_1_u {
-	struct ipa_hwio_def_ipa_state_coal_master_1_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_filt_rout_hash_en_s {
-	u32	ipv6_router_hash_en : 1;
-	u32	reserved0 : 3;
-	u32	ipv6_filter_hash_en : 1;
-	u32	reserved1 : 3;
-	u32	ipv4_router_hash_en : 1;
-	u32	reserved2 : 3;
-	u32	ipv4_filter_hash_en : 1;
-	u32	reserved3 : 19;
-};
-union ipa_hwio_def_ipa_filt_rout_hash_en_u {
-	struct ipa_hwio_def_ipa_filt_rout_hash_en_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_filt_rout_hash_flush_s {
-	u32	ipv6_router_hash_flush : 1;
-	u32	reserved0 : 3;
-	u32	ipv6_filter_hash_flush : 1;
-	u32	reserved1 : 3;
-	u32	ipv4_router_hash_flush : 1;
-	u32	reserved2 : 3;
-	u32	ipv4_filter_hash_flush : 1;
-	u32	reserved3 : 19;
-};
-union ipa_hwio_def_ipa_filt_rout_hash_flush_u {
-	struct ipa_hwio_def_ipa_filt_rout_hash_flush_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_ipv4_filter_init_values_s {
-	u32	ip_v4_filter_init_hashed_addr : 16;
-	u32	ip_v4_filter_init_non_hashed_addr : 16;
-};
-union ipa_hwio_def_ipa_ipv4_filter_init_values_u {
-	struct ipa_hwio_def_ipa_ipv4_filter_init_values_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_ipv6_filter_init_values_s {
-	u32	ip_v6_filter_init_hashed_addr : 16;
-	u32	ip_v6_filter_init_non_hashed_addr : 16;
-};
-union ipa_hwio_def_ipa_ipv6_filter_init_values_u {
-	struct ipa_hwio_def_ipa_ipv6_filter_init_values_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_ipv4_route_init_values_s {
-	u32	ip_v4_route_init_hashed_addr : 16;
-	u32	ip_v4_route_init_non_hashed_addr : 16;
-};
-union ipa_hwio_def_ipa_ipv4_route_init_values_u {
-	struct ipa_hwio_def_ipa_ipv4_route_init_values_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_ipv6_route_init_values_s {
-	u32	ip_v6_route_init_hashed_addr : 16;
-	u32	ip_v6_route_init_non_hashed_addr : 16;
-};
-union ipa_hwio_def_ipa_ipv6_route_init_values_u {
-	struct ipa_hwio_def_ipa_ipv6_route_init_values_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_bam_activated_ports_s {
-	u32	endpoints : 31;
-	u32	reserved0 : 1;
-};
-union ipa_hwio_def_ipa_bam_activated_ports_u {
-	struct ipa_hwio_def_ipa_bam_activated_ports_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_sys_pkt_proc_cntxt_base_s {
-	u32	zero : 3;
-	u32	addr : 29;
-};
-union ipa_hwio_def_ipa_sys_pkt_proc_cntxt_base_u {
-	struct ipa_hwio_def_ipa_sys_pkt_proc_cntxt_base_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_sys_pkt_proc_cntxt_base_msb_s {
-	u32 addr : 32;
-};
-union ipa_hwio_def_ipa_sys_pkt_proc_cntxt_base_msb_u {
-	struct ipa_hwio_def_ipa_sys_pkt_proc_cntxt_base_msb_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_local_pkt_proc_cntxt_base_s {
-	u32	zero : 3;
-	u32	addr : 15;
-	u32	reserved0 : 14;
-};
-union ipa_hwio_def_ipa_local_pkt_proc_cntxt_base_u {
-	struct ipa_hwio_def_ipa_local_pkt_proc_cntxt_base_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_src_rsrc_grp_01_rsrc_type_n_s {
-	u32	src_rsrc_grp_0_min_limit : 6;
-	u32	reserved0 : 2;
-	u32	src_rsrc_grp_0_max_limit : 6;
-	u32	reserved1 : 2;
-	u32	src_rsrc_grp_1_min_limit : 6;
-	u32	reserved2 : 2;
-	u32	src_rsrc_grp_1_max_limit : 6;
-	u32	reserved3 : 2;
-};
-union ipa_hwio_def_ipa_src_rsrc_grp_01_rsrc_type_n_u {
-	struct ipa_hwio_def_ipa_src_rsrc_grp_01_rsrc_type_n_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_src_rsrc_grp_23_rsrc_type_n_s {
-	u32	src_rsrc_grp_2_min_limit : 6;
-	u32	reserved0 : 2;
-	u32	src_rsrc_grp_2_max_limit : 6;
-	u32	reserved1 : 2;
-	u32	src_rsrc_grp_3_min_limit : 6;
-	u32	reserved2 : 2;
-	u32	src_rsrc_grp_3_max_limit : 6;
-	u32	reserved3 : 2;
-};
-union ipa_hwio_def_ipa_src_rsrc_grp_23_rsrc_type_n_u {
-	struct ipa_hwio_def_ipa_src_rsrc_grp_23_rsrc_type_n_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_src_rsrc_grp_45_rsrc_type_n_s {
-	u32 src_rsrc_grp_4_min_limit : 6;
-	u32 reserved0 : 2;
-	u32 src_rsrc_grp_4_max_limit : 6;
-	u32 reserved1 : 18;
-};
-union ipa_hwio_def_ipa_src_rsrc_grp_45_rsrc_type_n_u {
-	struct ipa_hwio_def_ipa_src_rsrc_grp_45_rsrc_type_n_s	def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_src_rsrc_grp_0123_rsrc_type_cnt_n_s {
-	u32	src_rsrc_grp_0_cnt : 6;
-	u32	reserved0 : 2;
-	u32	src_rsrc_grp_1_cnt : 6;
-	u32	reserved1 : 2;
-	u32	src_rsrc_grp_2_cnt : 6;
-	u32	reserved2 : 2;
-	u32	src_rsrc_grp_3_cnt : 6;
-	u32	reserved3 : 2;
-};
-union ipa_hwio_def_ipa_src_rsrc_grp_0123_rsrc_type_cnt_n_u {
-	struct ipa_hwio_def_ipa_src_rsrc_grp_0123_rsrc_type_cnt_n_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_src_rsrc_grp_4567_rsrc_type_cnt_n_s {
-	u32 src_rsrc_grp_4_cnt : 6;
-	u32 reserved0 : 26;
-};
-union ipa_hwio_def_ipa_src_rsrc_grp_4567_rsrc_type_cnt_n_u {
-	struct ipa_hwio_def_ipa_src_rsrc_grp_4567_rsrc_type_cnt_n_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_dst_rsrc_grp_01_rsrc_type_n_s {
-	u32	dst_rsrc_grp_0_min_limit : 6;
-	u32	reserved0 : 2;
-	u32	dst_rsrc_grp_0_max_limit : 6;
-	u32	reserved1 : 2;
-	u32	dst_rsrc_grp_1_min_limit : 6;
-	u32	reserved2 : 2;
-	u32	dst_rsrc_grp_1_max_limit : 6;
-	u32	reserved3 : 2;
-};
-union ipa_hwio_def_ipa_dst_rsrc_grp_01_rsrc_type_n_u {
-	struct ipa_hwio_def_ipa_dst_rsrc_grp_01_rsrc_type_n_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_dst_rsrc_grp_23_rsrc_type_n_s {
-	u32	dst_rsrc_grp_2_min_limit : 6;
-	u32	reserved0 : 2;
-	u32	dst_rsrc_grp_2_max_limit : 6;
-	u32	reserved1 : 2;
-	u32	dst_rsrc_grp_3_min_limit : 6;
-	u32	reserved2 : 2;
-	u32	dst_rsrc_grp_3_max_limit : 6;
-	u32	reserved3 : 2;
-};
-union ipa_hwio_def_ipa_dst_rsrc_grp_23_rsrc_type_n_u {
-	struct ipa_hwio_def_ipa_dst_rsrc_grp_23_rsrc_type_n_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_dst_rsrc_grp_45_rsrc_type_n_s {
-	u32 dst_rsrc_grp_4_min_limit : 6;
-	u32 reserved0 : 2;
-	u32 dst_rsrc_grp_4_max_limit : 6;
-	u32 reserved1 : 18;
-};
-union ipa_hwio_def_ipa_dst_rsrc_grp_45_rsrc_type_n_u {
-	struct ipa_hwio_def_ipa_dst_rsrc_grp_45_rsrc_type_n_s	def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_dst_rsrc_grp_0123_rsrc_type_cnt_n_s {
-	u32	dst_rsrc_grp_0_cnt : 6;
-	u32	reserved0 : 2;
-	u32	dst_rsrc_grp_1_cnt : 6;
-	u32	reserved1 : 2;
-	u32	dst_rsrc_grp_2_cnt : 6;
-	u32	reserved2 : 2;
-	u32	dst_rsrc_grp_3_cnt : 6;
-	u32	reserved3 : 2;
-};
-union ipa_hwio_def_ipa_dst_rsrc_grp_0123_rsrc_type_cnt_n_u {
-	struct ipa_hwio_def_ipa_dst_rsrc_grp_0123_rsrc_type_cnt_n_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_dst_rsrc_grp_4567_rsrc_type_cnt_n_s {
-	u32 dst_rsrc_grp_4_cnt : 8;
-	u32 reserved0 : 24;
-};
-union ipa_hwio_def_ipa_dst_rsrc_grp_4567_rsrc_type_cnt_n_u {
-	struct ipa_hwio_def_ipa_dst_rsrc_grp_4567_rsrc_type_cnt_n_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_rsrc_grp_cfg_s {
-	u32	src_grp_special_valid : 1;
-	u32	reserved0 : 3;
-	u32	src_grp_special_index : 3;
-	u32	reserved1 : 1;
-	u32	dst_pipe_special_valid : 1;
-	u32	reserved2 : 3;
-	u32	dst_pipe_special_index : 5;
-	u32	reserved3 : 3;
-	u32	dst_grp_special_valid : 1;
-	u32	reserved4 : 3;
-	u32	dst_grp_special_index : 6;
-	u32	reserved5 : 2;
-};
-union ipa_hwio_def_ipa_rsrc_grp_cfg_u {
-	struct ipa_hwio_def_ipa_rsrc_grp_cfg_s	def;
-	u32					value;
-};
-struct ipa_hwio_def_ipa_pipeline_disable_s {
-	u32	reserved0 : 3;
-	u32	rx_cmdq_splitter_dis : 1;
-	u32	reserved1 : 28;
-};
-union ipa_hwio_def_ipa_pipeline_disable_u {
-	struct ipa_hwio_def_ipa_pipeline_disable_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_endp_init_ctrl_n_s {
-	u32	endp_suspend : 1;
-	u32	endp_delay : 1;
-	u32	reserved0 : 30;
-};
-union ipa_hwio_def_ipa_endp_init_ctrl_n_u {
-	struct ipa_hwio_def_ipa_endp_init_ctrl_n_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_endp_init_ctrl_scnd_n_s {
-	u32	reserved0 : 1;
-	u32	endp_delay : 1;
-	u32	reserved1 : 30;
-};
-union ipa_hwio_def_ipa_endp_init_ctrl_scnd_n_u {
-	struct ipa_hwio_def_ipa_endp_init_ctrl_scnd_n_s def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_endp_init_cfg_n_s {
-	u32	frag_offload_en : 1;
-	u32	cs_offload_en : 2;
-	u32	cs_metadata_hdr_offset : 4;
-	u32	reserved0 : 1;
-	u32	gen_qmb_master_sel : 1;
-	u32	reserved1 : 23;
-};
-union ipa_hwio_def_ipa_endp_init_cfg_n_u {
-	struct ipa_hwio_def_ipa_endp_init_cfg_n_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_endp_init_nat_n_s {
-	u32	nat_en : 2;
-	u32	reserved0 : 30;
-};
-union ipa_hwio_def_ipa_endp_init_nat_n_u {
-	struct ipa_hwio_def_ipa_endp_init_nat_n_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_endp_init_hdr_n_s {
-	u32	hdr_len : 6;
-	u32	hdr_ofst_metadata_valid : 1;
-	u32	hdr_ofst_metadata : 6;
-	u32	hdr_additional_const_len : 6;
-	u32	hdr_ofst_pkt_size_valid : 1;
-	u32	hdr_ofst_pkt_size : 6;
-	u32	hdr_a5_mux : 1;
-	u32	hdr_len_inc_deagg_hdr : 1;
-	u32	hdr_len_msb : 2;
-	u32	hdr_ofst_metadata_msb : 2;
-};
-union ipa_hwio_def_ipa_endp_init_hdr_n_u {
-	struct ipa_hwio_def_ipa_endp_init_hdr_n_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_endp_init_hdr_ext_n_s {
-	u32	hdr_endianness : 1;
-	u32	hdr_total_len_or_pad_valid : 1;
-	u32	hdr_total_len_or_pad : 1;
-	u32	hdr_payload_len_inc_padding : 1;
-	u32	hdr_total_len_or_pad_offset : 6;
-	u32	hdr_pad_to_alignment : 4;
-	u32	reserved0 : 2;
-	u32	hdr_total_len_or_pad_offset_msb : 2;
-	u32	hdr_ofst_pkt_size_msb : 2;
-	u32	hdr_additional_const_len_msb : 2;
-	u32	reserved1 : 10;
-};
-union ipa_hwio_def_ipa_endp_init_hdr_ext_n_u {
-	struct ipa_hwio_def_ipa_endp_init_hdr_ext_n_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_endp_init_hdr_metadata_mask_n_s {
-	u32 metadata_mask : 32;
-};
-union ipa_hwio_def_ipa_endp_init_hdr_metadata_mask_n_u {
-	struct ipa_hwio_def_ipa_endp_init_hdr_metadata_mask_n_s def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_endp_init_hdr_metadata_n_s {
-	u32 metadata : 32;
-};
-union ipa_hwio_def_ipa_endp_init_hdr_metadata_n_u {
-	struct ipa_hwio_def_ipa_endp_init_hdr_metadata_n_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_endp_init_mode_n_s {
-	u32	mode : 3;
-	u32	dcph_enable : 1;
-	u32	dest_pipe_index : 5;
-	u32	reserved0 : 3;
-	u32	byte_threshold : 16;
-	u32	pipe_replicate_en : 1;
-	u32	pad_en : 1;
-	u32	reserved1 : 2;
-};
-union ipa_hwio_def_ipa_endp_init_mode_n_u {
-	struct ipa_hwio_def_ipa_endp_init_mode_n_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_endp_init_aggr_n_s {
-	u32	aggr_en : 2;
-	u32	aggr_type : 3;
-	u32	aggr_byte_limit : 6;
-	u32	reserved0 : 1;
-	u32	aggr_time_limit : 5;
-	u32	aggr_pkt_limit : 6;
-	u32	aggr_sw_eof_active : 1;
-	u32	aggr_force_close : 1;
-	u32	reserved1 : 1;
-	u32	aggr_hard_byte_limit_enable : 1;
-	u32	aggr_gran_sel : 1;
-	u32	reserved2 : 4;
-};
-union ipa_hwio_def_ipa_endp_init_aggr_n_u {
-	struct ipa_hwio_def_ipa_endp_init_aggr_n_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_endp_init_hol_block_en_n_s {
-	u32	en : 1;
-	u32	reserved0 : 31;
-};
-union ipa_hwio_def_ipa_endp_init_hol_block_en_n_u {
-	struct ipa_hwio_def_ipa_endp_init_hol_block_en_n_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_endp_init_hol_block_timer_n_s {
-	u32	time_limit : 5;
-	u32	reserved0 : 3;
-	u32	gran_sel : 1;
-	u32	reserved1 : 23;
-};
-union ipa_hwio_def_ipa_endp_init_hol_block_timer_n_u {
-	struct ipa_hwio_def_ipa_endp_init_hol_block_timer_n_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_endp_init_deaggr_n_s {
-	u32	deaggr_hdr_len : 6;
-	u32	syspipe_err_detection : 1;
-	u32	packet_offset_valid : 1;
-	u32	packet_offset_location : 6;
-	u32	ignore_min_pkt_err : 1;
-	u32	reserved0 : 1;
-	u32	max_packet_len : 16;
-};
-union ipa_hwio_def_ipa_endp_init_deaggr_n_u {
-	struct ipa_hwio_def_ipa_endp_init_deaggr_n_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_endp_init_rsrc_grp_n_s {
-	u32	rsrc_grp : 3;
-	u32	reserved0 : 29;
-};
-union ipa_hwio_def_ipa_endp_init_rsrc_grp_n_u {
-	struct ipa_hwio_def_ipa_endp_init_rsrc_grp_n_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_endp_init_seq_n_s {
-	u32	hps_seq_type : 4;
-	u32	dps_seq_type : 4;
-	u32	hps_rep_seq_type : 4;
-	u32	dps_rep_seq_type : 4;
-	u32	reserved0 : 16;
-};
-union ipa_hwio_def_ipa_endp_init_seq_n_u {
-	struct ipa_hwio_def_ipa_endp_init_seq_n_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_endp_status_n_s {
-	u32	status_en : 1;
-	u32	status_endp : 5;
-	u32	reserved0 : 3;
-	u32	status_pkt_suppress : 1;
-	u32	reserved1 : 22;
-};
-union ipa_hwio_def_ipa_endp_status_n_u {
-	struct ipa_hwio_def_ipa_endp_status_n_s def;
-	u32					value;
-};
-struct ipa_hwio_def_ipa_endp_filter_router_hsh_cfg_n_s {
-	u32	filter_hash_msk_src_id : 1;
-	u32	filter_hash_msk_src_ip_add : 1;
-	u32	filter_hash_msk_dst_ip_add : 1;
-	u32	filter_hash_msk_src_port : 1;
-	u32	filter_hash_msk_dst_port : 1;
-	u32	filter_hash_msk_protocol : 1;
-	u32	filter_hash_msk_metadata : 1;
-	u32	reserved0 : 9;
-	u32	router_hash_msk_src_id : 1;
-	u32	router_hash_msk_src_ip_add : 1;
-	u32	router_hash_msk_dst_ip_add : 1;
-	u32	router_hash_msk_src_port : 1;
-	u32	router_hash_msk_dst_port : 1;
-	u32	router_hash_msk_protocol : 1;
-	u32	router_hash_msk_metadata : 1;
-	u32	reserved1 : 9;
-};
-union ipa_hwio_def_ipa_endp_filter_router_hsh_cfg_n_u {
-	struct ipa_hwio_def_ipa_endp_filter_router_hsh_cfg_n_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_nlo_pp_cfg1_s {
-	u32	nlo_ack_pp : 8;
-	u32	nlo_data_pp : 8;
-	u32	nlo_status_pp : 8;
-	u32	nlo_ack_max_vp : 6;
-	u32	reserved0 : 2;
-};
-union ipa_hwio_def_ipa_nlo_pp_cfg1_u {
-	struct ipa_hwio_def_ipa_nlo_pp_cfg1_s	def;
-	u32					value;
-};
-struct ipa_hwio_def_ipa_nlo_pp_cfg2_s {
-	u32	nlo_ack_close_padd : 8;
-	u32	nlo_data_close_padd : 8;
-	u32	nlo_ack_buffer_mode : 1;
-	u32	nlo_data_buffer_mode : 1;
-	u32	nlo_status_buffer_mode : 1;
-	u32	reserved0 : 13;
-};
-union ipa_hwio_def_ipa_nlo_pp_cfg2_u {
-	struct ipa_hwio_def_ipa_nlo_pp_cfg2_s	def;
-	u32					value;
-};
-struct ipa_hwio_def_ipa_nlo_pp_ack_limit_cfg_s {
-	u32	nlo_ack_lower_size : 16;
-	u32	nlo_ack_upper_size : 16;
-};
-union ipa_hwio_def_ipa_nlo_pp_ack_limit_cfg_u {
-	struct ipa_hwio_def_ipa_nlo_pp_ack_limit_cfg_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_nlo_pp_data_limit_cfg_s {
-	u32	nlo_data_lower_size : 16;
-	u32	nlo_data_upper_size : 16;
-};
-union ipa_hwio_def_ipa_nlo_pp_data_limit_cfg_u {
-	struct ipa_hwio_def_ipa_nlo_pp_data_limit_cfg_s def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_nlo_min_dsm_cfg_s {
-	u32	nlo_ack_min_dsm_len : 16;
-	u32	nlo_data_min_dsm_len : 16;
-};
-union ipa_hwio_def_ipa_nlo_min_dsm_cfg_u {
-	struct ipa_hwio_def_ipa_nlo_min_dsm_cfg_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_nlo_vp_flush_req_s {
-	u32	vp_flush_pp_indx : 8;
-	u32	reserved0 : 8;
-	u32	vp_flush_vp_indx : 8;
-	u32	reserved1 : 7;
-	u32	vp_flush_req : 1;
-};
-union ipa_hwio_def_ipa_nlo_vp_flush_req_u {
-	struct ipa_hwio_def_ipa_nlo_vp_flush_req_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_nlo_vp_flush_cookie_s {
-	u32 vp_flush_cookie : 32;
-};
-union ipa_hwio_def_ipa_nlo_vp_flush_cookie_u {
-	struct ipa_hwio_def_ipa_nlo_vp_flush_cookie_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_nlo_vp_flush_ack_s {
-	u32	vp_flush_ack : 1;
-	u32	reserved0 : 31;
-};
-union ipa_hwio_def_ipa_nlo_vp_flush_ack_u {
-	struct ipa_hwio_def_ipa_nlo_vp_flush_ack_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_nlo_vp_dsm_open_s {
-	u32 vp_dsm_open : 32;
-};
-union ipa_hwio_def_ipa_nlo_vp_dsm_open_u {
-	struct ipa_hwio_def_ipa_nlo_vp_dsm_open_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_nlo_vp_qbap_open_s {
-	u32 vp_qbap_open : 32;
-};
-union ipa_hwio_def_ipa_nlo_vp_qbap_open_u {
-	struct ipa_hwio_def_ipa_nlo_vp_qbap_open_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_rsrc_mngr_db_cfg_s {
-	u32	rsrc_grp_sel : 3;
-	u32	reserved0 : 1;
-	u32	rsrc_type_sel : 3;
-	u32	reserved1 : 1;
-	u32	rsrc_id_sel : 6;
-	u32	reserved2 : 18;
-};
-union ipa_hwio_def_ipa_rsrc_mngr_db_cfg_u {
-	struct ipa_hwio_def_ipa_rsrc_mngr_db_cfg_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_rsrc_mngr_db_rsrc_read_s {
-	u32	rsrc_occupied : 1;
-	u32	rsrc_next_valid : 1;
-	u32	reserved0 : 2;
-	u32	rsrc_next_index : 6;
-	u32	reserved1 : 22;
-};
-union ipa_hwio_def_ipa_rsrc_mngr_db_rsrc_read_u {
-	struct ipa_hwio_def_ipa_rsrc_mngr_db_rsrc_read_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_rsrc_mngr_db_list_read_s {
-	u32	rsrc_list_valid : 1;
-	u32	rsrc_list_hold : 1;
-	u32	reserved0 : 2;
-	u32	rsrc_list_head_rsrc : 6;
-	u32	reserved1 : 2;
-	u32	rsrc_list_head_cnt : 7;
-	u32	reserved2 : 1;
-	u32	rsrc_list_entry_cnt : 7;
-	u32	reserved3 : 5;
-};
-union ipa_hwio_def_ipa_rsrc_mngr_db_list_read_u {
-	struct ipa_hwio_def_ipa_rsrc_mngr_db_list_read_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_debug_data_s {
-	u32 debug_data : 32;
-};
-union ipa_hwio_def_ipa_debug_data_u {
-	struct ipa_hwio_def_ipa_debug_data_s	def;
-	u32					value;
-};
-struct ipa_hwio_def_ipa_testbus_sel_s {
-	u32	testbus_en : 1;
-	u32	reserved0 : 3;
-	u32	external_block_select : 8;
-	u32	internal_block_select : 8;
-	u32	pipe_select : 5;
-	u32	reserved1 : 7;
-};
-union ipa_hwio_def_ipa_testbus_sel_u {
-	struct ipa_hwio_def_ipa_testbus_sel_s	def;
-	u32					value;
-};
-struct ipa_hwio_def_ipa_step_mode_breakpoints_s {
-	u32 hw_en : 32;
-};
-union ipa_hwio_def_ipa_step_mode_breakpoints_u {
-	struct ipa_hwio_def_ipa_step_mode_breakpoints_s def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_step_mode_status_s {
-	u32 hw_en : 32;
-};
-union ipa_hwio_def_ipa_step_mode_status_u {
-	struct ipa_hwio_def_ipa_step_mode_status_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_log_s {
-	u32	reserved0 : 1;
-	u32	log_en : 1;
-	u32	reserved1 : 2;
-	u32	log_pipe : 5;
-	u32	reserved2 : 3;
-	u32	log_length : 8;
-	u32	log_reduction_en : 1;
-	u32	log_dpl_l2_remove_en : 1;
-	u32	reserved3 : 10;
-};
-union ipa_hwio_def_ipa_log_u {
-	struct ipa_hwio_def_ipa_log_s	def;
-	u32				value;
-};
-struct ipa_hwio_def_ipa_log_buf_hw_cmd_addr_s {
-	u32 start_addr : 32;
-};
-union ipa_hwio_def_ipa_log_buf_hw_cmd_addr_u {
-	struct ipa_hwio_def_ipa_log_buf_hw_cmd_addr_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_log_buf_hw_cmd_addr_msb_s {
-	u32 start_addr : 32;
-};
-union ipa_hwio_def_ipa_log_buf_hw_cmd_addr_msb_u {
-	struct ipa_hwio_def_ipa_log_buf_hw_cmd_addr_msb_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_log_buf_hw_cmd_write_ptr_s {
-	u32 writr_addr : 32;
-};
-union ipa_hwio_def_ipa_log_buf_hw_cmd_write_ptr_u {
-	struct ipa_hwio_def_ipa_log_buf_hw_cmd_write_ptr_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_log_buf_hw_cmd_write_ptr_msb_s {
-	u32 writr_addr : 32;
-};
-union ipa_hwio_def_ipa_log_buf_hw_cmd_write_ptr_msb_u {
-	struct ipa_hwio_def_ipa_log_buf_hw_cmd_write_ptr_msb_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_log_buf_hw_cmd_cfg_s {
-	u32	size : 16;
-	u32	enable : 1;
-	u32	skip_ddr_dma : 1;
-	u32	reserved0 : 14;
-};
-union ipa_hwio_def_ipa_log_buf_hw_cmd_cfg_u {
-	struct ipa_hwio_def_ipa_log_buf_hw_cmd_cfg_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_log_buf_hw_cmd_ram_ptr_s {
-	u32	read_ptr : 14;
-	u32	reserved0 : 2;
-	u32	write_ptr : 14;
-	u32	reserved1 : 1;
-	u32	skip_ddr_wrap_happened : 1;
-};
-union ipa_hwio_def_ipa_log_buf_hw_cmd_ram_ptr_u {
-	struct ipa_hwio_def_ipa_log_buf_hw_cmd_ram_ptr_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_rx_splt_cmdq_cmd_n_s {
-	u32	write_cmd : 1;
-	u32	pop_cmd : 1;
-	u32	release_rd_cmd : 1;
-	u32	release_wr_cmd : 1;
-	u32	release_rd_pkt : 1;
-	u32	release_wr_pkt : 1;
-	u32	release_rd_pkt_enhanced : 1;
-	u32	reserved0 : 25;
-};
-union ipa_hwio_def_ipa_rx_splt_cmdq_cmd_n_u {
-	struct ipa_hwio_def_ipa_rx_splt_cmdq_cmd_n_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_rx_splt_cmdq_cfg_n_s {
-	u32	block_rd : 1;
-	u32	block_wr : 1;
-	u32	reserved0 : 30;
-};
-union ipa_hwio_def_ipa_rx_splt_cmdq_cfg_n_u {
-	struct ipa_hwio_def_ipa_rx_splt_cmdq_cfg_n_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_rx_splt_cmdq_data_wr_0_n_s {
-	u32	cmdq_packet_len_f : 16;
-	u32	cmdq_src_len_f : 16;
-};
-union ipa_hwio_def_ipa_rx_splt_cmdq_data_wr_0_n_u {
-	struct ipa_hwio_def_ipa_rx_splt_cmdq_data_wr_0_n_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_rx_splt_cmdq_data_wr_1_n_s {
-	u32	cmdq_src_pipe_f : 8;
-	u32	cmdq_order_f : 2;
-	u32	cmdq_flags_f : 6;
-	u32	cmdq_opcode_f : 8;
-	u32	cmdq_metadata_f : 8;
-};
-union ipa_hwio_def_ipa_rx_splt_cmdq_data_wr_1_n_u {
-	struct ipa_hwio_def_ipa_rx_splt_cmdq_data_wr_1_n_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_rx_splt_cmdq_data_wr_2_n_s {
-	u32 cmdq_addr_lsb_f : 32;
-};
-union ipa_hwio_def_ipa_rx_splt_cmdq_data_wr_2_n_u {
-	struct ipa_hwio_def_ipa_rx_splt_cmdq_data_wr_2_n_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_rx_splt_cmdq_data_wr_3_n_s {
-	u32 cmdq_addr_msb_f : 32;
-};
-union ipa_hwio_def_ipa_rx_splt_cmdq_data_wr_3_n_u {
-	struct ipa_hwio_def_ipa_rx_splt_cmdq_data_wr_3_n_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_rx_splt_cmdq_data_rd_0_n_s {
-	u32	cmdq_packet_len_f : 16;
-	u32	cmdq_src_len_f : 16;
-};
-union ipa_hwio_def_ipa_rx_splt_cmdq_data_rd_0_n_u {
-	struct ipa_hwio_def_ipa_rx_splt_cmdq_data_rd_0_n_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_rx_splt_cmdq_data_rd_1_n_s {
-	u32	cmdq_src_pipe_f : 8;
-	u32	cmdq_order_f : 2;
-	u32	cmdq_flags_f : 6;
-	u32	cmdq_opcode_f : 8;
-	u32	cmdq_metadata_f : 8;
-};
-union ipa_hwio_def_ipa_rx_splt_cmdq_data_rd_1_n_u {
-	struct ipa_hwio_def_ipa_rx_splt_cmdq_data_rd_1_n_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_rx_splt_cmdq_data_rd_2_n_s {
-	u32 cmdq_addr_lsb_f : 32;
-};
-union ipa_hwio_def_ipa_rx_splt_cmdq_data_rd_2_n_u {
-	struct ipa_hwio_def_ipa_rx_splt_cmdq_data_rd_2_n_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_rx_splt_cmdq_data_rd_3_n_s {
-	u32 cmdq_addr_msb_f : 32;
-};
-union ipa_hwio_def_ipa_rx_splt_cmdq_data_rd_3_n_u {
-	struct ipa_hwio_def_ipa_rx_splt_cmdq_data_rd_3_n_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_rx_splt_cmdq_status_n_s {
-	u32	status : 1;
-	u32	cmdq_empty : 1;
-	u32	cmdq_full : 1;
-	u32	cmdq_count : 2;
-	u32	cmdq_depth : 2;
-	u32	reserved0 : 25;
-};
-union ipa_hwio_def_ipa_rx_splt_cmdq_status_n_u {
-	struct ipa_hwio_def_ipa_rx_splt_cmdq_status_n_s def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_tx_commander_cmdq_status_s {
-	u32	status : 1;
-	u32	cmdq_empty : 1;
-	u32	cmdq_full : 1;
-	u32	reserved0 : 29;
-};
-union ipa_hwio_def_ipa_tx_commander_cmdq_status_u {
-	struct ipa_hwio_def_ipa_tx_commander_cmdq_status_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_rx_hps_cmdq_cmd_s {
-	u32	write_cmd : 1;
-	u32	pop_cmd : 1;
-	u32	cmd_client : 3;
-	u32	rd_req : 1;
-	u32	reserved0 : 26;
-};
-union ipa_hwio_def_ipa_rx_hps_cmdq_cmd_u {
-	struct ipa_hwio_def_ipa_rx_hps_cmdq_cmd_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_rx_hps_cmdq_cfg_wr_s {
-	u32	block_wr : 5;
-	u32	reserved0 : 27;
-};
-union ipa_hwio_def_ipa_rx_hps_cmdq_cfg_wr_u {
-	struct ipa_hwio_def_ipa_rx_hps_cmdq_cfg_wr_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_rx_hps_cmdq_cfg_rd_s {
-	u32	block_rd : 5;
-	u32	reserved0 : 27;
-};
-union ipa_hwio_def_ipa_rx_hps_cmdq_cfg_rd_u {
-	struct ipa_hwio_def_ipa_rx_hps_cmdq_cfg_rd_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_rx_hps_cmdq_data_rd_0_s {
-	u32	cmdq_packet_len_f : 16;
-	u32	cmdq_dest_len_f : 16;
-};
-union ipa_hwio_def_ipa_rx_hps_cmdq_data_rd_0_u {
-	struct ipa_hwio_def_ipa_rx_hps_cmdq_data_rd_0_s def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_rx_hps_cmdq_data_rd_1_s {
-	u32	cmdq_src_pipe_f : 8;
-	u32	cmdq_order_f : 2;
-	u32	cmdq_flags_f : 6;
-	u32	cmdq_opcode_f : 8;
-	u32	cmdq_metadata_f : 8;
-};
-union ipa_hwio_def_ipa_rx_hps_cmdq_data_rd_1_u {
-	struct ipa_hwio_def_ipa_rx_hps_cmdq_data_rd_1_s def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_rx_hps_cmdq_data_rd_2_s {
-	u32 cmdq_addr_lsb_f : 32;
-};
-union ipa_hwio_def_ipa_rx_hps_cmdq_data_rd_2_u {
-	struct ipa_hwio_def_ipa_rx_hps_cmdq_data_rd_2_s def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_rx_hps_cmdq_data_rd_3_s {
-	u32 cmdq_addr_msb_f : 32;
-};
-union ipa_hwio_def_ipa_rx_hps_cmdq_data_rd_3_u {
-	struct ipa_hwio_def_ipa_rx_hps_cmdq_data_rd_3_s def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_rx_hps_cmdq_status_s {
-	u32	status : 1;
-	u32	cmdq_full : 1;
-	u32	cmdq_depth : 7;
-	u32	reserved0 : 23;
-};
-union ipa_hwio_def_ipa_rx_hps_cmdq_status_u {
-	struct ipa_hwio_def_ipa_rx_hps_cmdq_status_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_rx_hps_cmdq_status_empty_s {
-	u32	cmdq_empty : 5;
-	u32	reserved0 : 27;
-};
-union ipa_hwio_def_ipa_rx_hps_cmdq_status_empty_u {
-	struct ipa_hwio_def_ipa_rx_hps_cmdq_status_empty_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_rx_hps_cmdq_count_s {
-	u32	fifo_count : 7;
-	u32	reserved0 : 25;
-};
-union ipa_hwio_def_ipa_rx_hps_cmdq_count_u {
-	struct ipa_hwio_def_ipa_rx_hps_cmdq_count_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_rx_hps_clients_min_depth_0_s {
-	u32	client_0_min_depth : 4;
-	u32	reserved0 : 4;
-	u32	client_1_min_depth : 4;
-	u32	reserved1 : 4;
-	u32	client_2_min_depth : 4;
-	u32	reserved2 : 4;
-	u32	client_3_min_depth : 4;
-	u32	client_4_min_depth : 4;
-};
-union ipa_hwio_def_ipa_rx_hps_clients_min_depth_0_u {
-	struct ipa_hwio_def_ipa_rx_hps_clients_min_depth_0_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_rx_hps_clients_max_depth_0_s {
-	u32	client_0_max_depth : 4;
-	u32	reserved0 : 4;
-	u32	client_1_max_depth : 4;
-	u32	reserved1 : 4;
-	u32	client_2_max_depth : 4;
-	u32	reserved2 : 4;
-	u32	client_3_max_depth : 4;
-	u32	client_4_max_depth : 4;
-};
-union ipa_hwio_def_ipa_rx_hps_clients_max_depth_0_u {
-	struct ipa_hwio_def_ipa_rx_hps_clients_max_depth_0_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_hps_dps_cmdq_cmd_s {
-	u32	write_cmd : 1;
-	u32	pop_cmd : 1;
-	u32	cmd_client : 5;
-	u32	rd_req : 1;
-	u32	reserved0 : 24;
-};
-union ipa_hwio_def_ipa_hps_dps_cmdq_cmd_u {
-	struct ipa_hwio_def_ipa_hps_dps_cmdq_cmd_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_hps_dps_cmdq_data_rd_0_s {
-	u32	cmdq_ctx_id_f : 4;
-	u32	cmdq_src_id_f : 8;
-	u32	cmdq_src_pipe_f : 5;
-	u32	cmdq_opcode_f : 2;
-	u32	cmdq_rep_f : 1;
-	u32	reserved0 : 12;
-};
-union ipa_hwio_def_ipa_hps_dps_cmdq_data_rd_0_u {
-	struct ipa_hwio_def_ipa_hps_dps_cmdq_data_rd_0_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_hps_dps_cmdq_status_s {
-	u32	status : 1;
-	u32	cmdq_full : 1;
-	u32	cmdq_depth : 6;
-	u32	reserved0 : 24;
-};
-union ipa_hwio_def_ipa_hps_dps_cmdq_status_u {
-	struct ipa_hwio_def_ipa_hps_dps_cmdq_status_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_hps_dps_cmdq_status_empty_s {
-	u32	cmdq_empty : 31;
-	u32	reserved0 : 1;
-};
-union ipa_hwio_def_ipa_hps_dps_cmdq_status_empty_u {
-	struct ipa_hwio_def_ipa_hps_dps_cmdq_status_empty_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_hps_dps_cmdq_count_s {
-	u32	fifo_count : 6;
-	u32	reserved0 : 26;
-};
-union ipa_hwio_def_ipa_hps_dps_cmdq_count_u {
-	struct ipa_hwio_def_ipa_hps_dps_cmdq_count_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_dps_tx_cmdq_cmd_s {
-	u32	write_cmd : 1;
-	u32	pop_cmd : 1;
-	u32	cmd_client : 4;
-	u32	reserved0 : 1;
-	u32	rd_req : 1;
-	u32	reserved1 : 24;
-};
-union ipa_hwio_def_ipa_dps_tx_cmdq_cmd_u {
-	struct ipa_hwio_def_ipa_dps_tx_cmdq_cmd_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_dps_tx_cmdq_data_rd_0_s {
-	u32	cmdq_ctx_id_f : 4;
-	u32	cmdq_src_id_f : 8;
-	u32	cmdq_src_pipe_f : 5;
-	u32	cmdq_opcode_f : 2;
-	u32	cmdq_rep_f : 1;
-	u32	reserved0 : 12;
-};
-union ipa_hwio_def_ipa_dps_tx_cmdq_data_rd_0_u {
-	struct ipa_hwio_def_ipa_dps_tx_cmdq_data_rd_0_s def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_dps_tx_cmdq_status_s {
-	u32	status : 1;
-	u32	cmdq_full : 1;
-	u32	cmdq_depth : 7;
-	u32	reserved0 : 23;
-};
-union ipa_hwio_def_ipa_dps_tx_cmdq_status_u {
-	struct ipa_hwio_def_ipa_dps_tx_cmdq_status_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_dps_tx_cmdq_status_empty_s {
-	u32	cmdq_empty : 10;
-	u32	reserved0 : 22;
-};
-union ipa_hwio_def_ipa_dps_tx_cmdq_status_empty_u {
-	struct ipa_hwio_def_ipa_dps_tx_cmdq_status_empty_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_dps_tx_cmdq_count_s {
-	u32	fifo_count : 7;
-	u32	reserved0 : 25;
-};
-union ipa_hwio_def_ipa_dps_tx_cmdq_count_u {
-	struct ipa_hwio_def_ipa_dps_tx_cmdq_count_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_log_buf_hw_snif_el_en_s {
-	u32	bitmap : 3;
-	u32	reserved0 : 29;
-};
-union ipa_hwio_def_ipa_log_buf_hw_snif_el_en_u {
-	struct ipa_hwio_def_ipa_log_buf_hw_snif_el_en_s def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_log_buf_hw_snif_el_wr_n_rd_sel_s {
-	u32	bitmap : 3;
-	u32	reserved0 : 29;
-};
-union ipa_hwio_def_ipa_log_buf_hw_snif_el_wr_n_rd_sel_u {
-	struct ipa_hwio_def_ipa_log_buf_hw_snif_el_wr_n_rd_sel_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_log_buf_hw_snif_el_cli_mux_s {
-	u32	all_cli_mux_concat : 12;
-	u32	reserved0 : 20;
-};
-union ipa_hwio_def_ipa_log_buf_hw_snif_el_cli_mux_u {
-	struct ipa_hwio_def_ipa_log_buf_hw_snif_el_cli_mux_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_log_buf_hw_snif_el_comp_val_0_cli_n_s {
-	u32 value : 32;
-};
-union ipa_hwio_def_ipa_log_buf_hw_snif_el_comp_val_0_cli_n_u {
-	struct ipa_hwio_def_ipa_log_buf_hw_snif_el_comp_val_0_cli_n_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_log_buf_hw_snif_el_comp_val_1_cli_n_s {
-	u32 value : 32;
-};
-union ipa_hwio_def_ipa_log_buf_hw_snif_el_comp_val_1_cli_n_u {
-	struct ipa_hwio_def_ipa_log_buf_hw_snif_el_comp_val_1_cli_n_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_log_buf_hw_snif_el_comp_val_2_cli_n_s {
-	u32 value : 32;
-};
-union ipa_hwio_def_ipa_log_buf_hw_snif_el_comp_val_2_cli_n_u {
-	struct ipa_hwio_def_ipa_log_buf_hw_snif_el_comp_val_2_cli_n_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_log_buf_hw_snif_el_comp_val_3_cli_n_s {
-	u32 value : 32;
-};
-union ipa_hwio_def_ipa_log_buf_hw_snif_el_comp_val_3_cli_n_u {
-	struct ipa_hwio_def_ipa_log_buf_hw_snif_el_comp_val_3_cli_n_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_log_buf_hw_snif_el_mask_val_0_cli_n_s {
-	u32 value : 32;
-};
-union ipa_hwio_def_ipa_log_buf_hw_snif_el_mask_val_0_cli_n_u {
-	struct ipa_hwio_def_ipa_log_buf_hw_snif_el_mask_val_0_cli_n_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_log_buf_hw_snif_el_mask_val_1_cli_n_s {
-	u32 value : 32;
-};
-union ipa_hwio_def_ipa_log_buf_hw_snif_el_mask_val_1_cli_n_u {
-	struct ipa_hwio_def_ipa_log_buf_hw_snif_el_mask_val_1_cli_n_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_log_buf_hw_snif_el_mask_val_2_cli_n_s {
-	u32 value : 32;
-};
-union ipa_hwio_def_ipa_log_buf_hw_snif_el_mask_val_2_cli_n_u {
-	struct ipa_hwio_def_ipa_log_buf_hw_snif_el_mask_val_2_cli_n_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_log_buf_hw_snif_el_mask_val_3_cli_n_s {
-	u32 value : 32;
-};
-union ipa_hwio_def_ipa_log_buf_hw_snif_el_mask_val_3_cli_n_u {
-	struct ipa_hwio_def_ipa_log_buf_hw_snif_el_mask_val_3_cli_n_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_log_buf_hw_snif_legacy_rx_s {
-	u32	src_group_sel : 3;
-	u32	reserved0 : 29;
-};
-union ipa_hwio_def_ipa_log_buf_hw_snif_legacy_rx_u {
-	struct ipa_hwio_def_ipa_log_buf_hw_snif_legacy_rx_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_ackmngr_cmdq_cmd_s {
-	u32	write_cmd : 1;
-	u32	pop_cmd : 1;
-	u32	cmd_client : 5;
-	u32	rd_req : 1;
-	u32	reserved0 : 24;
-};
-union ipa_hwio_def_ipa_ackmngr_cmdq_cmd_u {
-	struct ipa_hwio_def_ipa_ackmngr_cmdq_cmd_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_ackmngr_cmdq_data_rd_s {
-	u32	cmdq_src_id : 8;
-	u32	cmdq_length : 16;
-	u32	cmdq_origin : 1;
-	u32	cmdq_sent : 1;
-	u32	cmdq_src_id_valid : 1;
-	u32	reserved0 : 5;
-};
-union ipa_hwio_def_ipa_ackmngr_cmdq_data_rd_u {
-	struct ipa_hwio_def_ipa_ackmngr_cmdq_data_rd_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_ackmngr_cmdq_status_s {
-	u32	status : 1;
-	u32	cmdq_full : 1;
-	u32	cmdq_depth : 7;
-	u32	reserved0 : 23;
-};
-union ipa_hwio_def_ipa_ackmngr_cmdq_status_u {
-	struct ipa_hwio_def_ipa_ackmngr_cmdq_status_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_ackmngr_cmdq_status_empty_s {
-	u32	cmdq_empty : 13;
-	u32	reserved0 : 19;
-};
-union ipa_hwio_def_ipa_ackmngr_cmdq_status_empty_u {
-	struct ipa_hwio_def_ipa_ackmngr_cmdq_status_empty_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_ackmngr_cmdq_count_s {
-	u32	fifo_count : 7;
-	u32	reserved0 : 25;
-};
-union ipa_hwio_def_ipa_ackmngr_cmdq_count_u {
-	struct ipa_hwio_def_ipa_ackmngr_cmdq_count_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_gsi_fifo_status_ctrl_s {
-	u32	ipa_gsi_fifo_status_port_sel : 5;
-	u32	ipa_gsi_fifo_status_en : 1;
-	u32	reserved0 : 26;
-};
-union ipa_hwio_def_ipa_gsi_fifo_status_ctrl_u {
-	struct ipa_hwio_def_ipa_gsi_fifo_status_ctrl_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_gsi_tlv_fifo_status_s {
-	u32	fifo_wr_ptr : 8;
-	u32	fifo_rd_ptr : 8;
-	u32	fifo_rd_pub_ptr : 8;
-	u32	fifo_empty : 1;
-	u32	fifo_empty_pub : 1;
-	u32	fifo_almost_full : 1;
-	u32	fifo_full : 1;
-	u32	fifo_almost_full_pub : 1;
-	u32	fifo_full_pub : 1;
-	u32	fifo_head_is_bubble : 1;
-	u32	reserved0 : 1;
-};
-union ipa_hwio_def_ipa_gsi_tlv_fifo_status_u {
-	struct ipa_hwio_def_ipa_gsi_tlv_fifo_status_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_gsi_aos_fifo_status_s {
-	u32	fifo_wr_ptr : 8;
-	u32	fifo_rd_ptr : 8;
-	u32	fifo_rd_pub_ptr : 8;
-	u32	fifo_empty : 1;
-	u32	fifo_empty_pub : 1;
-	u32	fifo_almost_full : 1;
-	u32	fifo_full : 1;
-	u32	fifo_almost_full_pub : 1;
-	u32	fifo_full_pub : 1;
-	u32	fifo_head_is_bubble : 1;
-	u32	reserved0 : 1;
-};
-union ipa_hwio_def_ipa_gsi_aos_fifo_status_u {
-	struct ipa_hwio_def_ipa_gsi_aos_fifo_status_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_log_buf_sw_comp_val_0_s {
-	u32 value : 32;
-};
-union ipa_hwio_def_ipa_log_buf_sw_comp_val_0_u {
-	struct ipa_hwio_def_ipa_log_buf_sw_comp_val_0_s def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_log_buf_sw_comp_val_1_s {
-	u32 value : 32;
-};
-union ipa_hwio_def_ipa_log_buf_sw_comp_val_1_u {
-	struct ipa_hwio_def_ipa_log_buf_sw_comp_val_1_s def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_log_buf_sw_comp_val_2_s {
-	u32 value : 32;
-};
-union ipa_hwio_def_ipa_log_buf_sw_comp_val_2_u {
-	struct ipa_hwio_def_ipa_log_buf_sw_comp_val_2_s def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_log_buf_sw_comp_val_3_s {
-	u32 value : 32;
-};
-union ipa_hwio_def_ipa_log_buf_sw_comp_val_3_u {
-	struct ipa_hwio_def_ipa_log_buf_sw_comp_val_3_s def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_log_buf_sw_comp_val_4_s {
-	u32 value : 32;
-};
-union ipa_hwio_def_ipa_log_buf_sw_comp_val_4_u {
-	struct ipa_hwio_def_ipa_log_buf_sw_comp_val_4_s def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_log_buf_sw_comp_val_5_s {
-	u32 value : 32;
-};
-union ipa_hwio_def_ipa_log_buf_sw_comp_val_5_u {
-	struct ipa_hwio_def_ipa_log_buf_sw_comp_val_5_s def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_log_buf_sw_comp_val_6_s {
-	u32 value : 32;
-};
-union ipa_hwio_def_ipa_log_buf_sw_comp_val_6_u {
-	struct ipa_hwio_def_ipa_log_buf_sw_comp_val_6_s def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_log_buf_sw_comp_val_7_s {
-	u32 value : 32;
-};
-union ipa_hwio_def_ipa_log_buf_sw_comp_val_7_u {
-	struct ipa_hwio_def_ipa_log_buf_sw_comp_val_7_s def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_log_buf_sw_mask_val_0_s {
-	u32 value : 32;
-};
-union ipa_hwio_def_ipa_log_buf_sw_mask_val_0_u {
-	struct ipa_hwio_def_ipa_log_buf_sw_mask_val_0_s def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_log_buf_sw_mask_val_1_s {
-	u32 value : 32;
-};
-union ipa_hwio_def_ipa_log_buf_sw_mask_val_1_u {
-	struct ipa_hwio_def_ipa_log_buf_sw_mask_val_1_s def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_log_buf_sw_mask_val_2_s {
-	u32 value : 32;
-};
-union ipa_hwio_def_ipa_log_buf_sw_mask_val_2_u {
-	struct ipa_hwio_def_ipa_log_buf_sw_mask_val_2_s def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_log_buf_sw_mask_val_3_s {
-	u32 value : 32;
-};
-union ipa_hwio_def_ipa_log_buf_sw_mask_val_3_u {
-	struct ipa_hwio_def_ipa_log_buf_sw_mask_val_3_s def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_log_buf_sw_mask_val_4_s {
-	u32 value : 32;
-};
-union ipa_hwio_def_ipa_log_buf_sw_mask_val_4_u {
-	struct ipa_hwio_def_ipa_log_buf_sw_mask_val_4_s def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_log_buf_sw_mask_val_5_s {
-	u32 value : 32;
-};
-union ipa_hwio_def_ipa_log_buf_sw_mask_val_5_u {
-	struct ipa_hwio_def_ipa_log_buf_sw_mask_val_5_s def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_log_buf_sw_mask_val_6_s {
-	u32 value : 32;
-};
-union ipa_hwio_def_ipa_log_buf_sw_mask_val_6_u {
-	struct ipa_hwio_def_ipa_log_buf_sw_mask_val_6_s def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_log_buf_sw_mask_val_7_s {
-	u32 value : 32;
-};
-union ipa_hwio_def_ipa_log_buf_sw_mask_val_7_u {
-	struct ipa_hwio_def_ipa_log_buf_sw_mask_val_7_s def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_ntf_tx_cmdq_cmd_s {
-	u32	write_cmd : 1;
-	u32	pop_cmd : 1;
-	u32	cmd_client : 5;
-	u32	rd_req : 1;
-	u32	reserved0 : 24;
-};
-union ipa_hwio_def_ipa_ntf_tx_cmdq_cmd_u {
-	struct ipa_hwio_def_ipa_ntf_tx_cmdq_cmd_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_ntf_tx_cmdq_data_rd_0_s {
-	u32	cmdq_ctx_id_f : 4;
-	u32	cmdq_src_id_f : 8;
-	u32	cmdq_src_pipe_f : 5;
-	u32	cmdq_opcode_f : 2;
-	u32	cmdq_rep_f : 1;
-	u32	reserved0 : 12;
-};
-union ipa_hwio_def_ipa_ntf_tx_cmdq_data_rd_0_u {
-	struct ipa_hwio_def_ipa_ntf_tx_cmdq_data_rd_0_s def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_ntf_tx_cmdq_status_s {
-	u32	status : 1;
-	u32	cmdq_full : 1;
-	u32	cmdq_depth : 7;
-	u32	reserved0 : 23;
-};
-union ipa_hwio_def_ipa_ntf_tx_cmdq_status_u {
-	struct ipa_hwio_def_ipa_ntf_tx_cmdq_status_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_ntf_tx_cmdq_status_empty_s {
-	u32	cmdq_empty : 31;
-	u32	reserved0 : 1;
-};
-union ipa_hwio_def_ipa_ntf_tx_cmdq_status_empty_u {
-	struct ipa_hwio_def_ipa_ntf_tx_cmdq_status_empty_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_ntf_tx_cmdq_count_s {
-	u32	fifo_count : 7;
-	u32	reserved0 : 25;
-};
-union ipa_hwio_def_ipa_ntf_tx_cmdq_count_u {
-	struct ipa_hwio_def_ipa_ntf_tx_cmdq_count_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_prod_ackmngr_cmdq_cmd_s {
-	u32	write_cmd : 1;
-	u32	pop_cmd : 1;
-	u32	cmd_client : 5;
-	u32	rd_req : 1;
-	u32	reserved0 : 24;
-};
-union ipa_hwio_def_ipa_prod_ackmngr_cmdq_cmd_u {
-	struct ipa_hwio_def_ipa_prod_ackmngr_cmdq_cmd_s def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_prod_ackmngr_cmdq_data_rd_s {
-	u32	cmdq_src_id : 8;
-	u32	cmdq_length : 16;
-	u32	cmdq_origin : 1;
-	u32	cmdq_sent : 1;
-	u32	cmdq_src_id_valid : 1;
-	u32	cmdq_userdata : 5;
-};
-union ipa_hwio_def_ipa_prod_ackmngr_cmdq_data_rd_u {
-	struct ipa_hwio_def_ipa_prod_ackmngr_cmdq_data_rd_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_prod_ackmngr_cmdq_status_s {
-	u32	status : 1;
-	u32	cmdq_full : 1;
-	u32	cmdq_depth : 7;
-	u32	reserved0 : 23;
-};
-union ipa_hwio_def_ipa_prod_ackmngr_cmdq_status_u {
-	struct ipa_hwio_def_ipa_prod_ackmngr_cmdq_status_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_prod_ackmngr_cmdq_status_empty_s {
-	u32	cmdq_empty : 31;
-	u32	reserved0 : 1;
-};
-union ipa_hwio_def_ipa_prod_ackmngr_cmdq_status_empty_u {
-	struct ipa_hwio_def_ipa_prod_ackmngr_cmdq_status_empty_s
-		def;
-	u32 value;
-};
-struct ipa_hwio_def_ipa_prod_ackmngr_cmdq_count_s {
-	u32	fifo_count : 7;
-	u32	reserved0 : 25;
-};
-union ipa_hwio_def_ipa_prod_ackmngr_cmdq_count_u {
-	struct ipa_hwio_def_ipa_prod_ackmngr_cmdq_count_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_spare_reg_1_s {
-	u32	spare_bit0 : 1;
-	u32	spare_bit1 : 1;
-	u32	genqmb_aooowr : 1;
-	u32	spare_bit3 : 1;
-	u32	spare_bit4 : 1;
-	u32	acl_inorder_multi_disable : 1;
-	u32	acl_dispatcher_frag_notif_check_disable : 1;
-	u32	acl_dispatcher_frag_notif_check_each_cmd_disable : 1;
-	u32	spare_bit8 : 1;
-	u32	acl_dispatcher_frag_notif_check_notif_mid_disable : 1;
-	u32	acl_dispatcher_pkt_check_disable : 1;
-	u32	tx_gives_sspnd_ack_on_open_aggr_frame : 1;
-	u32	spare_bit12 : 1;
-	u32	tx_block_aggr_query_on_holb_packet : 1;
-	u32	frag_mngr_fairness_eviction_on_constructing : 1;
-	u32	rx_cmdq_splitter_cmdq_pending_mux_disable : 1;
-	u32	qmb_ram_rd_cache_disable : 1;
-	u32	rx_stall_on_mbim_deaggr_error : 1;
-	u32	rx_stall_on_gen_deaggr_error : 1;
-	u32	spare_bit19 : 1;
-	u32	revert_warb_fix : 1;
-	u32	gsi_if_out_of_buf_stop_reset_mask_enable : 1;
-	u32	bam_idle_in_ipa_misc_cgc_en : 1;
-	u32	spare_bit23 : 1;
-	u32	spare_bit24 : 1;
-	u32	spare_bit25 : 1;
-	u32	ram_slaveway_access_protection_disable : 1;
-	u32	dcph_ram_rd_prefetch_disable : 1;
-	u32	warb_force_arb_round_finish_special_disable : 1;
-	u32	spare_ackinj_pipe8_mask_enable : 1;
-	u32	spare_bit30 : 1;
-	u32	spare_bit31 : 1;
-};
-union ipa_hwio_def_ipa_spare_reg_1_u {
-	struct ipa_hwio_def_ipa_spare_reg_1_s	def;
-	u32					value;
-};
-struct ipa_hwio_def_ipa_spare_reg_2_s {
-	u32	tx_bresp_inj_with_flop : 1;
-	u32	cmdq_split_not_wait_data_desc_prior_hdr_push : 1;
-	u32	spare_bits : 30;
-};
-union ipa_hwio_def_ipa_spare_reg_2_u {
-	struct ipa_hwio_def_ipa_spare_reg_2_s	def;
-	u32					value;
-};
-struct ipa_hwio_def_ipa_endp_gsi_cfg1_n_s {
-	u32	reserved0 : 16;
-	u32	endp_en : 1;
-	u32	reserved1 : 14;
-	u32	init_endp : 1;
-};
-union ipa_hwio_def_ipa_endp_gsi_cfg1_n_u {
-	struct ipa_hwio_def_ipa_endp_gsi_cfg1_n_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_endp_gsi_cfg_tlv_n_s {
-	u32	fifo_base_addr : 16;
-	u32	fifo_size : 8;
-	u32	reserved0 : 8;
-};
-union ipa_hwio_def_ipa_endp_gsi_cfg_tlv_n_u {
-	struct ipa_hwio_def_ipa_endp_gsi_cfg_tlv_n_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_endp_gsi_cfg_aos_n_s {
-	u32	fifo_base_addr : 16;
-	u32	fifo_size : 8;
-	u32	reserved0 : 8;
-};
-union ipa_hwio_def_ipa_endp_gsi_cfg_aos_n_u {
-	struct ipa_hwio_def_ipa_endp_gsi_cfg_aos_n_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_ctxh_ctrl_s {
-	u32	ctxh_lock_id : 4;
-	u32	reserved0 : 27;
-	u32	ctxh_lock : 1;
-};
-union ipa_hwio_def_ipa_ctxh_ctrl_u {
-	struct ipa_hwio_def_ipa_ctxh_ctrl_s	def;
-	u32					value;
-};
-struct ipa_hwio_def_ipa_irq_stts_ee_n_s {
-	u32	bad_snoc_access_irq : 1;
-	u32	reserved0 : 1;
-	u32	uc_irq_0 : 1;
-	u32	uc_irq_1 : 1;
-	u32	uc_irq_2 : 1;
-	u32	uc_irq_3 : 1;
-	u32	uc_in_q_not_empty_irq : 1;
-	u32	uc_rx_cmd_q_not_full_irq : 1;
-	u32	proc_to_uc_ack_q_not_empty_irq : 1;
-	u32	rx_err_irq : 1;
-	u32	deaggr_err_irq : 1;
-	u32	tx_err_irq : 1;
-	u32	step_mode_irq : 1;
-	u32	proc_err_irq : 1;
-	u32	tx_suspend_irq : 1;
-	u32	tx_holb_drop_irq : 1;
-	u32	bam_gsi_idle_irq : 1;
-	u32	pipe_yellow_marker_below_irq : 1;
-	u32	pipe_red_marker_below_irq : 1;
-	u32	pipe_yellow_marker_above_irq : 1;
-	u32	pipe_red_marker_above_irq : 1;
-	u32	ucp_irq : 1;
-	u32	reserved1 : 1;
-	u32	gsi_ee_irq : 1;
-	u32	gsi_ipa_if_tlv_rcvd_irq : 1;
-	u32	gsi_uc_irq : 1;
-	u32	tlv_len_min_dsm_irq : 1;
-	u32	reserved2 : 5;
-};
-union ipa_hwio_def_ipa_irq_stts_ee_n_u {
-	struct ipa_hwio_def_ipa_irq_stts_ee_n_s def;
-	u32					value;
-};
-struct ipa_hwio_def_ipa_irq_en_ee_n_s {
-	u32	bad_snoc_access_irq_en : 1;
-	u32	reserved0 : 1;
-	u32	uc_irq_0_irq_en : 1;
-	u32	uc_irq_1_irq_en : 1;
-	u32	uc_irq_2_irq_en : 1;
-	u32	uc_irq_3_irq_en : 1;
-	u32	uc_in_q_not_empty_irq_en : 1;
-	u32	uc_rx_cmd_q_not_full_irq_en : 1;
-	u32	proc_to_uc_ack_q_not_empty_irq_en : 1;
-	u32	rx_err_irq_en : 1;
-	u32	deaggr_err_irq_en : 1;
-	u32	tx_err_irq_en : 1;
-	u32	step_mode_irq_en : 1;
-	u32	proc_err_irq_en : 1;
-	u32	tx_suspend_irq_en : 1;
-	u32	tx_holb_drop_irq_en : 1;
-	u32	bam_gsi_idle_irq_en : 1;
-	u32	pipe_yellow_marker_below_irq_en : 1;
-	u32	pipe_red_marker_below_irq_en : 1;
-	u32	pipe_yellow_marker_above_irq_en : 1;
-	u32	pipe_red_marker_above_irq_en : 1;
-	u32	ucp_irq_en : 1;
-	u32	reserved1 : 1;
-	u32	gsi_ee_irq_en : 1;
-	u32	gsi_ipa_if_tlv_rcvd_irq_en : 1;
-	u32	gsi_uc_irq_en : 1;
-	u32	tlv_len_min_dsm_irq_en : 1;
-	u32	reserved2 : 5;
-};
-union ipa_hwio_def_ipa_irq_en_ee_n_u {
-	struct ipa_hwio_def_ipa_irq_en_ee_n_s	def;
-	u32					value;
-};
-struct ipa_hwio_def_ipa_snoc_fec_ee_n_s {
-	u32	client : 8;
-	u32	qmb_index : 1;
-	u32	reserved0 : 3;
-	u32	tid : 4;
-	u32	reserved1 : 15;
-	u32	read_not_write : 1;
-};
-union ipa_hwio_def_ipa_snoc_fec_ee_n_u {
-	struct ipa_hwio_def_ipa_snoc_fec_ee_n_s def;
-	u32					value;
-};
-struct ipa_hwio_def_ipa_fec_addr_ee_n_s {
-	u32 addr : 32;
-};
-union ipa_hwio_def_ipa_fec_addr_ee_n_u {
-	struct ipa_hwio_def_ipa_fec_addr_ee_n_s def;
-	u32					value;
-};
-struct ipa_hwio_def_ipa_fec_attr_ee_n_s {
-	u32	opcode : 6;
-	u32	error_info : 26;
-};
-union ipa_hwio_def_ipa_fec_attr_ee_n_u {
-	struct ipa_hwio_def_ipa_fec_attr_ee_n_s def;
-	u32					value;
-};
-struct ipa_hwio_def_ipa_suspend_irq_info_ee_n_s {
-	u32	endpoints : 31;
-	u32	reserved0 : 1;
-};
-union ipa_hwio_def_ipa_suspend_irq_info_ee_n_u {
-	struct ipa_hwio_def_ipa_suspend_irq_info_ee_n_s def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_suspend_irq_en_ee_n_s {
-	u32	endpoints : 31;
-	u32	reserved0 : 1;
-};
-union ipa_hwio_def_ipa_suspend_irq_en_ee_n_u {
-	struct ipa_hwio_def_ipa_suspend_irq_en_ee_n_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_holb_drop_irq_info_ee_n_s {
-	u32	reserved0 : 13;
-	u32	endpoints : 18;
-	u32	reserved1 : 1;
-};
-union ipa_hwio_def_ipa_holb_drop_irq_info_ee_n_u {
-	struct ipa_hwio_def_ipa_holb_drop_irq_info_ee_n_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_log_buf_status_addr_s {
-	u32 start_addr : 32;
-};
-union ipa_hwio_def_ipa_log_buf_status_addr_u {
-	struct ipa_hwio_def_ipa_log_buf_status_addr_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_log_buf_status_addr_msb_s {
-	u32 start_addr : 32;
-};
-union ipa_hwio_def_ipa_log_buf_status_addr_msb_u {
-	struct ipa_hwio_def_ipa_log_buf_status_addr_msb_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_log_buf_status_write_ptr_s {
-	u32 write_addr : 32;
-};
-union ipa_hwio_def_ipa_log_buf_status_write_ptr_u {
-	struct ipa_hwio_def_ipa_log_buf_status_write_ptr_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_log_buf_status_write_ptr_msb_s {
-	u32 write_addr : 32;
-};
-union ipa_hwio_def_ipa_log_buf_status_write_ptr_msb_u {
-	struct ipa_hwio_def_ipa_log_buf_status_write_ptr_msb_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_log_buf_status_cfg_s {
-	u32	size : 16;
-	u32	enable : 1;
-	u32	reserved0 : 15;
-};
-union ipa_hwio_def_ipa_log_buf_status_cfg_u {
-	struct ipa_hwio_def_ipa_log_buf_status_cfg_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_log_buf_status_ram_ptr_s {
-	u32	read_ptr : 16;
-	u32	write_ptr : 16;
-};
-union ipa_hwio_def_ipa_log_buf_status_ram_ptr_u {
-	struct ipa_hwio_def_ipa_log_buf_status_ram_ptr_s	def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_uc_qmb_sys_addr_s {
-	u32 addr : 32;
-};
-union ipa_hwio_def_ipa_uc_qmb_sys_addr_u {
-	struct ipa_hwio_def_ipa_uc_qmb_sys_addr_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_uc_qmb_sys_addr_msb_s {
-	u32 addr_msb : 32;
-};
-union ipa_hwio_def_ipa_uc_qmb_sys_addr_msb_u {
-	struct ipa_hwio_def_ipa_uc_qmb_sys_addr_msb_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_uc_qmb_local_addr_s {
-	u32	addr : 18;
-	u32	reserved0 : 14;
-};
-union ipa_hwio_def_ipa_uc_qmb_local_addr_u {
-	struct ipa_hwio_def_ipa_uc_qmb_local_addr_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_uc_qmb_length_s {
-	u32	length : 7;
-	u32	reserved0 : 25;
-};
-union ipa_hwio_def_ipa_uc_qmb_length_u {
-	struct ipa_hwio_def_ipa_uc_qmb_length_s def;
-	u32					value;
-};
-struct ipa_hwio_def_ipa_uc_qmb_trigger_s {
-	u32	direction : 1;
-	u32	reserved0 : 3;
-	u32	posting : 2;
-	u32	reserved1 : 26;
-};
-union ipa_hwio_def_ipa_uc_qmb_trigger_u {
-	struct ipa_hwio_def_ipa_uc_qmb_trigger_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_uc_qmb_pending_tid_s {
-	u32	tid : 6;
-	u32	reserved0 : 2;
-	u32	error_bus : 1;
-	u32	reserved1 : 3;
-	u32	error_max_os : 1;
-	u32	reserved2 : 3;
-	u32	error_max_comp : 1;
-	u32	reserved3 : 3;
-	u32	error_security : 1;
-	u32	reserved4 : 11;
-};
-union ipa_hwio_def_ipa_uc_qmb_pending_tid_u {
-	struct ipa_hwio_def_ipa_uc_qmb_pending_tid_s	def;
-	u32						value;
-};
-struct ipa_hwio_def_ipa_uc_qmb_completed_rd_fifo_peek_s {
-	u32	tid : 6;
-	u32	reserved0 : 2;
-	u32	error : 1;
-	u32	reserved1 : 3;
-	u32	valid : 1;
-	u32	reserved2 : 19;
-};
-union ipa_hwio_def_ipa_uc_qmb_completed_rd_fifo_peek_u {
-	struct ipa_hwio_def_ipa_uc_qmb_completed_rd_fifo_peek_s def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_uc_qmb_completed_wr_fifo_peek_s {
-	u32	tid : 6;
-	u32	reserved0 : 2;
-	u32	error : 1;
-	u32	reserved1 : 3;
-	u32	valid : 1;
-	u32	reserved2 : 19;
-};
-union ipa_hwio_def_ipa_uc_qmb_completed_wr_fifo_peek_u {
-	struct ipa_hwio_def_ipa_uc_qmb_completed_wr_fifo_peek_s def;
-	u32							value;
-};
-struct ipa_hwio_def_ipa_uc_qmb_misc_s {
-	u32	user : 10;
-	u32	reserved0 : 2;
-	u32	rd_priority : 2;
-	u32	reserved1 : 2;
-	u32	wr_priority : 2;
-	u32	reserved2 : 2;
-	u32	ooord : 1;
-	u32	reserved3 : 3;
-	u32	ooowr : 1;
-	u32	reserved4 : 3;
-	u32	swap : 1;
-	u32	irq_coal : 1;
-	u32	posted_stall : 1;
-	u32	qmb_hready_bcr : 1;
-};
-union ipa_hwio_def_ipa_uc_qmb_misc_u {
-	struct ipa_hwio_def_ipa_uc_qmb_misc_s	def;
-	u32					value;
-};
-struct ipa_hwio_def_ipa_uc_qmb_status_s {
-	u32	max_outstanding_rd : 4;
-	u32	outstanding_rd_cnt : 4;
-	u32	completed_rd_cnt : 4;
-	u32	completed_rd_fifo_full : 1;
-	u32	reserved0 : 3;
-	u32	max_outstanding_wr : 4;
-	u32	outstanding_wr_cnt : 4;
-	u32	completed_wr_cnt : 4;
-	u32	completed_wr_fifo_full : 1;
-	u32	reserved1 : 3;
-};
-union ipa_hwio_def_ipa_uc_qmb_status_u {
-	struct ipa_hwio_def_ipa_uc_qmb_status_s def;
-	u32					value;
-};
-struct ipa_hwio_def_ipa_uc_qmb_bus_attrib_s {
-	u32	memtype : 3;
-	u32	reserved0 : 1;
-	u32	noallocate : 1;
-	u32	reserved1 : 3;
-	u32	innershared : 1;
-	u32	reserved2 : 3;
-	u32	shared : 1;
-	u32	reserved3 : 19;
-};
-union ipa_hwio_def_ipa_uc_qmb_bus_attrib_u {
-	struct ipa_hwio_def_ipa_uc_qmb_bus_attrib_s	def;
-	u32						value;
-};
-#endif

ipa/ipa_v3/dump/ipa4.5/ipa_pkt_cntxt.h

@@ -1,183 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2019, The Linux Foundation. All rights reserved.
- */
-#if !defined(_IPA_PKT_CNTXT_H_)
-#define _IPA_PKT_CNTXT_H_
-
-#define IPA_HW_PKT_CTNTX_MAX        0x10
-#define IPA_HW_NUM_SAVE_PKT_CTNTX   0x8
-#define IPA_HW_PKT_CTNTX_START_ADDR 0xE434CA00
-#define IPA_HW_PKT_CTNTX_SIZE       (sizeof(ipa_pkt_ctntx_opcode_state_s) + \
-				     sizeof(ipa_pkt_ctntx_u))
-
-/*
- * Packet Context States
- */
-enum ipa_hw_pkt_cntxt_state_e {
-	IPA_HW_PKT_CNTXT_STATE_HFETCHER_INIT = 1,
-	IPA_HW_PKT_CNTXT_STATE_HFETCHER_DMAR,
-	IPA_HW_PKT_CNTXT_STATE_HFETCHER_DMAR_REP,
-	IPA_HW_PKT_CNTXT_STATE_H_DCPH,
-	IPA_HW_PKT_CNTXT_STATE_PKT_PARSER,
-	IPA_HW_PKT_CNTXT_STATE_FILTER_NAT,
-	IPA_HW_PKT_CNTXT_STATE_ROUTER,
-	IPA_HW_PKT_CNTXT_STATE_HDRI,
-	IPA_HW_PKT_CNTXT_STATE_UCP,
-	IPA_HW_PKT_CNTXT_STATE_ENQUEUER,
-	IPA_HW_PKT_CNTXT_STATE_DFETCHER,
-	IPA_HW_PKT_CNTXT_STATE_D_DCPH,
-	IPA_HW_PKT_CNTXT_STATE_DISPATCHER,
-	IPA_HW_PKT_CNTXT_STATE_TX,
-	IPA_HW_PKT_CNTXT_STATE_TX_ZLT,
-	IPA_HW_PKT_CNTXT_STATE_DFETCHER_DMAR,
-	IPA_HW_PKT_CNTXT_STATE_DCMP,
-};
-
-/*
- * Packet Context fields as received from VI/Design
- */
-struct ipa_pkt_ctntx_s {
-	u64	opcode                           : 8;
-	u64	state                            : 5;
-	u64	not_used_1                       : 2;
-	u64	tx_pkt_dma_done                  : 1;
-	u64	exc_deagg                        : 1;
-	u64	exc_pkt_version                  : 1;
-	u64	exc_pkt_len                      : 1;
-	u64	exc_threshold                    : 1;
-	u64	exc_sw                           : 1;
-	u64	exc_nat                          : 1;
-	u64	exc_frag_miss                    : 1;
-	u64	filter_bypass                    : 1;
-	u64	router_bypass                    : 1;
-	u64	nat_bypass                       : 1;
-	u64	hdri_bypass                      : 1;
-	u64	dcph_bypass                      : 1;
-	u64	security_credentials_select      : 1;
-	u64	pkt_2nd_pass                     : 1;
-	u64	xlat_bypass                      : 1;
-	u64	dcph_valid                       : 1;
-	u64	ucp_on                           : 1;
-	u64	replication                      : 1;
-	u64	src_status_en                    : 1;
-	u64	dest_status_en                   : 1;
-	u64	frag_status_en                   : 1;
-	u64	eot_dest                         : 1;
-	u64	eot_notif                        : 1;
-	u64	prev_eot_dest                    : 1;
-	u64	src_hdr_len                      : 8;
-	u64	tx_valid_sectors                 : 8;
-	u64	rx_flags                         : 8;
-	u64	rx_packet_length                 : 16;
-	u64	revised_packet_length            : 16;
-	u64	frag_en                          : 1;
-	u64	frag_bypass                      : 1;
-	u64	frag_process                     : 1;
-	u64	notif_pipe                       : 5;
-	u64	src_id                           : 8;
-	u64	tx_pkt_transferred               : 1;
-	u64	src_pipe                         : 5;
-	u64	dest_pipe                        : 5;
-	u64	frag_pipe                        : 5;
-	u64	ihl_offset                       : 8;
-	u64	protocol                         : 8;
-	u64	tos                              : 8;
-	u64	id                               : 16;
-	u64	v6_reserved                      : 4;
-	u64	ff                               : 1;
-	u64	mf                               : 1;
-	u64	pkt_israg                        : 1;
-	u64	tx_holb_timer_overflow           : 1;
-	u64	tx_holb_timer_running            : 1;
-	u64	trnseq_0                         : 3;
-	u64	trnseq_1                         : 3;
-	u64	trnseq_2                         : 3;
-	u64	trnseq_3                         : 3;
-	u64	trnseq_4                         : 3;
-	u64	trnseq_ex_length                 : 8;
-	u64	trnseq_4_length                  : 8;
-	u64	trnseq_4_offset                  : 8;
-	u64	dps_tx_pop_cnt                   : 2;
-	u64	dps_tx_push_cnt                  : 2;
-	u64	vol_ic_dcph_cfg                  : 1;
-	u64	vol_ic_tag_stts                  : 1;
-	u64	vol_ic_pxkt_init_e               : 1;
-	u64	vol_ic_pkt_init                  : 1;
-	u64	tx_holb_counter                  : 32;
-	u64	trnseq_0_length                  : 8;
-	u64	trnseq_0_offset                  : 8;
-	u64	trnseq_1_length                  : 8;
-	u64	trnseq_1_offset                  : 8;
-	u64	trnseq_2_length                  : 8;
-	u64	trnseq_2_offset                  : 8;
-	u64	trnseq_3_length                  : 8;
-	u64	trnseq_3_offset                  : 8;
-	u64	dmar_valid_length                : 16;
-	u64	dcph_valid_length                : 16;
-	u64	frag_hdr_offset                  : 9;
-	u64	ip_payload_offset                : 9;
-	u64	frag_rule                        : 4;
-	u64	frag_table                       : 1;
-	u64	frag_hit                         : 1;
-	u64	data_cmdq_ptr                    : 8;
-	u64	filter_result                    : 6;
-	u64	router_result                    : 6;
-	u64	nat_result                       : 6;
-	u64	hdri_result                      : 6;
-	u64	dcph_result                      : 6;
-	u64	dcph_result_valid                : 1;
-	u32	not_used_2                       : 4;
-	u64	tx_pkt_suspended                 : 1;
-	u64	tx_pkt_dropped                   : 1;
-	u32	not_used_3                       : 3;
-	u64	metadata_valid                   : 1;
-	u64	metadata_type                    : 4;
-	u64	ul_cs_start_diff                 : 9;
-	u64	cs_disable_trlr_vld_bit          : 1;
-	u64	cs_required                      : 1;
-	u64	dest_hdr_len                     : 8;
-	u64	fr_l                             : 1;
-	u64	fl_h                             : 1;
-	u64	fr_g                             : 1;
-	u64	fr_ret                           : 1;
-	u64	fr_rule_id                       : 10;
-	u64	rt_l                             : 1;
-	u64	rt_h                             : 1;
-	u64	rtng_tbl_index                   : 5;
-	u64	rt_match                         : 1;
-	u64	rt_rule_id                       : 10;
-	u64	nat_tbl_index                    : 13;
-	u64	nat_type                         : 2;
-	u64	hdr_l                            : 1;
-	u64	header_offset                    : 10;
-	u64	not_used_4                       : 1;
-	u64	filter_result_valid              : 1;
-	u64	router_result_valid              : 1;
-	u64	nat_result_valid                 : 1;
-	u64	hdri_result_valid                : 1;
-	u64	not_used_5                       : 1;
-	u64	stream_id                        : 8;
-	u64	not_used_6                       : 6;
-	u64	dcph_context_index               : 2;
-	u64	dcph_cfg_size                    : 16;
-	u64	dcph_cfg_count                   : 32;
-	u64	tag_info                         : 48;
-	u64	ucp_cmd_id                       : 16;
-	u64	metadata                         : 32;
-	u64	ucp_cmd_params                   : 32;
-	u64	nat_ip_address                   : 32;
-	u64	nat_ip_cs_diff                   : 16;
-	u64	frag_dest_pipe                   : 5;
-	u64	frag_nat_type                    : 2;
-	u64	fragr_ret                        : 1;
-	u64	frag_protocol                    : 8;
-	u64	src_ip_address                   : 32;
-	u64	dest_ip_address                  : 32;
-	u64	not_used_7                       : 37;
-	u64	frag_hdr_l                       : 1;
-	u64	frag_header_offset               : 10;
-	u64	frag_id                          : 16;
-} __packed;
-
-#endif /* #if !defined(_IPA_PKT_CNTXT_H_) */

ipa/ipa_v3/dump/ipa_reg_dump.c

@@ -1,1632 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2019, The Linux Foundation. All rights reserved.
- */
-#include "ipa_reg_dump.h"
-#include "ipa_access_control.h"
-
-/* Total size required for test bus */
-#define IPA_MEM_OVERLAY_SIZE     0x66000
-
-/*
- * The following structure contains a hierarchy of structures that
- * ultimately leads to a series of leafs. The leafs are structures
- * containing detailed, bit level, register definitions.
- */
-static struct regs_save_hierarchy_s ipa_reg_save;
-
-static unsigned int ipa_testbus_mem[IPA_MEM_OVERLAY_SIZE];
-
-/*
- * The following data structure contains a list of the registers
- * (whose data are to be copied) and the locations (within
- * ipa_reg_save above) into which the registers' values need to be
- * copied.
- */
-static struct map_src_dst_addr_s ipa_regs_to_save_array[] = {
-	/*
-	 * =====================================================================
-	 * IPA register definitions begin here...
-	 * =====================================================================
-	 */
-
-	/* IPA General Registers */
-	GEN_SRC_DST_ADDR_MAP(IPA_STATE,
-			     ipa.gen,
-			     ipa_state),
-	GEN_SRC_DST_ADDR_MAP(IPA_STATE_RX_ACTIVE,
-			     ipa.gen,
-			     ipa_state_rx_active),
-	GEN_SRC_DST_ADDR_MAP(IPA_STATE_TX_WRAPPER,
-			     ipa.gen,
-			     ipa_state_tx_wrapper),
-	GEN_SRC_DST_ADDR_MAP(IPA_STATE_TX0,
-			     ipa.gen,
-			     ipa_state_tx0),
-	GEN_SRC_DST_ADDR_MAP(IPA_STATE_TX1,
-			     ipa.gen,
-			     ipa_state_tx1),
-	GEN_SRC_DST_ADDR_MAP(IPA_STATE_AGGR_ACTIVE,
-			     ipa.gen,
-			     ipa_state_aggr_active),
-	GEN_SRC_DST_ADDR_MAP(IPA_STATE_DFETCHER,
-			     ipa.gen,
-			     ipa_state_dfetcher),
-	GEN_SRC_DST_ADDR_MAP(IPA_STATE_FETCHER_MASK_0,
-			     ipa.gen,
-			     ipa_state_fetcher_mask_0),
-	GEN_SRC_DST_ADDR_MAP(IPA_STATE_FETCHER_MASK_1,
-			     ipa.gen,
-			     ipa_state_fetcher_mask_1),
-	GEN_SRC_DST_ADDR_MAP(IPA_STATE_GSI_AOS,
-			     ipa.gen,
-			     ipa_state_gsi_aos),
-	GEN_SRC_DST_ADDR_MAP(IPA_STATE_GSI_IF,
-			     ipa.gen,
-			     ipa_state_gsi_if),
-	GEN_SRC_DST_ADDR_MAP(IPA_STATE_GSI_SKIP,
-			     ipa.gen,
-			     ipa_state_gsi_skip),
-	GEN_SRC_DST_ADDR_MAP(IPA_STATE_GSI_TLV,
-			     ipa.gen,
-			     ipa_state_gsi_tlv),
-	GEN_SRC_DST_ADDR_MAP(IPA_DPL_TIMER_LSB,
-			     ipa.gen,
-			     ipa_dpl_timer_lsb),
-	GEN_SRC_DST_ADDR_MAP(IPA_DPL_TIMER_MSB,
-			     ipa.gen,
-			     ipa_dpl_timer_msb),
-	GEN_SRC_DST_ADDR_MAP(IPA_PROC_IPH_CFG,
-			     ipa.gen,
-			     ipa_proc_iph_cfg),
-	GEN_SRC_DST_ADDR_MAP(IPA_ROUTE,
-			     ipa.gen,
-			     ipa_route),
-	GEN_SRC_DST_ADDR_MAP(IPA_SPARE_REG_1,
-			     ipa.gen,
-			     ipa_spare_reg_1),
-	GEN_SRC_DST_ADDR_MAP(IPA_SPARE_REG_2,
-			     ipa.gen,
-			     ipa_spare_reg_2),
-	GEN_SRC_DST_ADDR_MAP(IPA_LOG,
-			     ipa.gen,
-			     ipa_log),
-	GEN_SRC_DST_ADDR_MAP(IPA_LOG_BUF_STATUS_CFG,
-			     ipa.gen,
-			     ipa_log_buf_status_cfg),
-	GEN_SRC_DST_ADDR_MAP(IPA_LOG_BUF_STATUS_ADDR,
-			     ipa.gen,
-			     ipa_log_buf_status_addr),
-	GEN_SRC_DST_ADDR_MAP(IPA_LOG_BUF_STATUS_WRITE_PTR,
-			     ipa.gen,
-			     ipa_log_buf_status_write_ptr),
-	GEN_SRC_DST_ADDR_MAP(IPA_LOG_BUF_STATUS_RAM_PTR,
-			     ipa.gen,
-			     ipa_log_buf_status_ram_ptr),
-	GEN_SRC_DST_ADDR_MAP(IPA_LOG_BUF_HW_CMD_CFG,
-			     ipa.gen,
-			     ipa_log_buf_hw_cmd_cfg),
-	GEN_SRC_DST_ADDR_MAP(IPA_LOG_BUF_HW_CMD_ADDR,
-			     ipa.gen,
-			     ipa_log_buf_hw_cmd_addr),
-	GEN_SRC_DST_ADDR_MAP(IPA_LOG_BUF_HW_CMD_WRITE_PTR,
-			     ipa.gen,
-			     ipa_log_buf_hw_cmd_write_ptr),
-	GEN_SRC_DST_ADDR_MAP(IPA_LOG_BUF_HW_CMD_RAM_PTR,
-			     ipa.gen,
-			     ipa_log_buf_hw_cmd_ram_ptr),
-	GEN_SRC_DST_ADDR_MAP(IPA_STATE_DPL_FIFO,
-			     ipa.gen,
-			     ipa_state_dpl_fifo),
-	GEN_SRC_DST_ADDR_MAP(IPA_COMP_HW_VERSION,
-			     ipa.gen,
-			     ipa_comp_hw_version),
-	GEN_SRC_DST_ADDR_MAP(IPA_FILT_ROUT_HASH_EN,
-			     ipa.gen,
-			     ipa_filt_rout_hash_en),
-	GEN_SRC_DST_ADDR_MAP(IPA_FILT_ROUT_HASH_FLUSH,
-			     ipa.gen,
-			     ipa_filt_rout_hash_flush),
-	GEN_SRC_DST_ADDR_MAP(IPA_STATE_FETCHER,
-			     ipa.gen,
-			     ipa_state_fetcher),
-	GEN_SRC_DST_ADDR_MAP(IPA_IPV4_FILTER_INIT_VALUES,
-			     ipa.gen,
-			     ipa_ipv4_filter_init_values),
-	GEN_SRC_DST_ADDR_MAP(IPA_IPV6_FILTER_INIT_VALUES,
-			     ipa.gen,
-			     ipa_ipv6_filter_init_values),
-	GEN_SRC_DST_ADDR_MAP(IPA_IPV4_ROUTE_INIT_VALUES,
-			     ipa.gen,
-			     ipa_ipv4_route_init_values),
-	GEN_SRC_DST_ADDR_MAP(IPA_IPV6_ROUTE_INIT_VALUES,
-			     ipa.gen,
-			     ipa_ipv6_route_init_values),
-	GEN_SRC_DST_ADDR_MAP(IPA_BAM_ACTIVATED_PORTS,
-			     ipa.gen,
-			     ipa_bam_activated_ports),
-	GEN_SRC_DST_ADDR_MAP(IPA_TX_COMMANDER_CMDQ_STATUS,
-			     ipa.gen,
-			     ipa_tx_commander_cmdq_status),
-	GEN_SRC_DST_ADDR_MAP(IPA_LOG_BUF_HW_SNIF_EL_EN,
-			     ipa.gen,
-			     ipa_log_buf_hw_snif_el_en),
-	GEN_SRC_DST_ADDR_MAP(IPA_LOG_BUF_HW_SNIF_EL_WR_N_RD_SEL,
-			     ipa.gen,
-			     ipa_log_buf_hw_snif_el_wr_n_rd_sel),
-	GEN_SRC_DST_ADDR_MAP(IPA_LOG_BUF_HW_SNIF_EL_CLI_MUX,
-			     ipa.gen,
-			     ipa_log_buf_hw_snif_el_cli_mux),
-	GEN_SRC_DST_ADDR_MAP(IPA_STATE_ACL,
-			     ipa.gen,
-			     ipa_state_acl),
-	GEN_SRC_DST_ADDR_MAP(IPA_SYS_PKT_PROC_CNTXT_BASE,
-			     ipa.gen,
-			     ipa_sys_pkt_proc_cntxt_base),
-	GEN_SRC_DST_ADDR_MAP(IPA_SYS_PKT_PROC_CNTXT_BASE_MSB,
-			     ipa.gen,
-			     ipa_sys_pkt_proc_cntxt_base_msb),
-	GEN_SRC_DST_ADDR_MAP(IPA_LOCAL_PKT_PROC_CNTXT_BASE,
-			     ipa.gen,
-			     ipa_local_pkt_proc_cntxt_base),
-	GEN_SRC_DST_ADDR_MAP(IPA_RSRC_GRP_CFG,
-			     ipa.gen,
-			     ipa_rsrc_grp_cfg),
-	GEN_SRC_DST_ADDR_MAP(IPA_PIPELINE_DISABLE,
-			     ipa.gen,
-			     ipa_pipeline_disable),
-	GEN_SRC_DST_ADDR_MAP(IPA_COMP_CFG,
-			     ipa.gen,
-			     ipa_comp_cfg),
-	GEN_SRC_DST_ADDR_MAP(IPA_STATE_NLO_AGGR,
-			     ipa.gen,
-			     ipa_state_nlo_aggr),
-	GEN_SRC_DST_ADDR_MAP(IPA_NLO_PP_CFG1,
-			     ipa.gen,
-			     ipa_nlo_pp_cfg1),
-	GEN_SRC_DST_ADDR_MAP(IPA_NLO_PP_CFG2,
-			     ipa.gen,
-			     ipa_nlo_pp_cfg2),
-	GEN_SRC_DST_ADDR_MAP(IPA_NLO_PP_ACK_LIMIT_CFG,
-			     ipa.gen,
-			     ipa_nlo_pp_ack_limit_cfg),
-	GEN_SRC_DST_ADDR_MAP(IPA_NLO_PP_DATA_LIMIT_CFG,
-			     ipa.gen,
-			     ipa_nlo_pp_data_limit_cfg),
-	GEN_SRC_DST_ADDR_MAP(IPA_NLO_MIN_DSM_CFG,
-			     ipa.gen,
-			     ipa_nlo_min_dsm_cfg),
-	GEN_SRC_DST_ADDR_MAP(IPA_NLO_VP_FLUSH_REQ,
-			     ipa.gen,
-			     ipa_nlo_vp_flush_req),
-	GEN_SRC_DST_ADDR_MAP(IPA_NLO_VP_FLUSH_COOKIE,
-			     ipa.gen,
-			     ipa_nlo_vp_flush_cookie),
-	GEN_SRC_DST_ADDR_MAP(IPA_NLO_VP_FLUSH_ACK,
-			     ipa.gen,
-			     ipa_nlo_vp_flush_ack),
-	GEN_SRC_DST_ADDR_MAP(IPA_NLO_VP_DSM_OPEN,
-			     ipa.gen,
-			     ipa_nlo_vp_dsm_open),
-	GEN_SRC_DST_ADDR_MAP(IPA_NLO_VP_QBAP_OPEN,
-			     ipa.gen,
-			     ipa_nlo_vp_qbap_open),
-
-	/* Debug Registers */
-	GEN_SRC_DST_ADDR_MAP(IPA_DEBUG_DATA,
-			     ipa.dbg,
-			     ipa_debug_data),
-	GEN_SRC_DST_ADDR_MAP(IPA_STEP_MODE_BREAKPOINTS,
-			     ipa.dbg,
-			     ipa_step_mode_breakpoints),
-	GEN_SRC_DST_ADDR_MAP(IPA_STEP_MODE_STATUS,
-			     ipa.dbg,
-			     ipa_step_mode_status),
-
-	IPA_REG_SAVE_RX_SPLT_CMDQ(
-		IPA_RX_SPLT_CMDQ_CMD_n, ipa_rx_splt_cmdq_cmd_n),
-	IPA_REG_SAVE_RX_SPLT_CMDQ(
-		IPA_RX_SPLT_CMDQ_CFG_n, ipa_rx_splt_cmdq_cfg_n),
-	IPA_REG_SAVE_RX_SPLT_CMDQ(
-		IPA_RX_SPLT_CMDQ_DATA_WR_0_n, ipa_rx_splt_cmdq_data_wr_0_n),
-	IPA_REG_SAVE_RX_SPLT_CMDQ(
-		IPA_RX_SPLT_CMDQ_DATA_WR_1_n, ipa_rx_splt_cmdq_data_wr_1_n),
-	IPA_REG_SAVE_RX_SPLT_CMDQ(
-		IPA_RX_SPLT_CMDQ_DATA_WR_2_n, ipa_rx_splt_cmdq_data_wr_2_n),
-	IPA_REG_SAVE_RX_SPLT_CMDQ(
-		IPA_RX_SPLT_CMDQ_DATA_WR_3_n, ipa_rx_splt_cmdq_data_wr_3_n),
-	IPA_REG_SAVE_RX_SPLT_CMDQ(
-		IPA_RX_SPLT_CMDQ_DATA_RD_0_n, ipa_rx_splt_cmdq_data_rd_0_n),
-	IPA_REG_SAVE_RX_SPLT_CMDQ(
-		IPA_RX_SPLT_CMDQ_DATA_RD_1_n, ipa_rx_splt_cmdq_data_rd_1_n),
-	IPA_REG_SAVE_RX_SPLT_CMDQ(
-		IPA_RX_SPLT_CMDQ_DATA_RD_2_n, ipa_rx_splt_cmdq_data_rd_2_n),
-	IPA_REG_SAVE_RX_SPLT_CMDQ(
-		IPA_RX_SPLT_CMDQ_DATA_RD_3_n, ipa_rx_splt_cmdq_data_rd_3_n),
-	IPA_REG_SAVE_RX_SPLT_CMDQ(
-		IPA_RX_SPLT_CMDQ_STATUS_n, ipa_rx_splt_cmdq_status_n),
-
-	GEN_SRC_DST_ADDR_MAP(IPA_RX_HPS_CMDQ_CFG_WR,
-				  ipa.dbg,
-				  ipa_rx_hps_cmdq_cfg_wr),
-	GEN_SRC_DST_ADDR_MAP(IPA_RX_HPS_CMDQ_CFG_RD,
-				  ipa.dbg,
-				  ipa_rx_hps_cmdq_cfg_rd),
-	GEN_SRC_DST_ADDR_MAP(IPA_RX_HPS_CMDQ_CMD,
-			     ipa.dbg,
-			     ipa_rx_hps_cmdq_cmd),
-	GEN_SRC_DST_ADDR_MAP(IPA_RX_HPS_CMDQ_STATUS_EMPTY,
-			     ipa.dbg,
-			     ipa_rx_hps_cmdq_status_empty),
-	GEN_SRC_DST_ADDR_MAP(IPA_RX_HPS_CLIENTS_MIN_DEPTH_0,
-			     ipa.dbg,
-			     ipa_rx_hps_clients_min_depth_0),
-	GEN_SRC_DST_ADDR_MAP(IPA_RX_HPS_CLIENTS_MAX_DEPTH_0,
-			     ipa.dbg,
-			     ipa_rx_hps_clients_max_depth_0),
-	GEN_SRC_DST_ADDR_MAP(IPA_HPS_DPS_CMDQ_CMD,
-			     ipa.dbg,
-			     ipa_hps_dps_cmdq_cmd),
-	GEN_SRC_DST_ADDR_MAP(IPA_HPS_DPS_CMDQ_STATUS_EMPTY,
-			     ipa.dbg,
-			     ipa_hps_dps_cmdq_status_empty),
-	GEN_SRC_DST_ADDR_MAP(IPA_DPS_TX_CMDQ_CMD,
-			     ipa.dbg,
-			     ipa_dps_tx_cmdq_cmd),
-	GEN_SRC_DST_ADDR_MAP(IPA_DPS_TX_CMDQ_STATUS_EMPTY,
-			     ipa.dbg,
-			     ipa_dps_tx_cmdq_status_empty),
-	GEN_SRC_DST_ADDR_MAP(IPA_ACKMNGR_CMDQ_CMD,
-			     ipa.dbg,
-			     ipa_ackmngr_cmdq_cmd),
-	GEN_SRC_DST_ADDR_MAP(IPA_ACKMNGR_CMDQ_STATUS_EMPTY,
-			     ipa.dbg,
-			     ipa_ackmngr_cmdq_status_empty),
-
-	/*
-	 * NOTE: That GEN_SRC_DST_ADDR_MAP() not used below.  This is
-	 *       because the following registers are not scaler, rather
-	 *       they are register arrays...
-	 */
-	IPA_REG_SAVE_CFG_ENTRY_GEN_EE(IPA_IRQ_STTS_EE_n,
-				      ipa_irq_stts_ee_n),
-	IPA_REG_SAVE_CFG_ENTRY_GEN_EE(IPA_IRQ_EN_EE_n,
-				      ipa_irq_en_ee_n),
-	IPA_REG_SAVE_CFG_ENTRY_GEN_EE(IPA_FEC_ADDR_EE_n,
-				      ipa_fec_addr_ee_n),
-	IPA_REG_SAVE_CFG_ENTRY_GEN_EE(IPA_FEC_ATTR_EE_n,
-				      ipa_fec_attr_ee_n),
-	IPA_REG_SAVE_CFG_ENTRY_GEN_EE(IPA_SNOC_FEC_EE_n,
-				      ipa_snoc_fec_ee_n),
-	IPA_REG_SAVE_CFG_ENTRY_GEN_EE(IPA_HOLB_DROP_IRQ_INFO_EE_n,
-				      ipa_holb_drop_irq_info_ee_n),
-	IPA_REG_SAVE_CFG_ENTRY_GEN_EE(IPA_SUSPEND_IRQ_INFO_EE_n,
-				      ipa_suspend_irq_info_ee_n),
-	IPA_REG_SAVE_CFG_ENTRY_GEN_EE(IPA_SUSPEND_IRQ_EN_EE_n,
-				      ipa_suspend_irq_en_ee_n),
-
-	/* Pipe Endp Registers */
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP(IPA_ENDP_INIT_CTRL_n,
-					 ipa_endp_init_ctrl_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP(IPA_ENDP_INIT_CTRL_SCND_n,
-					 ipa_endp_init_ctrl_scnd_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP(IPA_ENDP_INIT_CFG_n,
-					 ipa_endp_init_cfg_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP(IPA_ENDP_INIT_NAT_n,
-					 ipa_endp_init_nat_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP(IPA_ENDP_INIT_HDR_n,
-					 ipa_endp_init_hdr_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP(IPA_ENDP_INIT_HDR_EXT_n,
-					 ipa_endp_init_hdr_ext_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP(IPA_ENDP_INIT_HDR_METADATA_MASK_n,
-					 ipa_endp_init_hdr_metadata_mask_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP(IPA_ENDP_INIT_HDR_METADATA_n,
-					 ipa_endp_init_hdr_metadata_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP(IPA_ENDP_INIT_MODE_n,
-					 ipa_endp_init_mode_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP(IPA_ENDP_INIT_AGGR_n,
-					 ipa_endp_init_aggr_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP(IPA_ENDP_INIT_HOL_BLOCK_EN_n,
-					 ipa_endp_init_hol_block_en_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP(IPA_ENDP_INIT_HOL_BLOCK_TIMER_n,
-					 ipa_endp_init_hol_block_timer_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP(IPA_ENDP_INIT_DEAGGR_n,
-					 ipa_endp_init_deaggr_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP(IPA_ENDP_STATUS_n,
-					 ipa_endp_status_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP(IPA_ENDP_INIT_RSRC_GRP_n,
-					 ipa_endp_init_rsrc_grp_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP(IPA_ENDP_INIT_SEQ_n,
-					 ipa_endp_init_seq_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP(IPA_ENDP_GSI_CFG_TLV_n,
-					 ipa_endp_gsi_cfg_tlv_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP(IPA_ENDP_GSI_CFG_AOS_n,
-					 ipa_endp_gsi_cfg_aos_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP(IPA_ENDP_GSI_CFG1_n,
-					 ipa_endp_gsi_cfg1_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP(IPA_ENDP_FILTER_ROUTER_HSH_CFG_n,
-					 ipa_endp_filter_router_hsh_cfg_n),
-
-	/* Source Resource Group Config Registers */
-	IPA_REG_SAVE_CFG_ENTRY_SRC_RSRC_GRP(IPA_SRC_RSRC_GRP_01_RSRC_TYPE_n,
-					    ipa_src_rsrc_grp_01_rsrc_type_n),
-	IPA_REG_SAVE_CFG_ENTRY_SRC_RSRC_GRP(IPA_SRC_RSRC_GRP_23_RSRC_TYPE_n,
-					    ipa_src_rsrc_grp_23_rsrc_type_n),
-	IPA_REG_SAVE_CFG_ENTRY_SRC_RSRC_GRP(IPA_SRC_RSRC_GRP_45_RSRC_TYPE_n,
-					    ipa_src_rsrc_grp_45_rsrc_type_n),
-
-	/* Destination Resource Group Config Registers */
-	IPA_REG_SAVE_CFG_ENTRY_DST_RSRC_GRP(IPA_DST_RSRC_GRP_01_RSRC_TYPE_n,
-					    ipa_dst_rsrc_grp_01_rsrc_type_n),
-	IPA_REG_SAVE_CFG_ENTRY_DST_RSRC_GRP(IPA_DST_RSRC_GRP_23_RSRC_TYPE_n,
-					    ipa_dst_rsrc_grp_23_rsrc_type_n),
-	IPA_REG_SAVE_CFG_ENTRY_DST_RSRC_GRP(IPA_DST_RSRC_GRP_45_RSRC_TYPE_n,
-					    ipa_dst_rsrc_grp_45_rsrc_type_n),
-
-	/* Source Resource Group Count Registers */
-	IPA_REG_SAVE_CFG_ENTRY_SRC_RSRC_CNT_GRP(
-		IPA_SRC_RSRC_GRP_0123_RSRC_TYPE_CNT_n,
-		ipa_src_rsrc_grp_0123_rsrc_type_cnt_n),
-	IPA_REG_SAVE_CFG_ENTRY_SRC_RSRC_CNT_GRP(
-		IPA_SRC_RSRC_GRP_4567_RSRC_TYPE_CNT_n,
-		ipa_src_rsrc_grp_4567_rsrc_type_cnt_n),
-
-	/* Destination Resource Group Count Registers */
-	IPA_REG_SAVE_CFG_ENTRY_DST_RSRC_CNT_GRP(
-		IPA_DST_RSRC_GRP_0123_RSRC_TYPE_CNT_n,
-		ipa_dst_rsrc_grp_0123_rsrc_type_cnt_n),
-	IPA_REG_SAVE_CFG_ENTRY_DST_RSRC_CNT_GRP(
-		IPA_DST_RSRC_GRP_4567_RSRC_TYPE_CNT_n,
-		ipa_dst_rsrc_grp_4567_rsrc_type_cnt_n),
-
-	/*
-	 * =====================================================================
-	 * GSI register definitions begin here...
-	 * =====================================================================
-	 */
-
-	/* GSI General Registers */
-	GEN_SRC_DST_ADDR_MAP(GSI_CFG,
-			     gsi.gen,
-			     gsi_cfg),
-	GEN_SRC_DST_ADDR_MAP(GSI_REE_CFG,
-			     gsi.gen,
-			     gsi_ree_cfg),
-	IPA_REG_SAVE_GSI_VER(
-			     IPA_GSI_TOP_GSI_INST_RAM_n,
-			     ipa_gsi_top_gsi_inst_ram_n),
-
-	/* GSI Debug Registers */
-	GEN_SRC_DST_ADDR_MAP(IPA_GSI_TOP_GSI_DEBUG_BUSY_REG,
-			     gsi.debug,
-			     ipa_gsi_top_gsi_debug_busy_reg),
-	GEN_SRC_DST_ADDR_MAP(IPA_GSI_TOP_GSI_DEBUG_EVENT_PENDING,
-			     gsi.debug,
-			     ipa_gsi_top_gsi_debug_event_pending),
-	GEN_SRC_DST_ADDR_MAP(IPA_GSI_TOP_GSI_DEBUG_TIMER_PENDING,
-			     gsi.debug,
-			     ipa_gsi_top_gsi_debug_timer_pending),
-	GEN_SRC_DST_ADDR_MAP(IPA_GSI_TOP_GSI_DEBUG_RD_WR_PENDING,
-			     gsi.debug,
-			     ipa_gsi_top_gsi_debug_rd_wr_pending),
-	GEN_SRC_DST_ADDR_MAP(IPA_GSI_TOP_GSI_DEBUG_PC_FROM_SW,
-			     gsi.debug,
-			     ipa_gsi_top_gsi_debug_pc_from_sw),
-	GEN_SRC_DST_ADDR_MAP(IPA_GSI_TOP_GSI_DEBUG_SW_STALL,
-			     gsi.debug,
-			     ipa_gsi_top_gsi_debug_sw_stall),
-	GEN_SRC_DST_ADDR_MAP(IPA_GSI_TOP_GSI_DEBUG_PC_FOR_DEBUG,
-			     gsi.debug,
-			     ipa_gsi_top_gsi_debug_pc_for_debug),
-	GEN_SRC_DST_ADDR_MAP(IPA_GSI_TOP_GSI_DEBUG_QSB_LOG_ERR_TRNS_ID,
-			     gsi.debug,
-			     ipa_gsi_top_gsi_debug_qsb_log_err_trns_id),
-
-	IPA_REG_SAVE_CFG_ENTRY_GSI_QSB_DEBUG(
-		GSI_DEBUG_QSB_LOG_LAST_MISC_IDn, qsb_log_last_misc),
-
-	/* GSI IRAM pointers Registers */
-	GEN_SRC_DST_ADDR_MAP(IPA_GSI_TOP_GSI_IRAM_PTR_CH_CMD,
-			     gsi.debug.gsi_iram_ptrs,
-			     ipa_gsi_top_gsi_iram_ptr_ch_cmd),
-	GEN_SRC_DST_ADDR_MAP(IPA_GSI_TOP_GSI_IRAM_PTR_EE_GENERIC_CMD,
-			     gsi.debug.gsi_iram_ptrs,
-			     ipa_gsi_top_gsi_iram_ptr_ee_generic_cmd),
-	GEN_SRC_DST_ADDR_MAP(IPA_GSI_TOP_GSI_IRAM_PTR_CH_DB,
-			     gsi.debug.gsi_iram_ptrs,
-			     ipa_gsi_top_gsi_iram_ptr_ch_db),
-	GEN_SRC_DST_ADDR_MAP(IPA_GSI_TOP_GSI_IRAM_PTR_EV_DB,
-			     gsi.debug.gsi_iram_ptrs,
-			     ipa_gsi_top_gsi_iram_ptr_ev_db),
-	GEN_SRC_DST_ADDR_MAP(IPA_GSI_TOP_GSI_IRAM_PTR_NEW_RE,
-			     gsi.debug.gsi_iram_ptrs,
-			     ipa_gsi_top_gsi_iram_ptr_new_re),
-	GEN_SRC_DST_ADDR_MAP(IPA_GSI_TOP_GSI_IRAM_PTR_CH_DIS_COMP,
-			     gsi.debug.gsi_iram_ptrs,
-			     ipa_gsi_top_gsi_iram_ptr_ch_dis_comp),
-	GEN_SRC_DST_ADDR_MAP(IPA_GSI_TOP_GSI_IRAM_PTR_CH_EMPTY,
-			     gsi.debug.gsi_iram_ptrs,
-			     ipa_gsi_top_gsi_iram_ptr_ch_empty),
-	GEN_SRC_DST_ADDR_MAP(IPA_GSI_TOP_GSI_IRAM_PTR_EVENT_GEN_COMP,
-			     gsi.debug.gsi_iram_ptrs,
-			     ipa_gsi_top_gsi_iram_ptr_event_gen_comp),
-	GEN_SRC_DST_ADDR_MAP(IPA_GSI_TOP_GSI_IRAM_PTR_TIMER_EXPIRED,
-			     gsi.debug.gsi_iram_ptrs,
-			     ipa_gsi_top_gsi_iram_ptr_timer_expired),
-	GEN_SRC_DST_ADDR_MAP(IPA_GSI_TOP_GSI_IRAM_PTR_WRITE_ENG_COMP,
-			     gsi.debug.gsi_iram_ptrs,
-			     ipa_gsi_top_gsi_iram_ptr_write_eng_comp),
-	GEN_SRC_DST_ADDR_MAP(IPA_GSI_TOP_GSI_IRAM_PTR_READ_ENG_COMP,
-			     gsi.debug.gsi_iram_ptrs,
-			     ipa_gsi_top_gsi_iram_ptr_read_eng_comp),
-	GEN_SRC_DST_ADDR_MAP(IPA_GSI_TOP_GSI_IRAM_PTR_UC_GP_INT,
-			     gsi.debug.gsi_iram_ptrs,
-			     ipa_gsi_top_gsi_iram_ptr_uc_gp_int),
-	GEN_SRC_DST_ADDR_MAP(IPA_GSI_TOP_GSI_IRAM_PTR_INT_MOD_STOPPED,
-			     gsi.debug.gsi_iram_ptrs,
-			     ipa_gsi_top_gsi_iram_ptr_int_mod_stopped),
-
-	/* GSI SHRAM pointers Registers */
-	GEN_SRC_DST_ADDR_MAP(IPA_GSI_TOP_GSI_SHRAM_PTR_CH_CNTXT_BASE_ADDR,
-			     gsi.debug.gsi_shram_ptrs,
-			     ipa_gsi_top_gsi_shram_ptr_ch_cntxt_base_addr),
-	GEN_SRC_DST_ADDR_MAP(IPA_GSI_TOP_GSI_SHRAM_PTR_EV_CNTXT_BASE_ADDR,
-			     gsi.debug.gsi_shram_ptrs,
-			     ipa_gsi_top_gsi_shram_ptr_ev_cntxt_base_addr),
-	GEN_SRC_DST_ADDR_MAP(IPA_GSI_TOP_GSI_SHRAM_PTR_RE_STORAGE_BASE_ADDR,
-			     gsi.debug.gsi_shram_ptrs,
-			     ipa_gsi_top_gsi_shram_ptr_re_storage_base_addr),
-	GEN_SRC_DST_ADDR_MAP(IPA_GSI_TOP_GSI_SHRAM_PTR_RE_ESC_BUF_BASE_ADDR,
-			     gsi.debug.gsi_shram_ptrs,
-			     ipa_gsi_top_gsi_shram_ptr_re_esc_buf_base_addr),
-	GEN_SRC_DST_ADDR_MAP(IPA_GSI_TOP_GSI_SHRAM_PTR_EE_SCRACH_BASE_ADDR,
-			     gsi.debug.gsi_shram_ptrs,
-			     ipa_gsi_top_gsi_shram_ptr_ee_scrach_base_addr),
-	GEN_SRC_DST_ADDR_MAP(IPA_GSI_TOP_GSI_SHRAM_PTR_FUNC_STACK_BASE_ADDR,
-			     gsi.debug.gsi_shram_ptrs,
-			     ipa_gsi_top_gsi_shram_ptr_func_stack_base_addr),
-
-	/*
-	 * NOTE: That GEN_SRC_DST_ADDR_MAP() not used below.  This is
-	 *       because the following registers are not scaler, rather
-	 *       they are register arrays...
-	 */
-
-	/* GSI General EE Registers */
-	IPA_REG_SAVE_CFG_ENTRY_GSI_GENERAL_EE(GSI_MANAGER_EE_QOS_n,
-					      gsi_manager_ee_qos_n),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_GENERAL_EE(EE_n_GSI_STATUS,
-					      ee_n_gsi_status),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_GENERAL_EE(EE_n_CNTXT_TYPE_IRQ,
-					      ee_n_cntxt_type_irq),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_GENERAL_EE(EE_n_CNTXT_TYPE_IRQ_MSK,
-					      ee_n_cntxt_type_irq_msk),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_GENERAL_EE(EE_n_CNTXT_SRC_GSI_CH_IRQ,
-					      ee_n_cntxt_src_gsi_ch_irq),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_GENERAL_EE(EE_n_CNTXT_SRC_EV_CH_IRQ,
-					      ee_n_cntxt_src_ev_ch_irq),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_GENERAL_EE(EE_n_CNTXT_SRC_GSI_CH_IRQ_MSK,
-					      ee_n_cntxt_src_gsi_ch_irq_msk),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_GENERAL_EE(EE_n_CNTXT_SRC_EV_CH_IRQ_MSK,
-					      ee_n_cntxt_src_ev_ch_irq_msk),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_GENERAL_EE(EE_n_CNTXT_SRC_IEOB_IRQ,
-					      ee_n_cntxt_src_ieob_irq),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_GENERAL_EE(EE_n_CNTXT_SRC_IEOB_IRQ_MSK,
-					      ee_n_cntxt_src_ieob_irq_msk),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_GENERAL_EE(EE_n_CNTXT_GSI_IRQ_STTS,
-					      ee_n_cntxt_gsi_irq_stts),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_GENERAL_EE(EE_n_CNTXT_GLOB_IRQ_STTS,
-					      ee_n_cntxt_glob_irq_stts),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_GENERAL_EE(EE_n_ERROR_LOG,
-					      ee_n_error_log),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_GENERAL_EE(EE_n_CNTXT_SCRATCH_0,
-					      ee_n_cntxt_scratch_0),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_GENERAL_EE(EE_n_CNTXT_SCRATCH_1,
-					      ee_n_cntxt_scratch_1),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_GENERAL_EE(EE_n_CNTXT_INTSET,
-					      ee_n_cntxt_intset),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_GENERAL_EE(EE_n_CNTXT_MSI_BASE_LSB,
-					      ee_n_cntxt_msi_base_lsb),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_GENERAL_EE(EE_n_CNTXT_MSI_BASE_MSB,
-					      ee_n_cntxt_msi_base_msb),
-
-	/* GSI Channel Context Registers */
-	IPA_REG_SAVE_CFG_ENTRY_GSI_CH_CNTXT(EE_n_GSI_CH_k_CNTXT_0,
-					    ee_n_gsi_ch_k_cntxt_0),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_CH_CNTXT(EE_n_GSI_CH_k_CNTXT_1,
-					    ee_n_gsi_ch_k_cntxt_1),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_CH_CNTXT(EE_n_GSI_CH_k_CNTXT_2,
-					    ee_n_gsi_ch_k_cntxt_2),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_CH_CNTXT(EE_n_GSI_CH_k_CNTXT_3,
-					    ee_n_gsi_ch_k_cntxt_3),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_CH_CNTXT(EE_n_GSI_CH_k_CNTXT_4,
-					    ee_n_gsi_ch_k_cntxt_4),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_CH_CNTXT(EE_n_GSI_CH_k_CNTXT_5,
-					    ee_n_gsi_ch_k_cntxt_5),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_CH_CNTXT(EE_n_GSI_CH_k_CNTXT_6,
-					    ee_n_gsi_ch_k_cntxt_6),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_CH_CNTXT(EE_n_GSI_CH_k_CNTXT_7,
-					    ee_n_gsi_ch_k_cntxt_7),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_CH_CNTXT(EE_n_GSI_CH_k_RE_FETCH_READ_PTR,
-					    ee_n_gsi_ch_k_re_fetch_read_ptr),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_CH_CNTXT(EE_n_GSI_CH_k_RE_FETCH_WRITE_PTR,
-					    ee_n_gsi_ch_k_re_fetch_write_ptr),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_CH_CNTXT(EE_n_GSI_CH_k_QOS,
-					    ee_n_gsi_ch_k_qos),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_CH_CNTXT(EE_n_GSI_CH_k_SCRATCH_0,
-					    ee_n_gsi_ch_k_scratch_0),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_CH_CNTXT(EE_n_GSI_CH_k_SCRATCH_1,
-					    ee_n_gsi_ch_k_scratch_1),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_CH_CNTXT(EE_n_GSI_CH_k_SCRATCH_2,
-					    ee_n_gsi_ch_k_scratch_2),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_CH_CNTXT(EE_n_GSI_CH_k_SCRATCH_3,
-					    ee_n_gsi_ch_k_scratch_3),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_CH_CNTXT(GSI_MAP_EE_n_CH_k_VP_TABLE,
-					    gsi_map_ee_n_ch_k_vp_table),
-
-	/* GSI Channel Event Context Registers */
-	IPA_REG_SAVE_CFG_ENTRY_GSI_EVT_CNTXT(EE_n_EV_CH_k_CNTXT_0,
-					     ee_n_ev_ch_k_cntxt_0),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_EVT_CNTXT(EE_n_EV_CH_k_CNTXT_1,
-					     ee_n_ev_ch_k_cntxt_1),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_EVT_CNTXT(EE_n_EV_CH_k_CNTXT_2,
-					     ee_n_ev_ch_k_cntxt_2),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_EVT_CNTXT(EE_n_EV_CH_k_CNTXT_3,
-					     ee_n_ev_ch_k_cntxt_3),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_EVT_CNTXT(EE_n_EV_CH_k_CNTXT_4,
-					     ee_n_ev_ch_k_cntxt_4),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_EVT_CNTXT(EE_n_EV_CH_k_CNTXT_5,
-					     ee_n_ev_ch_k_cntxt_5),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_EVT_CNTXT(EE_n_EV_CH_k_CNTXT_6,
-					     ee_n_ev_ch_k_cntxt_6),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_EVT_CNTXT(EE_n_EV_CH_k_CNTXT_7,
-					     ee_n_ev_ch_k_cntxt_7),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_EVT_CNTXT(EE_n_EV_CH_k_CNTXT_8,
-					     ee_n_ev_ch_k_cntxt_8),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_EVT_CNTXT(EE_n_EV_CH_k_CNTXT_9,
-					     ee_n_ev_ch_k_cntxt_9),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_EVT_CNTXT(EE_n_EV_CH_k_CNTXT_10,
-					     ee_n_ev_ch_k_cntxt_10),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_EVT_CNTXT(EE_n_EV_CH_k_CNTXT_11,
-					     ee_n_ev_ch_k_cntxt_11),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_EVT_CNTXT(EE_n_EV_CH_k_CNTXT_12,
-					     ee_n_ev_ch_k_cntxt_12),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_EVT_CNTXT(EE_n_EV_CH_k_CNTXT_13,
-					     ee_n_ev_ch_k_cntxt_13),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_EVT_CNTXT(EE_n_EV_CH_k_SCRATCH_0,
-					     ee_n_ev_ch_k_scratch_0),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_EVT_CNTXT(EE_n_EV_CH_k_SCRATCH_1,
-					     ee_n_ev_ch_k_scratch_1),
-	IPA_REG_SAVE_CFG_ENTRY_GSI_EVT_CNTXT(GSI_DEBUG_EE_n_EV_k_VP_TABLE,
-					     gsi_debug_ee_n_ev_k_vp_table),
-
-#if defined(CONFIG_IPA3_REGDUMP_NUM_EXTRA_ENDP_REGS) && \
-	CONFIG_IPA3_REGDUMP_NUM_EXTRA_ENDP_REGS > 0
-	/* Endp Registers for remaining pipes */
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP_EXTRA(IPA_ENDP_INIT_CTRL_n,
-					       ipa_endp_init_ctrl_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP_EXTRA(IPA_ENDP_INIT_CTRL_SCND_n,
-					       ipa_endp_init_ctrl_scnd_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP_EXTRA(IPA_ENDP_INIT_CFG_n,
-					       ipa_endp_init_cfg_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP_EXTRA(IPA_ENDP_INIT_NAT_n,
-					       ipa_endp_init_nat_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP_EXTRA(IPA_ENDP_INIT_HDR_n,
-					       ipa_endp_init_hdr_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP_EXTRA(IPA_ENDP_INIT_HDR_EXT_n,
-					       ipa_endp_init_hdr_ext_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP_EXTRA
-		(IPA_ENDP_INIT_HDR_METADATA_MASK_n,
-		ipa_endp_init_hdr_metadata_mask_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP_EXTRA(IPA_ENDP_INIT_HDR_METADATA_n,
-					       ipa_endp_init_hdr_metadata_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP_EXTRA(IPA_ENDP_INIT_MODE_n,
-					       ipa_endp_init_mode_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP_EXTRA(IPA_ENDP_INIT_AGGR_n,
-					       ipa_endp_init_aggr_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP_EXTRA(IPA_ENDP_INIT_HOL_BLOCK_EN_n,
-					       ipa_endp_init_hol_block_en_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP_EXTRA(IPA_ENDP_INIT_HOL_BLOCK_TIMER_n,
-					       ipa_endp_init_hol_block_timer_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP_EXTRA(IPA_ENDP_INIT_DEAGGR_n,
-					       ipa_endp_init_deaggr_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP_EXTRA(IPA_ENDP_STATUS_n,
-					       ipa_endp_status_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP_EXTRA(IPA_ENDP_INIT_RSRC_GRP_n,
-					       ipa_endp_init_rsrc_grp_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP_EXTRA(IPA_ENDP_INIT_SEQ_n,
-					       ipa_endp_init_seq_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP_EXTRA(IPA_ENDP_GSI_CFG_TLV_n,
-					       ipa_endp_gsi_cfg_tlv_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP_EXTRA(IPA_ENDP_GSI_CFG_AOS_n,
-					       ipa_endp_gsi_cfg_aos_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP_EXTRA(IPA_ENDP_GSI_CFG1_n,
-					       ipa_endp_gsi_cfg1_n),
-	IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP_EXTRA
-		(IPA_ENDP_FILTER_ROUTER_HSH_CFG_n,
-		 ipa_endp_filter_router_hsh_cfg_n),
-#endif
-};
-
-/* IPA uC PER registers save Cfg array */
-static struct map_src_dst_addr_s ipa_uc_regs_to_save_array[] = {
-	/* HWP registers */
-	GEN_SRC_DST_ADDR_MAP(IPA_UC_QMB_SYS_ADDR,
-			     ipa.hwp,
-			     ipa_uc_qmb_sys_addr),
-	GEN_SRC_DST_ADDR_MAP(IPA_UC_QMB_LOCAL_ADDR,
-			     ipa.hwp,
-			     ipa_uc_qmb_local_addr),
-	GEN_SRC_DST_ADDR_MAP(IPA_UC_QMB_LENGTH,
-			     ipa.hwp,
-			     ipa_uc_qmb_length),
-	GEN_SRC_DST_ADDR_MAP(IPA_UC_QMB_TRIGGER,
-			     ipa.hwp,
-			     ipa_uc_qmb_trigger),
-	GEN_SRC_DST_ADDR_MAP(IPA_UC_QMB_PENDING_TID,
-			     ipa.hwp,
-			     ipa_uc_qmb_pending_tid),
-	GEN_SRC_DST_ADDR_MAP(IPA_UC_QMB_COMPLETED_RD_FIFO_PEEK,
-			     ipa.hwp,
-			     ipa_uc_qmb_completed_rd_fifo_peek),
-	GEN_SRC_DST_ADDR_MAP(IPA_UC_QMB_COMPLETED_WR_FIFO_PEEK,
-			     ipa.hwp,
-			     ipa_uc_qmb_completed_wr_fifo_peek),
-	GEN_SRC_DST_ADDR_MAP(IPA_UC_QMB_MISC,
-			     ipa.hwp,
-			     ipa_uc_qmb_misc),
-	GEN_SRC_DST_ADDR_MAP(IPA_UC_QMB_STATUS,
-			     ipa.hwp,
-			     ipa_uc_qmb_status),
-	GEN_SRC_DST_ADDR_MAP(IPA_UC_QMB_BUS_ATTRIB,
-			     ipa.hwp,
-			     ipa_uc_qmb_bus_attrib),
-};
-
-static void ipa_hal_save_regs_save_ipa_testbus(void);
-static void ipa_reg_save_gsi_fifo_status(void);
-static void ipa_reg_save_rsrc_cnts(void);
-static void ipa_hal_save_regs_ipa_cmdq(void);
-static void ipa_hal_save_regs_rsrc_db(void);
-static void ipa_reg_save_anomaly_check(void);
-
-static struct reg_access_funcs_s *get_access_funcs(u32 addr)
-{
-	u32 i, asub = ipa3_ctx->sd_state;
-
-	for (i = 0; i < ARRAY_SIZE(mem_access_map); i++) {
-		if (addr >= mem_access_map[i].addr_range_begin &&
-			addr <  mem_access_map[i].addr_range_end) {
-			return mem_access_map[i].access[asub];
-		}
-	}
-
-	IPAERR("Unknown register offset(0x%08X). Using dflt access methods\n",
-		   addr);
-
-	return &io_matrix[AA_COMBO];
-}
-
-static u32 in_dword(
-	u32 addr)
-{
-	struct reg_access_funcs_s *io = get_access_funcs(addr);
-
-	return io->read(ipa3_ctx->reg_collection_base + addr);
-}
-
-static u32 in_dword_masked(
-	u32 addr,
-	u32 mask)
-{
-	struct reg_access_funcs_s *io = get_access_funcs(addr);
-	u32 val;
-
-	val = io->read(ipa3_ctx->reg_collection_base + addr);
-
-	if (io->read == act_read)
-		return val & mask;
-
-	return val;
-}
-
-static void out_dword(
-	u32 addr,
-	u32 val)
-{
-	struct reg_access_funcs_s *io = get_access_funcs(addr);
-
-	io->write(ipa3_ctx->reg_collection_base + addr, val);
-}
-
-/*
- * FUNCTION:  ipa_save_gsi_ver
- *
- * Saves the gsi version
- *
- * @return
- * None
- */
-void ipa_save_gsi_ver(void)
-{
-	if (!ipa3_ctx->do_register_collection_on_crash)
-		return;
-
-	ipa_reg_save.gsi.fw_ver =
-		IPA_READ_1xVECTOR_REG(IPA_GSI_TOP_GSI_INST_RAM_n, 0);
-}
-
-/*
- * FUNCTION:  ipa_save_registers
- *
- * Saves all the IPA register values which are configured
- *
- * @return
- * None
- */
-void ipa_save_registers(void)
-{
-	u32 i = 0;
-	/* Fetch the number of registers configured to be saved */
-	u32 num_regs = ARRAY_SIZE(ipa_regs_to_save_array);
-	u32 num_uc_per_regs = ARRAY_SIZE(ipa_uc_regs_to_save_array);
-	union ipa_hwio_def_ipa_rsrc_mngr_db_cfg_u for_cfg;
-	union ipa_hwio_def_ipa_rsrc_mngr_db_rsrc_read_u for_read;
-
-	if (!ipa3_ctx->do_register_collection_on_crash)
-		return;
-
-	IPAERR("Commencing\n");
-
-	/*
-	 * Remove the GSI FIFO and the endp registers for extra pipes for
-	 * now.  These would be saved later
-	 */
-	num_regs -= (CONFIG_IPA3_REGDUMP_NUM_EXTRA_ENDP_REGS *
-		     IPA_REG_SAVE_NUM_EXTRA_ENDP_REGS);
-
-	memset(&for_cfg, 0, sizeof(for_cfg));
-	memset(&for_read, 0, sizeof(for_read));
-
-	/* Now save all the configured registers */
-	for (i = 0; i < num_regs; i++) {
-		/* Copy reg value to our data struct */
-		*(ipa_regs_to_save_array[i].dst_addr) =
-			in_dword(ipa_regs_to_save_array[i].src_addr);
-	}
-
-	/*
-	 * Set the active flag for all active pipe indexed registers.
-	 */
-	for (i = 0; i < IPA_HW_PIPE_ID_MAX; i++)
-		ipa_reg_save.ipa.pipes[i].active = true;
-
-	/* Now save the per endp registers for the remaining pipes */
-	for (i = 0; i < (CONFIG_IPA3_REGDUMP_NUM_EXTRA_ENDP_REGS *
-			 IPA_REG_SAVE_NUM_EXTRA_ENDP_REGS); i++) {
-		/* Copy reg value to our data struct */
-		*(ipa_regs_to_save_array[num_regs + i].dst_addr) =
-			in_dword(ipa_regs_to_save_array[num_regs + i].src_addr);
-	}
-
-	IPA_HW_REG_SAVE_CFG_ENTRY_PIPE_ENDP_EXTRA_ACTIVE();
-
-	num_regs += (CONFIG_IPA3_REGDUMP_NUM_EXTRA_ENDP_REGS *
-		     IPA_REG_SAVE_NUM_EXTRA_ENDP_REGS);
-
-	/* Saving GSI FIFO Status registers */
-	ipa_reg_save_gsi_fifo_status();
-
-	/*
-	 * On targets that support SSR, we generally want to disable
-	 * the following reg save functionality as it may cause stalls
-	 * in IPA after the SSR.
-	 *
-	 * To override this, set do_non_tn_collection_on_crash to
-	 * true, via dtsi, and the collection will be done.
-	 */
-	if (ipa3_ctx->do_non_tn_collection_on_crash) {
-		/* Save all the uC PER configured registers */
-		for (i = 0; i < num_uc_per_regs; i++) {
-			/* Copy reg value to our data struct */
-			*(ipa_uc_regs_to_save_array[i].dst_addr) =
-			    in_dword(ipa_uc_regs_to_save_array[i].src_addr);
-		}
-
-		/* Saving CMD Queue registers */
-		ipa_hal_save_regs_ipa_cmdq();
-
-		/* Collecting resource DB information */
-		ipa_hal_save_regs_rsrc_db();
-
-		/* Save IPA testbus */
-		if (ipa3_ctx->do_testbus_collection_on_crash)
-			ipa_hal_save_regs_save_ipa_testbus();
-	}
-
-	/* GSI test bus */
-	for (i = 0;
-	     i < ARRAY_SIZE(ipa_reg_save_gsi_ch_test_bus_selector_array);
-	     i++) {
-		ipa_reg_save.gsi.debug.gsi_test_bus.test_bus_selector[i] =
-			ipa_reg_save_gsi_ch_test_bus_selector_array[i];
-
-		/* Write test bus selector */
-		IPA_WRITE_SCALER_REG(
-			GSI_TEST_BUS_SEL,
-			ipa_reg_save_gsi_ch_test_bus_selector_array[i]);
-
-		ipa_reg_save.gsi.debug.gsi_test_bus.test_bus_reg[
-		    i].gsi_testbus_reg =
-		    (u32) IPA_READ_SCALER_REG(GSI_TEST_BUS_REG);
-	}
-
-	ipa_reg_save_rsrc_cnts();
-
-	for (i = 0; i < HWIO_GSI_DEBUG_SW_RF_n_READ_MAXn + 1; i++)
-		ipa_reg_save.gsi.debug.gsi_mcs_regs.mcs_reg[i].rf_reg =
-			IPA_READ_1xVECTOR_REG(GSI_DEBUG_SW_RF_n_READ, i);
-
-	for (i = 0; i < HWIO_GSI_DEBUG_COUNTERn_MAXn + 1; i++)
-		ipa_reg_save.gsi.debug.gsi_cnt_regs.cnt[i].counter_value =
-			(u16)IPA_READ_1xVECTOR_REG(GSI_DEBUG_COUNTERn, i);
-
-	for (i = 0; i < IPA_HW_REG_SAVE_GSI_NUM_CH_CNTXT_A7; i++) {
-		u32 phys_ch_idx = ipa_reg_save.gsi.ch_cntxt.a7[
-			i].gsi_map_ee_n_ch_k_vp_table.phy_ch;
-		u32 n = phys_ch_idx * IPA_REG_SAVE_BYTES_PER_CHNL_SHRAM;
-
-		if (!ipa_reg_save.gsi.ch_cntxt.a7[
-				i].gsi_map_ee_n_ch_k_vp_table.valid)
-			continue;
-
-		ipa_reg_save.gsi.ch_cntxt.a7[
-			i].mcs_channel_scratch.scratch4.shram =
-			IPA_READ_1xVECTOR_REG(
-				GSI_SHRAM_n,
-				n + IPA_GSI_OFFSET_WORDS_SCRATCH4);
-
-		ipa_reg_save.gsi.ch_cntxt.a7[
-			i].mcs_channel_scratch.scratch5.shram =
-			IPA_READ_1xVECTOR_REG(
-				GSI_SHRAM_n,
-				n + IPA_GSI_OFFSET_WORDS_SCRATCH5);
-	}
-
-	for (i = 0; i < IPA_HW_REG_SAVE_GSI_NUM_CH_CNTXT_UC; i++) {
-		u32 phys_ch_idx = ipa_reg_save.gsi.ch_cntxt.uc[
-			i].gsi_map_ee_n_ch_k_vp_table.phy_ch;
-		u32 n = phys_ch_idx * IPA_REG_SAVE_BYTES_PER_CHNL_SHRAM;
-
-		if (!ipa_reg_save.gsi.ch_cntxt.uc[
-				i].gsi_map_ee_n_ch_k_vp_table.valid)
-			continue;
-
-		ipa_reg_save.gsi.ch_cntxt.uc[
-			i].mcs_channel_scratch.scratch4.shram =
-			IPA_READ_1xVECTOR_REG(
-				GSI_SHRAM_n,
-				n + IPA_GSI_OFFSET_WORDS_SCRATCH4);
-
-		ipa_reg_save.gsi.ch_cntxt.uc[
-			i].mcs_channel_scratch.scratch5.shram =
-			IPA_READ_1xVECTOR_REG(
-				GSI_SHRAM_n,
-				n + IPA_GSI_OFFSET_WORDS_SCRATCH5);
-	}
-
-	/*
-	 * On targets that support SSR, we generally want to disable
-	 * the following reg save functionality as it may cause stalls
-	 * in IPA after the SSR.
-	 *
-	 * To override this, set do_non_tn_collection_on_crash to
-	 * true, via dtsi, and the collection will be done.
-	 */
-	if (ipa3_ctx->do_non_tn_collection_on_crash) {
-		u32 ofst = GEN_2xVECTOR_REG_OFST(IPA_CTX_ID_m_CTX_NUM_n, 0, 0);
-		struct reg_access_funcs_s *io = get_access_funcs(ofst);
-		/*
-		 * If the memory is accessible, copy pkt context directly from
-		 * IPA_CTX_ID register space
-		 */
-		if (io->read == act_read) {
-			memcpy((void *)ipa_reg_save.pkt_ctntx,
-				   (const void *)
-				   (ipa3_ctx->reg_collection_base + ofst),
-				   sizeof(ipa_reg_save.pkt_ctntx));
-
-			for_cfg.value =
-				IPA_READ_SCALER_REG(IPA_RSRC_MNGR_DB_CFG);
-
-			for_cfg.def.rsrc_type_sel = 0;
-
-			IPA_MASKED_WRITE_SCALER_REG(
-				IPA_RSRC_MNGR_DB_CFG,
-				for_cfg.value);
-
-			for (i = 0; i < IPA_HW_PKT_CTNTX_MAX; i++) {
-				for_cfg.def.rsrc_id_sel = i;
-
-				IPA_MASKED_WRITE_SCALER_REG(
-					IPA_RSRC_MNGR_DB_CFG,
-					for_cfg.value);
-
-				for_read.value =
-					IPA_READ_SCALER_REG(
-						IPA_RSRC_MNGR_DB_RSRC_READ);
-
-				if (for_read.def.rsrc_occupied) {
-					ipa_reg_save.pkt_ctntx_active[i] = true;
-					ipa_reg_save.pkt_cntxt_state[i] =
-						(enum ipa_hw_pkt_cntxt_state_e)
-						ipa_reg_save.pkt_ctntx[i].state;
-				}
-			}
-		} else {
-			IPAERR("IPA_CTX_ID is not currently accessible\n");
-		}
-	}
-
-	if (ipa3_ctx->do_ram_collection_on_crash) {
-		for (i = 0; i < IPA_IU_SIZE / sizeof(u32); i++) {
-			ipa_reg_save.ipa.ipa_iu_ptr[i] =
-				in_dword(IPA_IU_ADDR + (i * sizeof(u32)));
-		}
-		for (i = 0; i < IPA_SRAM_SIZE / sizeof(u32); i++) {
-			ipa_reg_save.ipa.ipa_sram_ptr[i] =
-				in_dword(IPA_SRAM_ADDR + (i * sizeof(u32)));
-		}
-		for (i = 0; i < IPA_MBOX_SIZE / sizeof(u32); i++) {
-			ipa_reg_save.ipa.ipa_mbox_ptr[i] =
-				in_dword(IPA_MBOX_ADDR + (i * sizeof(u32)));
-		}
-		for (i = 0; i < IPA_HRAM_SIZE / sizeof(u32); i++) {
-			ipa_reg_save.ipa.ipa_hram_ptr[i] =
-				in_dword(IPA_HRAM_ADDR + (i * sizeof(u32)));
-		}
-		for (i = 0; i < IPA_SEQ_SIZE / sizeof(u32); i++) {
-			ipa_reg_save.ipa.ipa_seq_ptr[i] =
-				in_dword(IPA_SEQ_ADDR + (i * sizeof(u32)));
-		}
-		for (i = 0; i < IPA_GSI_SIZE / sizeof(u32); i++) {
-			ipa_reg_save.ipa.ipa_gsi_ptr[i] =
-				in_dword(IPA_GSI_ADDR + (i * sizeof(u32)));
-		}
-		IPALOG_VnP_ADDRS(ipa_reg_save.ipa.ipa_iu_ptr);
-		IPALOG_VnP_ADDRS(ipa_reg_save.ipa.ipa_sram_ptr);
-		IPALOG_VnP_ADDRS(ipa_reg_save.ipa.ipa_mbox_ptr);
-		IPALOG_VnP_ADDRS(ipa_reg_save.ipa.ipa_hram_ptr);
-		IPALOG_VnP_ADDRS(ipa_reg_save.ipa.ipa_seq_ptr);
-		IPALOG_VnP_ADDRS(ipa_reg_save.ipa.ipa_gsi_ptr);
-	}
-
-	ipa_reg_save_anomaly_check();
-
-	IPAERR("Completed\n");
-}
-
-/*
- * FUNCTION:  ipa_reg_save_gsi_fifo_status
- *
- * This function saves the GSI FIFO Status registers for all endpoints
- *
- * @param
- *
- * @return
- */
-static void ipa_reg_save_gsi_fifo_status(void)
-{
-	union ipa_hwio_def_ipa_gsi_fifo_status_ctrl_u gsi_fifo_status_ctrl;
-	u8 i;
-
-	memset(&gsi_fifo_status_ctrl, 0, sizeof(gsi_fifo_status_ctrl));
-
-	for (i = 0; i < IPA_HW_PIPE_ID_MAX; i++) {
-		gsi_fifo_status_ctrl.def.ipa_gsi_fifo_status_en = 1;
-		gsi_fifo_status_ctrl.def.ipa_gsi_fifo_status_port_sel = i;
-
-		IPA_MASKED_WRITE_SCALER_REG(IPA_GSI_FIFO_STATUS_CTRL,
-				     gsi_fifo_status_ctrl.value);
-
-		ipa_reg_save.gsi_fifo_status[i].gsi_fifo_status_ctrl.value =
-			IPA_READ_SCALER_REG(IPA_GSI_FIFO_STATUS_CTRL);
-		ipa_reg_save.gsi_fifo_status[i].gsi_tlv_fifo_status.value =
-			IPA_READ_SCALER_REG(IPA_GSI_TLV_FIFO_STATUS);
-		ipa_reg_save.gsi_fifo_status[i].gsi_aos_fifo_status.value =
-			IPA_READ_SCALER_REG(IPA_GSI_AOS_FIFO_STATUS);
-	}
-}
-
-/*
- * FUNCTION:  ipa_reg_save_rsrc_cnts
- *
- * This function saves the resource counts for all PCIE and DDR
- * resource groups.
- *
- * @param
- * @return
- */
-static void ipa_reg_save_rsrc_cnts(void)
-{
-	union ipa_hwio_def_ipa_src_rsrc_grp_0123_rsrc_type_cnt_n_u
-		src_0123_rsrc_cnt;
-	union ipa_hwio_def_ipa_dst_rsrc_grp_0123_rsrc_type_cnt_n_u
-		dst_0123_rsrc_cnt;
-
-	ipa_reg_save.rsrc_cnts.pcie.resource_group = IPA_HW_PCIE_SRC_RSRP_GRP;
-	ipa_reg_save.rsrc_cnts.ddr.resource_group = IPA_HW_DDR_SRC_RSRP_GRP;
-
-	src_0123_rsrc_cnt.value =
-		IPA_READ_1xVECTOR_REG(IPA_SRC_RSRC_GRP_0123_RSRC_TYPE_CNT_n, 0);
-
-	ipa_reg_save.rsrc_cnts.pcie.src.pkt_cntxt =
-		src_0123_rsrc_cnt.def.src_rsrc_grp_0_cnt;
-	ipa_reg_save.rsrc_cnts.ddr.src.pkt_cntxt =
-		src_0123_rsrc_cnt.def.src_rsrc_grp_1_cnt;
-
-	src_0123_rsrc_cnt.value =
-		IPA_READ_1xVECTOR_REG(IPA_SRC_RSRC_GRP_0123_RSRC_TYPE_CNT_n, 1);
-
-	ipa_reg_save.rsrc_cnts.pcie.src.descriptor_list =
-		src_0123_rsrc_cnt.def.src_rsrc_grp_0_cnt;
-	ipa_reg_save.rsrc_cnts.ddr.src.descriptor_list =
-		src_0123_rsrc_cnt.def.src_rsrc_grp_1_cnt;
-
-	src_0123_rsrc_cnt.value =
-		IPA_READ_1xVECTOR_REG(IPA_SRC_RSRC_GRP_0123_RSRC_TYPE_CNT_n, 2);
-
-	ipa_reg_save.rsrc_cnts.pcie.src.data_descriptor_buffer =
-		src_0123_rsrc_cnt.def.src_rsrc_grp_0_cnt;
-	ipa_reg_save.rsrc_cnts.ddr.src.data_descriptor_buffer =
-		src_0123_rsrc_cnt.def.src_rsrc_grp_1_cnt;
-
-	src_0123_rsrc_cnt.value =
-		IPA_READ_1xVECTOR_REG(IPA_SRC_RSRC_GRP_0123_RSRC_TYPE_CNT_n, 3);
-
-	ipa_reg_save.rsrc_cnts.pcie.src.hps_dmars =
-		src_0123_rsrc_cnt.def.src_rsrc_grp_0_cnt;
-	ipa_reg_save.rsrc_cnts.ddr.src.hps_dmars =
-		src_0123_rsrc_cnt.def.src_rsrc_grp_1_cnt;
-
-	src_0123_rsrc_cnt.value =
-		IPA_READ_1xVECTOR_REG(IPA_SRC_RSRC_GRP_0123_RSRC_TYPE_CNT_n, 4);
-
-	ipa_reg_save.rsrc_cnts.pcie.src.reserved_acks =
-		src_0123_rsrc_cnt.def.src_rsrc_grp_0_cnt;
-	ipa_reg_save.rsrc_cnts.ddr.src.reserved_acks =
-		src_0123_rsrc_cnt.def.src_rsrc_grp_1_cnt;
-
-	dst_0123_rsrc_cnt.value =
-		IPA_READ_1xVECTOR_REG(IPA_DST_RSRC_GRP_0123_RSRC_TYPE_CNT_n, 0);
-
-	ipa_reg_save.rsrc_cnts.pcie.dst.reserved_sectors =
-		dst_0123_rsrc_cnt.def.dst_rsrc_grp_0_cnt;
-	ipa_reg_save.rsrc_cnts.ddr.dst.reserved_sectors =
-		dst_0123_rsrc_cnt.def.dst_rsrc_grp_1_cnt;
-
-	dst_0123_rsrc_cnt.value =
-		IPA_READ_1xVECTOR_REG(IPA_DST_RSRC_GRP_0123_RSRC_TYPE_CNT_n, 1);
-
-	ipa_reg_save.rsrc_cnts.pcie.dst.dps_dmars =
-		dst_0123_rsrc_cnt.def.dst_rsrc_grp_0_cnt;
-	ipa_reg_save.rsrc_cnts.ddr.dst.dps_dmars =
-		dst_0123_rsrc_cnt.def.dst_rsrc_grp_1_cnt;
-}
-
-/*
- * FUNCTION:  ipa_reg_save_rsrc_cnts_test_bus
- *
- * This function saves the resource counts for all PCIE and DDR
- * resource groups collected from test bus.
- *
- * @param
- *
- * @return
- */
-void ipa_reg_save_rsrc_cnts_test_bus(void)
-{
-	int32_t rsrc_type = 0;
-
-	ipa_reg_save.rsrc_cnts.pcie.resource_group = IPA_HW_PCIE_SRC_RSRP_GRP;
-	ipa_reg_save.rsrc_cnts.ddr.resource_group = IPA_HW_DDR_SRC_RSRP_GRP;
-
-	rsrc_type = 0;
-	ipa_reg_save.rsrc_cnts.pcie.src.pkt_cntxt =
-		IPA_DEBUG_TESTBUS_GET_RSRC_TYPE_CNT(rsrc_type,
-						    IPA_HW_PCIE_SRC_RSRP_GRP);
-
-	ipa_reg_save.rsrc_cnts.ddr.src.pkt_cntxt =
-		IPA_DEBUG_TESTBUS_GET_RSRC_TYPE_CNT(rsrc_type,
-						    IPA_HW_DDR_SRC_RSRP_GRP);
-
-	rsrc_type = 1;
-	ipa_reg_save.rsrc_cnts.pcie.src.descriptor_list =
-		IPA_DEBUG_TESTBUS_GET_RSRC_TYPE_CNT(rsrc_type,
-						    IPA_HW_PCIE_SRC_RSRP_GRP);
-
-	ipa_reg_save.rsrc_cnts.ddr.src.descriptor_list =
-		IPA_DEBUG_TESTBUS_GET_RSRC_TYPE_CNT(rsrc_type,
-						    IPA_HW_DDR_SRC_RSRP_GRP);
-
-	rsrc_type = 2;
-	ipa_reg_save.rsrc_cnts.pcie.src.data_descriptor_buffer =
-		IPA_DEBUG_TESTBUS_GET_RSRC_TYPE_CNT(rsrc_type,
-						    IPA_HW_PCIE_SRC_RSRP_GRP);
-
-	ipa_reg_save.rsrc_cnts.ddr.src.data_descriptor_buffer =
-		IPA_DEBUG_TESTBUS_GET_RSRC_TYPE_CNT(rsrc_type,
-						    IPA_HW_DDR_SRC_RSRP_GRP);
-
-	rsrc_type = 3;
-	ipa_reg_save.rsrc_cnts.pcie.src.hps_dmars =
-		IPA_DEBUG_TESTBUS_GET_RSRC_TYPE_CNT(rsrc_type,
-						    IPA_HW_PCIE_SRC_RSRP_GRP);
-
-	ipa_reg_save.rsrc_cnts.ddr.src.hps_dmars =
-		IPA_DEBUG_TESTBUS_GET_RSRC_TYPE_CNT(rsrc_type,
-						    IPA_HW_DDR_SRC_RSRP_GRP);
-
-	rsrc_type = 4;
-	ipa_reg_save.rsrc_cnts.pcie.src.reserved_acks =
-		IPA_DEBUG_TESTBUS_GET_RSRC_TYPE_CNT(rsrc_type,
-						    IPA_HW_PCIE_SRC_RSRP_GRP);
-
-	ipa_reg_save.rsrc_cnts.ddr.src.reserved_acks =
-		IPA_DEBUG_TESTBUS_GET_RSRC_TYPE_CNT(rsrc_type,
-						    IPA_HW_DDR_SRC_RSRP_GRP);
-
-	rsrc_type = 5;
-	ipa_reg_save.rsrc_cnts.pcie.dst.reserved_sectors =
-		IPA_DEBUG_TESTBUS_GET_RSRC_TYPE_CNT(rsrc_type,
-						    IPA_HW_PCIE_DEST_RSRP_GRP);
-
-	ipa_reg_save.rsrc_cnts.ddr.dst.reserved_sectors =
-		IPA_DEBUG_TESTBUS_GET_RSRC_TYPE_CNT(rsrc_type,
-						    IPA_HW_DDR_DEST_RSRP_GRP);
-
-	rsrc_type = 6;
-	ipa_reg_save.rsrc_cnts.pcie.dst.dps_dmars =
-		IPA_DEBUG_TESTBUS_GET_RSRC_TYPE_CNT(rsrc_type,
-						    IPA_HW_PCIE_DEST_RSRP_GRP);
-
-	ipa_reg_save.rsrc_cnts.ddr.dst.dps_dmars =
-		IPA_DEBUG_TESTBUS_GET_RSRC_TYPE_CNT(rsrc_type,
-						    IPA_HW_DDR_DEST_RSRP_GRP);
-}
-
-/*
- * FUNCTION:  ipa_hal_save_regs_ipa_cmdq
- *
- * This function saves the various IPA CMDQ registers
- *
- * @param
- *
- * @return
- */
-static void ipa_hal_save_regs_ipa_cmdq(void)
-{
-	int32_t i;
-	union ipa_hwio_def_ipa_rx_hps_cmdq_cmd_u rx_hps_cmdq_cmd = { { 0 } };
-	union ipa_hwio_def_ipa_hps_dps_cmdq_cmd_u hps_dps_cmdq_cmd = { { 0 } };
-	union ipa_hwio_def_ipa_dps_tx_cmdq_cmd_u dps_tx_cmdq_cmd = { { 0 } };
-	union ipa_hwio_def_ipa_ackmngr_cmdq_cmd_u ackmngr_cmdq_cmd = { { 0 } };
-	union ipa_hwio_def_ipa_prod_ackmngr_cmdq_cmd_u
-		prod_ackmngr_cmdq_cmd = { { 0 } };
-	union ipa_hwio_def_ipa_ntf_tx_cmdq_cmd_u ntf_tx_cmdq_cmd = { { 0 } };
-
-	/* Save RX_HPS CMDQ   */
-	for (i = 0; i < IPA_DEBUG_CMDQ_HPS_SELECT_NUM_GROUPS; i++) {
-		rx_hps_cmdq_cmd.def.rd_req = 0;
-		rx_hps_cmdq_cmd.def.cmd_client = i;
-		IPA_MASKED_WRITE_SCALER_REG(IPA_RX_HPS_CMDQ_CMD,
-				     rx_hps_cmdq_cmd.value);
-		ipa_reg_save.ipa.dbg.ipa_rx_hps_cmdq_count_arr[i].value =
-			IPA_READ_SCALER_REG(IPA_RX_HPS_CMDQ_COUNT);
-		ipa_reg_save.ipa.dbg.ipa_rx_hps_cmdq_status_arr[i].value =
-			IPA_READ_SCALER_REG(IPA_RX_HPS_CMDQ_STATUS);
-		rx_hps_cmdq_cmd.def.rd_req = 1;
-		rx_hps_cmdq_cmd.def.cmd_client = i;
-		IPA_MASKED_WRITE_SCALER_REG(IPA_RX_HPS_CMDQ_CMD,
-				     rx_hps_cmdq_cmd.value);
-		ipa_reg_save.ipa.dbg.ipa_rx_hps_cmdq_data_rd_0_arr[i].value =
-			IPA_READ_SCALER_REG(IPA_RX_HPS_CMDQ_DATA_RD_0);
-		ipa_reg_save.ipa.dbg.ipa_rx_hps_cmdq_data_rd_1_arr[i].value =
-			IPA_READ_SCALER_REG(IPA_RX_HPS_CMDQ_DATA_RD_1);
-		ipa_reg_save.ipa.dbg.ipa_rx_hps_cmdq_data_rd_2_arr[i].value =
-			IPA_READ_SCALER_REG(IPA_RX_HPS_CMDQ_DATA_RD_2);
-		ipa_reg_save.ipa.dbg.ipa_rx_hps_cmdq_data_rd_3_arr[i].value =
-			IPA_READ_SCALER_REG(IPA_RX_HPS_CMDQ_DATA_RD_3);
-	}
-
-	/* Save HPS_DPS CMDQ   */
-	for (i = 0; i < IPA_TESTBUS_SEL_EP_MAX + 1; i++) {
-		hps_dps_cmdq_cmd.def.rd_req = 0;
-		hps_dps_cmdq_cmd.def.cmd_client = i;
-		IPA_MASKED_WRITE_SCALER_REG(IPA_HPS_DPS_CMDQ_CMD,
-				     hps_dps_cmdq_cmd.value);
-		ipa_reg_save.ipa.dbg.ipa_hps_dps_cmdq_status_arr[i].value =
-			IPA_READ_SCALER_REG(IPA_HPS_DPS_CMDQ_STATUS);
-		ipa_reg_save.ipa.dbg.ipa_hps_dps_cmdq_count_arr[i].value =
-			IPA_READ_SCALER_REG(IPA_HPS_DPS_CMDQ_COUNT);
-
-		hps_dps_cmdq_cmd.def.rd_req = 1;
-		hps_dps_cmdq_cmd.def.cmd_client = i;
-		IPA_MASKED_WRITE_SCALER_REG(IPA_HPS_DPS_CMDQ_CMD,
-				     hps_dps_cmdq_cmd.value);
-		ipa_reg_save.ipa.dbg.ipa_hps_dps_cmdq_data_rd_0_arr[i].value =
-			IPA_READ_SCALER_REG(IPA_HPS_DPS_CMDQ_DATA_RD_0);
-	}
-
-	/* Save DPS_TX CMDQ   */
-	for (i = 0; i < IPA_DEBUG_CMDQ_DPS_SELECT_NUM_GROUPS; i++) {
-		dps_tx_cmdq_cmd.def.cmd_client = i;
-		dps_tx_cmdq_cmd.def.rd_req = 0;
-		IPA_MASKED_WRITE_SCALER_REG(IPA_DPS_TX_CMDQ_CMD,
-				     dps_tx_cmdq_cmd.value);
-		ipa_reg_save.ipa.dbg.ipa_dps_tx_cmdq_status_arr[i].value =
-			IPA_READ_SCALER_REG(IPA_DPS_TX_CMDQ_STATUS);
-		ipa_reg_save.ipa.dbg.ipa_dps_tx_cmdq_count_arr[i].value =
-			IPA_READ_SCALER_REG(IPA_DPS_TX_CMDQ_COUNT);
-
-		dps_tx_cmdq_cmd.def.cmd_client = i;
-		dps_tx_cmdq_cmd.def.rd_req = 1;
-		IPA_MASKED_WRITE_SCALER_REG(IPA_DPS_TX_CMDQ_CMD,
-				     dps_tx_cmdq_cmd.value);
-		ipa_reg_save.ipa.dbg.ipa_dps_tx_cmdq_data_rd_0_arr[i].value =
-			IPA_READ_SCALER_REG(IPA_DPS_TX_CMDQ_DATA_RD_0);
-	}
-
-	/* Save ACKMNGR CMDQ   */
-	for (i = 0; i < IPA_DEBUG_CMDQ_DPS_SELECT_NUM_GROUPS; i++) {
-		ackmngr_cmdq_cmd.def.rd_req = 0;
-		ackmngr_cmdq_cmd.def.cmd_client = i;
-		IPA_MASKED_WRITE_SCALER_REG(IPA_ACKMNGR_CMDQ_CMD,
-				     ackmngr_cmdq_cmd.value);
-		ipa_reg_save.ipa.dbg.ipa_ackmngr_cmdq_status_arr[i].value =
-			IPA_READ_SCALER_REG(IPA_ACKMNGR_CMDQ_STATUS);
-		ipa_reg_save.ipa.dbg.ipa_ackmngr_cmdq_count_arr[i].value =
-			IPA_READ_SCALER_REG(IPA_ACKMNGR_CMDQ_COUNT);
-
-		ackmngr_cmdq_cmd.def.rd_req = 1;
-		ackmngr_cmdq_cmd.def.cmd_client = i;
-		IPA_MASKED_WRITE_SCALER_REG(IPA_ACKMNGR_CMDQ_CMD,
-				     ackmngr_cmdq_cmd.value);
-		ipa_reg_save.ipa.dbg.ipa_ackmngr_cmdq_data_rd_arr[i].value =
-			IPA_READ_SCALER_REG(IPA_ACKMNGR_CMDQ_DATA_RD);
-	}
-
-	/* Save PROD ACKMNGR CMDQ   */
-	for (i = 0; i < IPA_TESTBUS_SEL_EP_MAX + 1; i++) {
-		prod_ackmngr_cmdq_cmd.def.rd_req = 0;
-		prod_ackmngr_cmdq_cmd.def.cmd_client = i;
-		IPA_MASKED_WRITE_SCALER_REG(IPA_PROD_ACKMNGR_CMDQ_CMD,
-				     prod_ackmngr_cmdq_cmd.value);
-		ipa_reg_save.ipa.dbg.ipa_prod_ackmngr_cmdq_status_arr[i].value
-			= IPA_READ_SCALER_REG(
-				IPA_PROD_ACKMNGR_CMDQ_STATUS);
-		ipa_reg_save.ipa.dbg.ipa_prod_ackmngr_cmdq_count_arr[i].value =
-			IPA_READ_SCALER_REG(IPA_PROD_ACKMNGR_CMDQ_COUNT);
-		prod_ackmngr_cmdq_cmd.def.rd_req = 1;
-		prod_ackmngr_cmdq_cmd.def.cmd_client = i;
-		IPA_MASKED_WRITE_SCALER_REG(IPA_PROD_ACKMNGR_CMDQ_CMD,
-				     prod_ackmngr_cmdq_cmd.value);
-		ipa_reg_save.ipa.dbg.ipa_prod_ackmngr_cmdq_data_rd_arr[
-			i].value =
-			IPA_READ_SCALER_REG(
-				IPA_PROD_ACKMNGR_CMDQ_DATA_RD);
-	}
-
-	/* Save NTF_TX CMDQ   */
-	for (i = 0; i < IPA_TESTBUS_SEL_EP_MAX + 1; i++) {
-		ntf_tx_cmdq_cmd.def.rd_req = 0;
-		ntf_tx_cmdq_cmd.def.cmd_client = i;
-		IPA_MASKED_WRITE_SCALER_REG(IPA_NTF_TX_CMDQ_CMD,
-				     ntf_tx_cmdq_cmd.value);
-		ipa_reg_save.ipa.dbg.ipa_ntf_tx_cmdq_status_arr[i].value =
-			IPA_READ_SCALER_REG(IPA_NTF_TX_CMDQ_STATUS);
-		ipa_reg_save.ipa.dbg.ipa_ntf_tx_cmdq_count_arr[i].value =
-			IPA_READ_SCALER_REG(IPA_NTF_TX_CMDQ_COUNT);
-		ntf_tx_cmdq_cmd.def.rd_req = 1;
-		ntf_tx_cmdq_cmd.def.cmd_client = i;
-		IPA_MASKED_WRITE_SCALER_REG(IPA_NTF_TX_CMDQ_CMD,
-				     ntf_tx_cmdq_cmd.value);
-		ipa_reg_save.ipa.dbg.ipa_ntf_tx_cmdq_data_rd_0_arr[i].value =
-			IPA_READ_SCALER_REG(IPA_NTF_TX_CMDQ_DATA_RD_0);
-	}
-}
-
-/*
- * FUNCTION:  ipa_hal_save_regs_save_ipa_testbus
- *
- * This function saves the IPA testbus
- *
- * @param
- *
- * @return
- */
-static void ipa_hal_save_regs_save_ipa_testbus(void)
-{
-	s32 sel_internal, sel_external, sel_ep;
-	union ipa_hwio_def_ipa_testbus_sel_u testbus_sel = { { 0 } };
-
-	if (ipa_reg_save.ipa.testbus == NULL) {
-		/*
-		 * Test-bus structure not allocated - exit test-bus collection
-		 */
-		IPADBG("ipa_reg_save.ipa.testbus was not allocated\n");
-		return;
-	}
-
-	/* Enable Test-bus */
-	testbus_sel.value = 0;
-	testbus_sel.def.testbus_en = true;
-
-	IPA_WRITE_SCALER_REG(IPA_TESTBUS_SEL, testbus_sel.value);
-
-	for (sel_external = 0;
-		 sel_external <= IPA_TESTBUS_SEL_EXTERNAL_MAX;
-		 sel_external++) {
-
-		for (sel_internal = 0;
-			 sel_internal <= IPA_TESTBUS_SEL_INTERNAL_MAX;
-			 sel_internal++) {
-
-			testbus_sel.value = 0;
-
-			testbus_sel.def.pipe_select = 0;
-			testbus_sel.def.external_block_select =
-				sel_external;
-			testbus_sel.def.internal_block_select =
-				sel_internal;
-
-			IPA_MASKED_WRITE_SCALER_REG(
-				IPA_TESTBUS_SEL,
-				testbus_sel.value);
-
-			ipa_reg_save.ipa.testbus->global.global[
-				sel_internal][sel_external].testbus_sel.value =
-				testbus_sel.value;
-
-			ipa_reg_save.ipa.testbus->global.global[
-				sel_internal][sel_external].testbus_data.value =
-				IPA_READ_SCALER_REG(IPA_DEBUG_DATA);
-		}
-	}
-
-	/* Collect per EP test bus */
-	for (sel_ep = 0;
-		 sel_ep <= IPA_TESTBUS_SEL_EP_MAX;
-		 sel_ep++) {
-
-		for (sel_external = 0;
-			 sel_external <=
-				 IPA_TESTBUS_SEL_EXTERNAL_MAX;
-			 sel_external++) {
-
-			for (sel_internal = 0;
-				 sel_internal <=
-					 IPA_TESTBUS_SEL_INTERNAL_PIPE_MAX;
-				 sel_internal++) {
-
-				testbus_sel.value = 0;
-
-				testbus_sel.def.pipe_select = sel_ep;
-				testbus_sel.def.external_block_select =
-					sel_external;
-				testbus_sel.def.internal_block_select =
-					sel_internal;
-
-				IPA_MASKED_WRITE_SCALER_REG(
-					IPA_TESTBUS_SEL,
-					testbus_sel.value);
-
-				ipa_reg_save.ipa.testbus->ep[sel_ep].entry_ep[
-					sel_internal][sel_external].
-					testbus_sel.value =
-					testbus_sel.value;
-
-				ipa_reg_save.ipa.testbus->ep[sel_ep].entry_ep[
-					sel_internal][sel_external].
-					testbus_data.value =
-					IPA_READ_SCALER_REG(
-						IPA_DEBUG_DATA);
-			}
-		}
-	}
-
-	/* Disable Test-bus */
-	testbus_sel.value = 0;
-
-	IPA_WRITE_SCALER_REG(
-		IPA_TESTBUS_SEL,
-		testbus_sel.value);
-}
-
-/*
- * FUNCTION:  ipa_reg_save_init
- *
- * This function initializes and memsets the register save struct.
- *
- * @param
- *
- * @return
- */
-int ipa_reg_save_init(u32 value)
-{
-	u32 i, num_regs = ARRAY_SIZE(ipa_regs_to_save_array);
-
-	if (!ipa3_ctx->do_register_collection_on_crash)
-		return 0;
-
-	memset(&ipa_reg_save, value, sizeof(ipa_reg_save));
-
-	ipa_reg_save.ipa.testbus = NULL;
-
-	if (ipa3_ctx->do_testbus_collection_on_crash) {
-		memset(ipa_testbus_mem, value, sizeof(ipa_testbus_mem));
-		ipa_reg_save.ipa.testbus =
-		    (struct ipa_reg_save_ipa_testbus_s *) ipa_testbus_mem;
-	}
-
-	/* setup access for register collection/dump on crash */
-	IPADBG("Mapping 0x%x bytes starting at 0x%x\n",
-	       ipa3_ctx->entire_ipa_block_size,
-	       ipa3_ctx->ipa_wrapper_base);
-
-	ipa3_ctx->reg_collection_base =
-		ioremap_nocache(ipa3_ctx->ipa_wrapper_base,
-			ipa3_ctx->entire_ipa_block_size);
-
-	if (!ipa3_ctx->reg_collection_base) {
-		IPAERR(":register collection ioremap err\n");
-		goto alloc_fail1;
-	}
-
-	num_regs -=
-		(CONFIG_IPA3_REGDUMP_NUM_EXTRA_ENDP_REGS *
-		 IPA_REG_SAVE_NUM_EXTRA_ENDP_REGS);
-
-	for (i = 0;
-		 i < (CONFIG_IPA3_REGDUMP_NUM_EXTRA_ENDP_REGS *
-			  IPA_REG_SAVE_NUM_EXTRA_ENDP_REGS);
-		 i++)
-		*(ipa_regs_to_save_array[num_regs + i].dst_addr) = 0x0;
-
-	ipa_reg_save.ipa.ipa_gsi_ptr  = NULL;
-	ipa_reg_save.ipa.ipa_seq_ptr  = NULL;
-	ipa_reg_save.ipa.ipa_hram_ptr = NULL;
-	ipa_reg_save.ipa.ipa_mbox_ptr = NULL;
-	ipa_reg_save.ipa.ipa_sram_ptr = NULL;
-	ipa_reg_save.ipa.ipa_iu_ptr   = NULL;
-
-	if (ipa3_ctx->do_ram_collection_on_crash) {
-		ipa_reg_save.ipa.ipa_iu_ptr =
-			alloc_and_init(IPA_IU_SIZE, value);
-		if (!ipa_reg_save.ipa.ipa_iu_ptr) {
-			IPAERR("ipa_iu_ptr memory alloc failed\n");
-			goto alloc_fail2;
-		}
-
-		ipa_reg_save.ipa.ipa_sram_ptr =
-			alloc_and_init(IPA_SRAM_SIZE, value);
-		if (!ipa_reg_save.ipa.ipa_sram_ptr) {
-			IPAERR("ipa_sram_ptr memory alloc failed\n");
-			goto alloc_fail2;
-		}
-
-		ipa_reg_save.ipa.ipa_mbox_ptr =
-			alloc_and_init(IPA_MBOX_SIZE, value);
-		if (!ipa_reg_save.ipa.ipa_mbox_ptr) {
-			IPAERR("ipa_mbox_ptr memory alloc failed\n");
-			goto alloc_fail2;
-		}
-
-		ipa_reg_save.ipa.ipa_hram_ptr =
-			alloc_and_init(IPA_HRAM_SIZE, value);
-		if (!ipa_reg_save.ipa.ipa_hram_ptr) {
-			IPAERR("ipa_hram_ptr memory alloc failed\n");
-			goto alloc_fail2;
-		}
-
-		ipa_reg_save.ipa.ipa_seq_ptr =
-			alloc_and_init(IPA_SEQ_SIZE, value);
-		if (!ipa_reg_save.ipa.ipa_seq_ptr) {
-			IPAERR("ipa_seq_ptr memory alloc failed\n");
-			goto alloc_fail2;
-		}
-
-		ipa_reg_save.ipa.ipa_gsi_ptr =
-			alloc_and_init(IPA_GSI_SIZE, value);
-		if (!ipa_reg_save.ipa.ipa_gsi_ptr) {
-			IPAERR("ipa_gsi_ptr memory alloc failed\n");
-			goto alloc_fail2;
-		}
-	}
-
-	return 0;
-
-alloc_fail2:
-	kfree(ipa_reg_save.ipa.ipa_seq_ptr);
-	kfree(ipa_reg_save.ipa.ipa_hram_ptr);
-	kfree(ipa_reg_save.ipa.ipa_mbox_ptr);
-	kfree(ipa_reg_save.ipa.ipa_sram_ptr);
-	kfree(ipa_reg_save.ipa.ipa_iu_ptr);
-	iounmap(ipa3_ctx->reg_collection_base);
-alloc_fail1:
-	return -ENOMEM;
-}
-
-/*
- * FUNCTION:  ipa_hal_save_regs_rsrc_db
- *
- * This function saves the various IPA RSRC_MNGR_DB registers
- *
- * @param
- *
- * @return
- */
-static void ipa_hal_save_regs_rsrc_db(void)
-{
-	u32 rsrc_type = 0;
-	u32 rsrc_id = 0;
-	u32 rsrc_group = 0;
-	union ipa_hwio_def_ipa_rsrc_mngr_db_cfg_u
-		ipa_rsrc_mngr_db_cfg = { { 0 } };
-
-	ipa_rsrc_mngr_db_cfg.def.rsrc_grp_sel = rsrc_group;
-
-	for (rsrc_type = 0; rsrc_type <= IPA_RSCR_MNGR_DB_RSRC_TYPE_MAX;
-	     rsrc_type++) {
-		for (rsrc_id = 0; rsrc_id <= IPA_RSCR_MNGR_DB_RSRC_ID_MAX;
-		     rsrc_id++) {
-			ipa_rsrc_mngr_db_cfg.def.rsrc_id_sel = rsrc_id;
-			ipa_rsrc_mngr_db_cfg.def.rsrc_type_sel = rsrc_type;
-			IPA_MASKED_WRITE_SCALER_REG(IPA_RSRC_MNGR_DB_CFG,
-					     ipa_rsrc_mngr_db_cfg.value);
-			ipa_reg_save.ipa.dbg.ipa_rsrc_mngr_db_rsrc_read_arr
-			    [rsrc_type][rsrc_id].value =
-			    IPA_READ_SCALER_REG(
-					IPA_RSRC_MNGR_DB_RSRC_READ);
-			ipa_reg_save.ipa.dbg.ipa_rsrc_mngr_db_list_read_arr
-			    [rsrc_type][rsrc_id].value =
-			    IPA_READ_SCALER_REG(
-					IPA_RSRC_MNGR_DB_LIST_READ);
-		}
-	}
-}
-
-/*
- * FUNCTION:  ipa_reg_save_anomaly_check
- *
- * Checks RX state and TX state upon crash dump collection and prints
- * anomalies.
- *
- * TBD- Add more anomaly checks in the future.
- *
- * @return
- */
-static void ipa_reg_save_anomaly_check(void)
-{
-	if ((ipa_reg_save.ipa.gen.ipa_state.rx_wait != 0)
-	    || (ipa_reg_save.ipa.gen.ipa_state.rx_idle != 1)) {
-		IPADBG(
-		    "RX ACTIVITY, ipa_state.rx_wait = %d, ipa_state.rx_idle = %d, ipa_state_rx_active.endpoints = %d (bitmask)\n",
-		    ipa_reg_save.ipa.gen.ipa_state.rx_wait,
-		    ipa_reg_save.ipa.gen.ipa_state.rx_idle,
-		    ipa_reg_save.ipa.gen.ipa_state_rx_active.endpoints);
-
-		if (ipa_reg_save.ipa.gen.ipa_state.tx_idle != 1) {
-			IPADBG(
-			    "TX ACTIVITY, ipa_state.idle = %d, ipa_state_tx_wrapper.tx0_idle = %d, ipa_state_tx_wrapper.tx1_idle = %d\n",
-			    ipa_reg_save.ipa.gen.ipa_state.tx_idle,
-			    ipa_reg_save.ipa.gen.ipa_state_tx_wrapper.tx0_idle,
-			    ipa_reg_save.ipa.gen.ipa_state_tx_wrapper.tx1_idle);
-
-			IPADBG(
-			    "ipa_state_tx0.last_cmd_pipe = %d, ipa_state_tx1.last_cmd_pipe = %d\n",
-			    ipa_reg_save.ipa.gen.ipa_state_tx0.last_cmd_pipe,
-			    ipa_reg_save.ipa.gen.ipa_state_tx1.last_cmd_pipe);
-		}
-	}
-}

ipa/ipa_v3/dump/ipa_reg_dump.h

@@ -1,1397 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2019, The Linux Foundation. All rights reserved.
- */
-#if !defined(_IPA_REG_DUMP_H_)
-#define _IPA_REG_DUMP_H_
-
-#include <linux/types.h>
-#include <linux/string.h>
-
-#include "../ipa_i.h"
-
-#include "ipa_pkt_cntxt.h"
-#include "ipa_hw_common_ex.h"
-
-#define IPA_0_IPA_WRAPPER_BASE 0 /* required by following includes */
-
-#include "ipa_hwio.h"
-#include "gsi_hwio.h"
-#include "ipa_gcc_hwio.h"
-
-#include "ipa_hwio_def.h"
-#include "gsi_hwio_def.h"
-#include "ipa_gcc_hwio_def.h"
-
-#define IPA_DEBUG_CMDQ_DPS_SELECT_NUM_GROUPS     0x6
-#define IPA_DEBUG_CMDQ_HPS_SELECT_NUM_GROUPS     0x4
-#define IPA_DEBUG_TESTBUS_RSRC_NUM_EP            7
-#define IPA_DEBUG_TESTBUS_RSRC_NUM_GRP           3
-#define IPA_TESTBUS_SEL_EP_MAX                   0x1F
-#define IPA_TESTBUS_SEL_EXTERNAL_MAX             0x40
-#define IPA_TESTBUS_SEL_INTERNAL_MAX             0xFF
-#define IPA_TESTBUS_SEL_INTERNAL_PIPE_MAX        0x40
-#define IPA_DEBUG_CMDQ_ACK_SELECT_NUM_GROUPS     0x9
-#define IPA_RSCR_MNGR_DB_RSRC_ID_MAX             0x3F
-#define IPA_RSCR_MNGR_DB_RSRC_TYPE_MAX           0xA
-
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_ZEROS   (0x0)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_MCS_0   (0x1)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_MCS_1   (0x2)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_MCS_2   (0x3)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_MCS_3   (0x4)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_MCS_4   (0x5)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_DB_ENG  (0x9)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_REE_0   (0xB)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_REE_1   (0xC)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_REE_2   (0xD)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_REE_3   (0xE)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_REE_4   (0xF)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_REE_5   (0x10)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_REE_6   (0x11)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_REE_7   (0x12)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_EVE_0   (0x13)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_EVE_1   (0x14)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_EVE_2   (0x15)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_EVE_3   (0x16)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_EVE_4   (0x17)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_EVE_5   (0x18)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_IE_0    (0x1B)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_IE_1    (0x1C)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_IC_0    (0x1F)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_IC_1    (0x20)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_IC_2    (0x21)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_IC_3    (0x22)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_IC_4    (0x23)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_MOQA_0  (0x27)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_MOQA_1  (0x28)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_MOQA_2  (0x29)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_MOQA_3  (0x2A)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_TMR_0   (0x2B)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_TMR_1   (0x2C)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_TMR_2   (0x2D)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_TMR_3   (0x2E)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_RD_WR_0 (0x33)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_RD_WR_1 (0x34)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_RD_WR_2 (0x35)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_RD_WR_3 (0x36)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_CSR     (0x3A)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_SDMA_0  (0x3C)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_SDMA_1  (0x3D)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_IE_2    (0x1D)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_CSR_1   (0x3E)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_CSR_2   (0x3F)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_MCS_5   (0x40)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_IC_5    (0x41)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_CSR_3   (0x42)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_TLV_0   (0x43)
-#define HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_REE_8   (0x44)
-
-#define IPA_DEBUG_TESTBUS_DEF_EXTERNAL           50
-#define IPA_DEBUG_TESTBUS_DEF_INTERNAL           6
-
-#define IPA_REG_SAVE_GSI_NUM_EE                  3
-
-#define IPA_REG_SAVE_NUM_EXTRA_ENDP_REGS         22
-
-#define IPA_GSI_OFFSET_WORDS_SCRATCH4            6
-#define IPA_GSI_OFFSET_WORDS_SCRATCH5            7
-
-#define IPA_DEBUG_TESTBUS_RSRC_TYPE_CNT_BIT_MASK 0x7E000
-#define IPA_DEBUG_TESTBUS_RSRC_TYPE_CNT_SHIFT    13
-
-#define IPA_REG_SAVE_HWP_GSI_EE                  2
-
-/*
- * A structure used to map a source address to destination address...
- */
-struct map_src_dst_addr_s {
-	u32  src_addr; /* register offset to copy value from */
-	u32 *dst_addr; /* memory address to copy register value to */
-};
-
-/*
- * A macro to generate the names of scaler (ie. non-vector) registers
- * that reside in the *hwio.h files (said files contain the manifest
- * constants for the registers' offsets in the register memory map).
- */
-#define GEN_SCALER_REG_OFST(reg_name) \
-	(HWIO_ ## reg_name ## _ADDR)
-/*
- * A macro designed to generate the rmsk associated with reg_name
- */
-#define GEN_SCALER_REG_RMSK(reg_name) \
-	(HWIO_ ## reg_name ## _RMSK)
-
-/*
- * A macro to generate the names of vector registers that reside in
- * the *hwio.h files (said files contain the manifest constants for
- * the registers' offsets in the register memory map). More
- * specifically, this macro will generate access to registers that are
- * addressed via one dimension.
- */
-#define GEN_1xVECTOR_REG_OFST(reg_name, row) \
-	(HWIO_ ## reg_name ## _ADDR(row))
-
-/*
- * A macro to generate the names of vector registers that reside in
- * the *hwio.h files (said files contain the manifest constants for
- * the registers' offsets in the register memory map). More
- * specifically, this macro will generate access to registers that are
- * addressed via two dimensions.
- */
-#define GEN_2xVECTOR_REG_OFST(reg_name, row, col) \
-	(HWIO_ ## reg_name ## _ADDR(row, col))
-
-/*
- * A macro to generate the access to scaler registers that reside in
- * the *hwio.h files (said files contain the manifest constants for
- * the registers' offsets in the register memory map). More
- * specifically, this macro will generate read access from a scaler
- * register..
- */
-#define IPA_READ_SCALER_REG(reg_name) \
-	HWIO_ ## reg_name ## _IN
-
-/*
- * A macro to generate the access to vector registers that reside in
- * the *hwio.h files (said files contain the manifest constants for
- * the registers' offsets in the register memory map). More
- * specifically, this macro will generate read access from a one
- * dimensional vector register...
- */
-#define IPA_READ_1xVECTOR_REG(reg_name, row) \
-	HWIO_ ## reg_name ## _INI(row)
-
-/*
- * A macro to generate the access to vector registers that reside in
- * the *hwio.h files (said files contain the manifest constants for
- * the registers' offsets in the register memory map). More
- * specifically, this macro will generate read access from a two
- * dimensional vector register...
- */
-#define IPA_READ_2xVECTOR_REG(reg_name, row, col) \
-	HWIO_ ## reg_name ## _INI2(row, col)
-
-/*
- * A macro to generate the access to scaler registers that reside in
- * the *hwio.h files (said files contain the manifest constants for
- * the registers' offsets in the register memory map). More
- * specifically, this macro will generate write access to a scaler
- * register..
- */
-#define IPA_WRITE_SCALER_REG(reg_name, val) \
-	HWIO_ ## reg_name ## _OUT(val)
-
-/*
- * Similar to the above, but with val masked by the register's rmsk...
- */
-#define IPA_MASKED_WRITE_SCALER_REG(reg_name, val) \
-	out_dword(GEN_SCALER_REG_OFST(reg_name), \
-			  (GEN_SCALER_REG_RMSK(reg_name) & val))
-
-/*
- * A macro to generate the access to vector registers that reside in
- * the *hwio.h files (said files contain the manifest constants for
- * the registers' offsets in the register memory map). More
- * specifically, this macro will generate write access to a one
- * dimensional vector register...
- */
-#define IPA_WRITE_1xVECTOR_REG(reg_name, row, val) \
-	HWIO_ ## reg_name ## _OUTI(row, val)
-
-/*
- * A macro to generate the access to vector registers that reside in
- * the *hwio.h files (said files contain the manifest constants for
- * the registers' offsets in the register memory map). More
- * specifically, this macro will generate write access to a two
- * dimensional vector register...
- */
-#define IPA_WRITE_2xVECTOR_REG(reg_name, row, col, val) \
-	HWIO_ ## reg_name ## _OUTI2(row, col, val)
-
-/*
- * Macro that helps generate a mapping between a register's address
- * and where the register's value will get stored (ie. source and
- * destination address mapping) upon dump...
- */
-#define GEN_SRC_DST_ADDR_MAP(reg_name, sub_struct, field_name) \
-	{ GEN_SCALER_REG_OFST(reg_name), \
-	  (u32 *)&ipa_reg_save.sub_struct.field_name }
-
-/*
- * Macro to get value of bits 18:13, used tp get rsrc cnts from
- * IPA_DEBUG_DATA
- */
-#define IPA_DEBUG_TESTBUS_DATA_GET_RSRC_CNT_BITS_FROM_DEBUG_DATA(x) \
-	((x & IPA_DEBUG_TESTBUS_RSRC_TYPE_CNT_BIT_MASK) >> \
-	 IPA_DEBUG_TESTBUS_RSRC_TYPE_CNT_SHIFT)
-
-/*
- * Macro to get rsrc cnt of specific rsrc type and rsrc grp from test
- * bus collected data
- */
-#define IPA_DEBUG_TESTBUS_GET_RSRC_TYPE_CNT(rsrc_type, rsrc_grp) \
-	IPA_DEBUG_TESTBUS_DATA_GET_RSRC_CNT_BITS_FROM_DEBUG_DATA( \
-		ipa_reg_save.ipa.testbus->ep_rsrc[rsrc_type].entry_ep \
-		[rsrc_grp].testbus_data.value)
-
-/*
- * Macro to pluck the gsi version from ram.
- */
-#define IPA_REG_SAVE_GSI_VER(reg_name, var_name)	\
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 0), \
-		(u32 *)&ipa_reg_save.gsi.gen.var_name }
-/*
- * Macro to define a particular register cfg entry for all 3 EE
- * indexed register
- */
-#define IPA_REG_SAVE_CFG_ENTRY_GEN_EE(reg_name, var_name) \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, IPA_HW_Q6_EE), \
-		(u32 *)&ipa_reg_save.ipa.gen_ee[IPA_HW_Q6_EE].var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE), \
-		(u32 *)&ipa_reg_save.ipa.gen_ee[IPA_HW_A7_EE].var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, IPA_HW_HWP_EE), \
-		(u32 *)&ipa_reg_save.ipa.gen_ee[IPA_HW_HWP_EE].var_name }
-
-#define IPA_REG_SAVE_CFG_ENTRY_GSI_FIFO(reg_name, var_name, index) \
-	{ GEN_SCALER_REG_OFST(reg_name), \
-		(u32 *)&ipa_reg_save.ipa.gsi_fifo_status[index].var_name }
-
-/*
- * Macro to define a particular register cfg entry for all pipe
- * indexed register
- */
-#define IPA_REG_SAVE_CFG_ENTRY_PIPE_ENDP_EXTRA(reg_name, var_name) \
-	{ 0, 0 }
-
-/*
- * Macro to define a particular register cfg entry for all resource
- * group register
- */
-#define IPA_REG_SAVE_CFG_ENTRY_SRC_RSRC_GRP(reg_name, var_name) \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 0), \
-		(u32 *)&ipa_reg_save.ipa.src_rsrc_grp[0].var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 1), \
-		(u32 *)&ipa_reg_save.ipa.src_rsrc_grp[1].var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 2), \
-		(u32 *)&ipa_reg_save.ipa.src_rsrc_grp[2].var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 3), \
-		(u32 *)&ipa_reg_save.ipa.src_rsrc_grp[3].var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 4), \
-		(u32 *)&ipa_reg_save.ipa.src_rsrc_grp[4].var_name }
-
-/*
- * Macro to define a particular register cfg entry for all resource
- * group register
- */
-#define IPA_REG_SAVE_CFG_ENTRY_DST_RSRC_GRP(reg_name, var_name) \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 0), \
-		(u32 *)&ipa_reg_save.ipa.dst_rsrc_grp[0].var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 1), \
-		(u32 *)&ipa_reg_save.ipa.dst_rsrc_grp[1].var_name }
-
-/*
- * Macro to define a particular register cfg entry for all source
- * resource group count register
- */
-#define IPA_REG_SAVE_CFG_ENTRY_SRC_RSRC_CNT_GRP(reg_name, var_name) \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 0), \
-		(u32 *)&ipa_reg_save.ipa.src_rsrc_cnt[0].var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 1), \
-		(u32 *)&ipa_reg_save.ipa.src_rsrc_cnt[1].var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 2), \
-		(u32 *)&ipa_reg_save.ipa.src_rsrc_cnt[2].var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 3), \
-		(u32 *)&ipa_reg_save.ipa.src_rsrc_cnt[3].var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 4), \
-		(u32 *)&ipa_reg_save.ipa.src_rsrc_cnt[4].var_name }
-
-/*
- * Macro to define a particular register cfg entry for all dest
- * resource group count register
- */
-#define IPA_REG_SAVE_CFG_ENTRY_DST_RSRC_CNT_GRP(reg_name, var_name) \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 0), \
-		(u32 *)&ipa_reg_save.ipa.dst_rsrc_cnt[0].var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 1), \
-		(u32 *)&ipa_reg_save.ipa.dst_rsrc_cnt[1].var_name }
-
-#define IPA_REG_SAVE_CFG_ENTRY_GSI_GENERAL_EE(reg_name, var_name) \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE), \
-		(u32 *)&ipa_reg_save.gsi.gen_ee[IPA_HW_A7_EE].var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, IPA_HW_Q6_EE), \
-		(u32 *)&ipa_reg_save.gsi.gen_ee[IPA_HW_Q6_EE].var_name }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, IPA_REG_SAVE_HWP_GSI_EE), \
-		(u32 *)&ipa_reg_save.gsi.gen_ee[IPA_REG_SAVE_HWP_GSI_EE].\
-			var_name }
-
-/*
- * Macro to define a particular register cfg entry for all GSI EE
- * register
- */
-#define IPA_REG_SAVE_CFG_ENTRY_GSI_CH_CNTXT(reg_name, var_name) \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 0), \
-		(u32 *)&ipa_reg_save.gsi.ch_cntxt.a7[0].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 1), \
-		(u32 *)&ipa_reg_save.gsi.ch_cntxt.a7[1].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 2), \
-		(u32 *)&ipa_reg_save.gsi.ch_cntxt.a7[2].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 3), \
-		(u32 *)&ipa_reg_save.gsi.ch_cntxt.a7[3].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 4), \
-		(u32 *)&ipa_reg_save.gsi.ch_cntxt.a7[4].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 5), \
-		(u32 *)&ipa_reg_save.gsi.ch_cntxt.a7[5].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 6), \
-		(u32 *)&ipa_reg_save.gsi.ch_cntxt.a7[6].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 7), \
-		(u32 *)&ipa_reg_save.gsi.ch_cntxt.a7[7].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 8), \
-		(u32 *)&ipa_reg_save.gsi.ch_cntxt.a7[8].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 9), \
-		(u32 *)&ipa_reg_save.gsi.ch_cntxt.a7[9].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 10), \
-		(u32 *)&ipa_reg_save.gsi.ch_cntxt.a7[10].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 11), \
-		(u32 *)&ipa_reg_save.gsi.ch_cntxt.a7[11].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 12), \
-		(u32 *)&ipa_reg_save.gsi.ch_cntxt.a7[12].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 13), \
-		(u32 *)&ipa_reg_save.gsi.ch_cntxt.a7[13].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 14), \
-		(u32 *)&ipa_reg_save.gsi.ch_cntxt.a7[14].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 15), \
-		(u32 *)&ipa_reg_save.gsi.ch_cntxt.a7[15].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 16), \
-		(u32 *)&ipa_reg_save.gsi.ch_cntxt.a7[16].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 17), \
-		(u32 *)&ipa_reg_save.gsi.ch_cntxt.a7[17].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 18), \
-		(u32 *)&ipa_reg_save.gsi.ch_cntxt.a7[18].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 19), \
-		(u32 *)&ipa_reg_save.gsi.ch_cntxt.a7[19].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_REG_SAVE_HWP_GSI_EE, 1),	\
-		(u32 *)&ipa_reg_save.gsi.ch_cntxt.uc[0].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_REG_SAVE_HWP_GSI_EE, 3), \
-		(u32 *)&ipa_reg_save.gsi.ch_cntxt.uc[1].var_name }
-
-#define IPA_REG_SAVE_CFG_ENTRY_GSI_EVT_CNTXT(reg_name, var_name) \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 0), \
-		(u32 *)&ipa_reg_save.gsi.evt_cntxt.a7[0].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 1), \
-		(u32 *)&ipa_reg_save.gsi.evt_cntxt.a7[1].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 2), \
-		(u32 *)&ipa_reg_save.gsi.evt_cntxt.a7[2].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 3), \
-		(u32 *)&ipa_reg_save.gsi.evt_cntxt.a7[3].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 4), \
-		(u32 *)&ipa_reg_save.gsi.evt_cntxt.a7[4].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 5), \
-		(u32 *)&ipa_reg_save.gsi.evt_cntxt.a7[5].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 6), \
-		(u32 *)&ipa_reg_save.gsi.evt_cntxt.a7[6].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 7), \
-		(u32 *)&ipa_reg_save.gsi.evt_cntxt.a7[7].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 8), \
-		(u32 *)&ipa_reg_save.gsi.evt_cntxt.a7[8].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 9), \
-		(u32 *)&ipa_reg_save.gsi.evt_cntxt.a7[9].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 10), \
-		(u32 *)&ipa_reg_save.gsi.evt_cntxt.a7[10].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 11), \
-		(u32 *)&ipa_reg_save.gsi.evt_cntxt.a7[11].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 12), \
-		(u32 *)&ipa_reg_save.gsi.evt_cntxt.a7[12].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 13), \
-		(u32 *)&ipa_reg_save.gsi.evt_cntxt.a7[13].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 14), \
-		(u32 *)&ipa_reg_save.gsi.evt_cntxt.a7[14].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 15), \
-		(u32 *)&ipa_reg_save.gsi.evt_cntxt.a7[15].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 16), \
-		(u32 *)&ipa_reg_save.gsi.evt_cntxt.a7[16].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 17), \
-		(u32 *)&ipa_reg_save.gsi.evt_cntxt.a7[17].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_HW_A7_EE, 18), \
-		(u32 *)&ipa_reg_save.gsi.evt_cntxt.a7[18].var_name }, \
-	{ GEN_2xVECTOR_REG_OFST(reg_name, IPA_REG_SAVE_HWP_GSI_EE, 1), \
-		(u32 *)&ipa_reg_save.gsi.evt_cntxt.uc[0].var_name }
-
-/*
- * Macro to define a particular register cfg entry for GSI QSB debug
- * registers
- */
-#define IPA_REG_SAVE_CFG_ENTRY_GSI_QSB_DEBUG(reg_name, var_name) \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 0), \
-		(u32 *)&ipa_reg_save.gsi.debug.gsi_qsb_debug.var_name[0] }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 1), \
-		(u32 *)&ipa_reg_save.gsi.debug.gsi_qsb_debug.var_name[1] }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 2), \
-		(u32 *)&ipa_reg_save.gsi.debug.gsi_qsb_debug.var_name[2] }, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 3), \
-		(u32 *)&ipa_reg_save.gsi.debug.gsi_qsb_debug.var_name[3] }
-
-#define IPA_REG_SAVE_RX_SPLT_CMDQ(reg_name, var_name) \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 0), \
-		(u32 *)&ipa_reg_save.ipa.dbg.var_name[0]}, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 1), \
-		(u32 *)&ipa_reg_save.ipa.dbg.var_name[1]}, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 2), \
-		(u32 *)&ipa_reg_save.ipa.dbg.var_name[2]}, \
-	{ GEN_1xVECTOR_REG_OFST(reg_name, 3), \
-		(u32 *)&ipa_reg_save.ipa.dbg.var_name[3]}
-
-/*
- * IPA HW Platform Type
- */
-enum ipa_hw_ee_e {
-	IPA_HW_A7_EE  = 0, /* A7's execution environment */
-	IPA_HW_Q6_EE  = 1, /* Q6's execution environment */
-	IPA_HW_HWP_EE = 3, /* HWP's execution environment */
-	IPA_HW_EE_MAX,     /* Max EE to support */
-};
-
-/*
- * General IPA register save data struct (ie. this is where register
- * values, once read, get placed...
- */
-struct ipa_gen_regs_s {
-	struct ipa_hwio_def_ipa_state_s
-	  ipa_state;
-	struct ipa_hwio_def_ipa_state_rx_active_s
-	  ipa_state_rx_active;
-	struct ipa_hwio_def_ipa_state_tx_wrapper_s
-	  ipa_state_tx_wrapper;
-	struct ipa_hwio_def_ipa_state_tx0_s
-	  ipa_state_tx0;
-	struct ipa_hwio_def_ipa_state_tx1_s
-	  ipa_state_tx1;
-	struct ipa_hwio_def_ipa_state_aggr_active_s
-	  ipa_state_aggr_active;
-	struct ipa_hwio_def_ipa_state_dfetcher_s
-	  ipa_state_dfetcher;
-	struct ipa_hwio_def_ipa_state_fetcher_mask_0_s
-	  ipa_state_fetcher_mask_0;
-	struct ipa_hwio_def_ipa_state_fetcher_mask_1_s
-	  ipa_state_fetcher_mask_1;
-	struct ipa_hwio_def_ipa_state_gsi_aos_s
-	  ipa_state_gsi_aos;
-	struct ipa_hwio_def_ipa_state_gsi_if_s
-	  ipa_state_gsi_if;
-	struct ipa_hwio_def_ipa_state_gsi_skip_s
-	  ipa_state_gsi_skip;
-	struct ipa_hwio_def_ipa_state_gsi_tlv_s
-	  ipa_state_gsi_tlv;
-	struct ipa_hwio_def_ipa_dpl_timer_lsb_s
-	  ipa_dpl_timer_lsb;
-	struct ipa_hwio_def_ipa_dpl_timer_msb_s
-	  ipa_dpl_timer_msb;
-	struct ipa_hwio_def_ipa_proc_iph_cfg_s
-	  ipa_proc_iph_cfg;
-	struct ipa_hwio_def_ipa_route_s
-	  ipa_route;
-	struct ipa_hwio_def_ipa_spare_reg_1_s
-	  ipa_spare_reg_1;
-	struct ipa_hwio_def_ipa_spare_reg_2_s
-	  ipa_spare_reg_2;
-	struct ipa_hwio_def_ipa_log_s
-	  ipa_log;
-	struct ipa_hwio_def_ipa_log_buf_status_cfg_s
-	  ipa_log_buf_status_cfg;
-	struct ipa_hwio_def_ipa_log_buf_status_addr_s
-	  ipa_log_buf_status_addr;
-	struct ipa_hwio_def_ipa_log_buf_status_write_ptr_s
-	  ipa_log_buf_status_write_ptr;
-	struct ipa_hwio_def_ipa_log_buf_status_ram_ptr_s
-	  ipa_log_buf_status_ram_ptr;
-	struct ipa_hwio_def_ipa_log_buf_hw_cmd_cfg_s
-	  ipa_log_buf_hw_cmd_cfg;
-	struct ipa_hwio_def_ipa_log_buf_hw_cmd_addr_s
-	  ipa_log_buf_hw_cmd_addr;
-	struct ipa_hwio_def_ipa_log_buf_hw_cmd_write_ptr_s
-	  ipa_log_buf_hw_cmd_write_ptr;
-	struct ipa_hwio_def_ipa_log_buf_hw_cmd_ram_ptr_s
-	  ipa_log_buf_hw_cmd_ram_ptr;
-	struct ipa_hwio_def_ipa_comp_hw_version_s
-	  ipa_comp_hw_version;
-	struct ipa_hwio_def_ipa_filt_rout_hash_en_s
-	  ipa_filt_rout_hash_en;
-	struct ipa_hwio_def_ipa_filt_rout_hash_flush_s
-	  ipa_filt_rout_hash_flush;
-	struct ipa_hwio_def_ipa_state_fetcher_s
-	  ipa_state_fetcher;
-	struct ipa_hwio_def_ipa_ipv4_filter_init_values_s
-	  ipa_ipv4_filter_init_values;
-	struct ipa_hwio_def_ipa_ipv6_filter_init_values_s
-	  ipa_ipv6_filter_init_values;
-	struct ipa_hwio_def_ipa_ipv4_route_init_values_s
-	  ipa_ipv4_route_init_values;
-	struct ipa_hwio_def_ipa_ipv6_route_init_values_s
-	  ipa_ipv6_route_init_values;
-	struct ipa_hwio_def_ipa_bam_activated_ports_s
-	  ipa_bam_activated_ports;
-	struct ipa_hwio_def_ipa_tx_commander_cmdq_status_s
-	  ipa_tx_commander_cmdq_status;
-	struct ipa_hwio_def_ipa_log_buf_hw_snif_el_en_s
-	  ipa_log_buf_hw_snif_el_en;
-	struct ipa_hwio_def_ipa_log_buf_hw_snif_el_wr_n_rd_sel_s
-	  ipa_log_buf_hw_snif_el_wr_n_rd_sel;
-	struct ipa_hwio_def_ipa_log_buf_hw_snif_el_cli_mux_s
-	  ipa_log_buf_hw_snif_el_cli_mux;
-	struct ipa_hwio_def_ipa_state_acl_s
-	  ipa_state_acl;
-	struct ipa_hwio_def_ipa_sys_pkt_proc_cntxt_base_s
-	  ipa_sys_pkt_proc_cntxt_base;
-	struct ipa_hwio_def_ipa_sys_pkt_proc_cntxt_base_msb_s
-	  ipa_sys_pkt_proc_cntxt_base_msb;
-	struct ipa_hwio_def_ipa_local_pkt_proc_cntxt_base_s
-	  ipa_local_pkt_proc_cntxt_base;
-	struct ipa_hwio_def_ipa_rsrc_grp_cfg_s
-	  ipa_rsrc_grp_cfg;
-	struct ipa_hwio_def_ipa_comp_cfg_s
-	  ipa_comp_cfg;
-	struct ipa_hwio_def_ipa_state_dpl_fifo_s
-	  ipa_state_dpl_fifo;
-	struct ipa_hwio_def_ipa_pipeline_disable_s
-	  ipa_pipeline_disable;
-	struct ipa_hwio_def_ipa_state_nlo_aggr_s
-	  ipa_state_nlo_aggr;
-	struct ipa_hwio_def_ipa_nlo_pp_cfg1_s
-	  ipa_nlo_pp_cfg1;
-	struct ipa_hwio_def_ipa_nlo_pp_cfg2_s
-	  ipa_nlo_pp_cfg2;
-	struct ipa_hwio_def_ipa_nlo_pp_ack_limit_cfg_s
-	  ipa_nlo_pp_ack_limit_cfg;
-	struct ipa_hwio_def_ipa_nlo_pp_data_limit_cfg_s
-	  ipa_nlo_pp_data_limit_cfg;
-	struct ipa_hwio_def_ipa_nlo_min_dsm_cfg_s
-	  ipa_nlo_min_dsm_cfg;
-	struct ipa_hwio_def_ipa_nlo_vp_flush_req_s
-	  ipa_nlo_vp_flush_req;
-	struct ipa_hwio_def_ipa_nlo_vp_flush_cookie_s
-	  ipa_nlo_vp_flush_cookie;
-	struct ipa_hwio_def_ipa_nlo_vp_flush_ack_s
-	  ipa_nlo_vp_flush_ack;
-	struct ipa_hwio_def_ipa_nlo_vp_dsm_open_s
-	  ipa_nlo_vp_dsm_open;
-	struct ipa_hwio_def_ipa_nlo_vp_qbap_open_s
-	  ipa_nlo_vp_qbap_open;
-};
-
-/*
- * General IPA register save data struct
- */
-struct ipa_reg_save_gen_ee_s {
-	struct ipa_hwio_def_ipa_irq_stts_ee_n_s
-	  ipa_irq_stts_ee_n;
-	struct ipa_hwio_def_ipa_irq_en_ee_n_s
-	  ipa_irq_en_ee_n;
-	struct ipa_hwio_def_ipa_fec_addr_ee_n_s
-	  ipa_fec_addr_ee_n;
-	struct ipa_hwio_def_ipa_fec_attr_ee_n_s
-	  ipa_fec_attr_ee_n;
-	struct ipa_hwio_def_ipa_snoc_fec_ee_n_s
-	  ipa_snoc_fec_ee_n;
-	struct ipa_hwio_def_ipa_holb_drop_irq_info_ee_n_s
-	  ipa_holb_drop_irq_info_ee_n;
-	struct ipa_hwio_def_ipa_suspend_irq_info_ee_n_s
-	  ipa_suspend_irq_info_ee_n;
-	struct ipa_hwio_def_ipa_suspend_irq_en_ee_n_s
-	  ipa_suspend_irq_en_ee_n;
-};
-
-/*
- * Pipe Endp IPA register save data struct
- */
-struct ipa_reg_save_pipe_endp_s {
-	struct ipa_hwio_def_ipa_endp_init_ctrl_n_s
-	  ipa_endp_init_ctrl_n;
-	struct ipa_hwio_def_ipa_endp_init_ctrl_scnd_n_s
-	  ipa_endp_init_ctrl_scnd_n;
-	struct ipa_hwio_def_ipa_endp_init_cfg_n_s
-	  ipa_endp_init_cfg_n;
-	struct ipa_hwio_def_ipa_endp_init_nat_n_s
-	  ipa_endp_init_nat_n;
-	struct ipa_hwio_def_ipa_endp_init_hdr_n_s
-	  ipa_endp_init_hdr_n;
-	struct ipa_hwio_def_ipa_endp_init_hdr_ext_n_s
-	  ipa_endp_init_hdr_ext_n;
-	struct ipa_hwio_def_ipa_endp_init_hdr_metadata_mask_n_s
-	  ipa_endp_init_hdr_metadata_mask_n;
-	struct ipa_hwio_def_ipa_endp_init_hdr_metadata_n_s
-	  ipa_endp_init_hdr_metadata_n;
-	struct ipa_hwio_def_ipa_endp_init_mode_n_s
-	  ipa_endp_init_mode_n;
-	struct ipa_hwio_def_ipa_endp_init_aggr_n_s
-	  ipa_endp_init_aggr_n;
-	struct ipa_hwio_def_ipa_endp_init_hol_block_en_n_s
-	  ipa_endp_init_hol_block_en_n;
-	struct ipa_hwio_def_ipa_endp_init_hol_block_timer_n_s
-	  ipa_endp_init_hol_block_timer_n;
-	struct ipa_hwio_def_ipa_endp_init_deaggr_n_s
-	  ipa_endp_init_deaggr_n;
-	struct ipa_hwio_def_ipa_endp_status_n_s
-	  ipa_endp_status_n;
-	struct ipa_hwio_def_ipa_endp_init_rsrc_grp_n_s
-	  ipa_endp_init_rsrc_grp_n;
-	struct ipa_hwio_def_ipa_endp_init_seq_n_s
-	  ipa_endp_init_seq_n;
-	struct ipa_hwio_def_ipa_endp_gsi_cfg_tlv_n_s
-	  ipa_endp_gsi_cfg_tlv_n;
-	struct ipa_hwio_def_ipa_endp_gsi_cfg_aos_n_s
-	  ipa_endp_gsi_cfg_aos_n;
-	struct ipa_hwio_def_ipa_endp_gsi_cfg1_n_s
-	  ipa_endp_gsi_cfg1_n;
-	struct ipa_hwio_def_ipa_endp_filter_router_hsh_cfg_n_s
-	  ipa_endp_filter_router_hsh_cfg_n;
-};
-
-/*
- * Pipe IPA register save data struct
- */
-struct ipa_reg_save_pipe_s {
-	u8				active;
-	struct ipa_reg_save_pipe_endp_s endp;
-};
-
-/*
- * HWP IPA register save data struct
- */
-struct ipa_reg_save_hwp_s {
-	struct ipa_hwio_def_ipa_uc_qmb_sys_addr_s
-	  ipa_uc_qmb_sys_addr;
-	struct ipa_hwio_def_ipa_uc_qmb_local_addr_s
-	  ipa_uc_qmb_local_addr;
-	struct ipa_hwio_def_ipa_uc_qmb_length_s
-	  ipa_uc_qmb_length;
-	struct ipa_hwio_def_ipa_uc_qmb_trigger_s
-	  ipa_uc_qmb_trigger;
-	struct ipa_hwio_def_ipa_uc_qmb_pending_tid_s
-	  ipa_uc_qmb_pending_tid;
-	struct ipa_hwio_def_ipa_uc_qmb_completed_rd_fifo_peek_s
-	  ipa_uc_qmb_completed_rd_fifo_peek;
-	struct ipa_hwio_def_ipa_uc_qmb_completed_wr_fifo_peek_s
-	  ipa_uc_qmb_completed_wr_fifo_peek;
-	struct ipa_hwio_def_ipa_uc_qmb_misc_s
-	  ipa_uc_qmb_misc;
-	struct ipa_hwio_def_ipa_uc_qmb_status_s
-	  ipa_uc_qmb_status;
-	struct ipa_hwio_def_ipa_uc_qmb_bus_attrib_s
-	  ipa_uc_qmb_bus_attrib;
-};
-
-/*
- * IPA TESTBUS entry struct
- */
-struct ipa_reg_save_ipa_testbus_entry_s {
-	union ipa_hwio_def_ipa_testbus_sel_u testbus_sel;
-	union ipa_hwio_def_ipa_debug_data_u testbus_data;
-};
-
-/* IPA TESTBUS global struct */
-struct ipa_reg_save_ipa_testbus_global_s {
-	struct ipa_reg_save_ipa_testbus_entry_s
-	global[IPA_TESTBUS_SEL_INTERNAL_MAX + 1]
-	[IPA_TESTBUS_SEL_EXTERNAL_MAX + 1];
-};
-
-/* IPA TESTBUS per EP struct */
-struct ipa_reg_save_ipa_testbus_ep_s {
-	struct ipa_reg_save_ipa_testbus_entry_s
-	entry_ep[IPA_TESTBUS_SEL_INTERNAL_PIPE_MAX + 1]
-	[IPA_TESTBUS_SEL_EXTERNAL_MAX + 1];
-};
-
-/* IPA TESTBUS per EP struct */
-struct ipa_reg_save_ipa_testbus_ep_rsrc_s {
-	struct ipa_reg_save_ipa_testbus_entry_s
-	  entry_ep[IPA_DEBUG_TESTBUS_RSRC_NUM_GRP];
-};
-
-/* IPA TESTBUS save data struct */
-struct ipa_reg_save_ipa_testbus_s {
-	struct ipa_reg_save_ipa_testbus_global_s global;
-	struct ipa_reg_save_ipa_testbus_ep_s
-	  ep[IPA_TESTBUS_SEL_EP_MAX + 1];
-	struct ipa_reg_save_ipa_testbus_ep_rsrc_s
-	  ep_rsrc[IPA_DEBUG_TESTBUS_RSRC_NUM_EP];
-};
-
-/*
- * Debug IPA register save data struct
- */
-struct ipa_reg_save_dbg_s {
-	struct ipa_hwio_def_ipa_debug_data_s
-	  ipa_debug_data;
-	struct ipa_hwio_def_ipa_step_mode_status_s
-	  ipa_step_mode_status;
-	struct ipa_hwio_def_ipa_step_mode_breakpoints_s
-	  ipa_step_mode_breakpoints;
-	struct ipa_hwio_def_ipa_rx_splt_cmdq_cmd_n_s
-	  ipa_rx_splt_cmdq_cmd_n[IPA_RX_SPLT_CMDQ_MAX];
-	struct ipa_hwio_def_ipa_rx_splt_cmdq_cfg_n_s
-	 ipa_rx_splt_cmdq_cfg_n[IPA_RX_SPLT_CMDQ_MAX];
-	struct ipa_hwio_def_ipa_rx_splt_cmdq_data_wr_0_n_s
-	  ipa_rx_splt_cmdq_data_wr_0_n[IPA_RX_SPLT_CMDQ_MAX];
-	struct ipa_hwio_def_ipa_rx_splt_cmdq_data_wr_1_n_s
-	  ipa_rx_splt_cmdq_data_wr_1_n[IPA_RX_SPLT_CMDQ_MAX];
-	struct ipa_hwio_def_ipa_rx_splt_cmdq_data_wr_2_n_s
-	  ipa_rx_splt_cmdq_data_wr_2_n[IPA_RX_SPLT_CMDQ_MAX];
-	struct ipa_hwio_def_ipa_rx_splt_cmdq_data_wr_3_n_s
-	  ipa_rx_splt_cmdq_data_wr_3_n[IPA_RX_SPLT_CMDQ_MAX];
-	struct ipa_hwio_def_ipa_rx_splt_cmdq_data_rd_0_n_s
-	  ipa_rx_splt_cmdq_data_rd_0_n[IPA_RX_SPLT_CMDQ_MAX];
-	struct ipa_hwio_def_ipa_rx_splt_cmdq_data_rd_1_n_s
-	  ipa_rx_splt_cmdq_data_rd_1_n[IPA_RX_SPLT_CMDQ_MAX];
-	struct ipa_hwio_def_ipa_rx_splt_cmdq_data_rd_2_n_s
-	  ipa_rx_splt_cmdq_data_rd_2_n[IPA_RX_SPLT_CMDQ_MAX];
-	struct ipa_hwio_def_ipa_rx_splt_cmdq_data_rd_3_n_s
-	  ipa_rx_splt_cmdq_data_rd_3_n[IPA_RX_SPLT_CMDQ_MAX];
-	struct ipa_hwio_def_ipa_rx_splt_cmdq_status_n_s
-	  ipa_rx_splt_cmdq_status_n[IPA_RX_SPLT_CMDQ_MAX];
-
-	union ipa_hwio_def_ipa_rx_hps_cmdq_cfg_wr_u
-		ipa_rx_hps_cmdq_cfg_wr;
-	union ipa_hwio_def_ipa_rx_hps_cmdq_cfg_rd_u
-		ipa_rx_hps_cmdq_cfg_rd;
-
-	struct ipa_hwio_def_ipa_rx_hps_cmdq_cmd_s
-	  ipa_rx_hps_cmdq_cmd;
-	union ipa_hwio_def_ipa_rx_hps_cmdq_data_rd_0_u
-		ipa_rx_hps_cmdq_data_rd_0_arr[
-		IPA_DEBUG_CMDQ_HPS_SELECT_NUM_GROUPS];
-	union ipa_hwio_def_ipa_rx_hps_cmdq_data_rd_1_u
-		ipa_rx_hps_cmdq_data_rd_1_arr[
-		IPA_DEBUG_CMDQ_HPS_SELECT_NUM_GROUPS];
-	union ipa_hwio_def_ipa_rx_hps_cmdq_data_rd_2_u
-		ipa_rx_hps_cmdq_data_rd_2_arr[
-		IPA_DEBUG_CMDQ_HPS_SELECT_NUM_GROUPS];
-	union ipa_hwio_def_ipa_rx_hps_cmdq_data_rd_3_u
-		ipa_rx_hps_cmdq_data_rd_3_arr[
-		IPA_DEBUG_CMDQ_HPS_SELECT_NUM_GROUPS];
-	union ipa_hwio_def_ipa_rx_hps_cmdq_count_u
-	  ipa_rx_hps_cmdq_count_arr[IPA_DEBUG_CMDQ_HPS_SELECT_NUM_GROUPS];
-	union ipa_hwio_def_ipa_rx_hps_cmdq_status_u
-	  ipa_rx_hps_cmdq_status_arr[IPA_DEBUG_CMDQ_HPS_SELECT_NUM_GROUPS];
-	struct ipa_hwio_def_ipa_rx_hps_cmdq_status_empty_s
-	  ipa_rx_hps_cmdq_status_empty;
-	struct ipa_hwio_def_ipa_rx_hps_clients_min_depth_0_s
-	  ipa_rx_hps_clients_min_depth_0;
-	struct ipa_hwio_def_ipa_rx_hps_clients_max_depth_0_s
-	  ipa_rx_hps_clients_max_depth_0;
-	struct ipa_hwio_def_ipa_hps_dps_cmdq_cmd_s
-	  ipa_hps_dps_cmdq_cmd;
-	union ipa_hwio_def_ipa_hps_dps_cmdq_data_rd_0_u
-		ipa_hps_dps_cmdq_data_rd_0_arr[IPA_TESTBUS_SEL_EP_MAX + 1];
-	union ipa_hwio_def_ipa_hps_dps_cmdq_count_u
-		ipa_hps_dps_cmdq_count_arr[IPA_TESTBUS_SEL_EP_MAX + 1];
-	union ipa_hwio_def_ipa_hps_dps_cmdq_status_u
-		ipa_hps_dps_cmdq_status_arr[IPA_TESTBUS_SEL_EP_MAX + 1];
-	struct ipa_hwio_def_ipa_hps_dps_cmdq_status_empty_s
-	  ipa_hps_dps_cmdq_status_empty;
-
-	struct ipa_hwio_def_ipa_dps_tx_cmdq_cmd_s
-	  ipa_dps_tx_cmdq_cmd;
-	union ipa_hwio_def_ipa_dps_tx_cmdq_data_rd_0_u
-		ipa_dps_tx_cmdq_data_rd_0_arr[
-		IPA_DEBUG_CMDQ_DPS_SELECT_NUM_GROUPS];
-	union ipa_hwio_def_ipa_dps_tx_cmdq_count_u
-		ipa_dps_tx_cmdq_count_arr[IPA_DEBUG_CMDQ_DPS_SELECT_NUM_GROUPS];
-	union ipa_hwio_def_ipa_dps_tx_cmdq_status_u
-	ipa_dps_tx_cmdq_status_arr[IPA_DEBUG_CMDQ_DPS_SELECT_NUM_GROUPS];
-	struct ipa_hwio_def_ipa_dps_tx_cmdq_status_empty_s
-	  ipa_dps_tx_cmdq_status_empty;
-
-	struct ipa_hwio_def_ipa_ackmngr_cmdq_cmd_s
-	  ipa_ackmngr_cmdq_cmd;
-	union ipa_hwio_def_ipa_ackmngr_cmdq_data_rd_u
-		ipa_ackmngr_cmdq_data_rd_arr[
-		IPA_DEBUG_CMDQ_ACK_SELECT_NUM_GROUPS];
-	union ipa_hwio_def_ipa_ackmngr_cmdq_count_u
-	  ipa_ackmngr_cmdq_count_arr[IPA_DEBUG_CMDQ_ACK_SELECT_NUM_GROUPS];
-	union ipa_hwio_def_ipa_ackmngr_cmdq_status_u
-		ipa_ackmngr_cmdq_status_arr[
-		IPA_DEBUG_CMDQ_ACK_SELECT_NUM_GROUPS];
-	struct ipa_hwio_def_ipa_ackmngr_cmdq_status_empty_s
-	  ipa_ackmngr_cmdq_status_empty;
-
-	struct ipa_hwio_def_ipa_prod_ackmngr_cmdq_cmd_s
-	  ipa_prod_ackmngr_cmdq_cmd;
-	union ipa_hwio_def_ipa_prod_ackmngr_cmdq_data_rd_u
-		ipa_prod_ackmngr_cmdq_data_rd_arr[IPA_TESTBUS_SEL_EP_MAX + 1];
-	union ipa_hwio_def_ipa_prod_ackmngr_cmdq_count_u
-		ipa_prod_ackmngr_cmdq_count_arr[IPA_TESTBUS_SEL_EP_MAX + 1];
-	union ipa_hwio_def_ipa_prod_ackmngr_cmdq_status_u
-		ipa_prod_ackmngr_cmdq_status_arr[IPA_TESTBUS_SEL_EP_MAX + 1];
-	struct ipa_hwio_def_ipa_prod_ackmngr_cmdq_status_empty_s
-	  ipa_prod_ackmngr_cmdq_status_empty;
-
-	struct ipa_hwio_def_ipa_ntf_tx_cmdq_cmd_s
-	  ipa_ntf_tx_cmdq_cmd;
-	union ipa_hwio_def_ipa_ntf_tx_cmdq_data_rd_0_u
-		ipa_ntf_tx_cmdq_data_rd_0_arr[IPA_TESTBUS_SEL_EP_MAX + 1];
-	union ipa_hwio_def_ipa_ntf_tx_cmdq_count_u
-		ipa_ntf_tx_cmdq_count_arr[IPA_TESTBUS_SEL_EP_MAX + 1];
-	union ipa_hwio_def_ipa_ntf_tx_cmdq_status_u
-		ipa_ntf_tx_cmdq_status_arr[IPA_TESTBUS_SEL_EP_MAX + 1];
-	struct ipa_hwio_def_ipa_ntf_tx_cmdq_status_empty_s
-	  ipa_ntf_tx_cmdq_status_empty;
-
-	union ipa_hwio_def_ipa_rsrc_mngr_db_rsrc_read_u
-		ipa_rsrc_mngr_db_rsrc_read_arr[IPA_RSCR_MNGR_DB_RSRC_TYPE_MAX +
-					       1][IPA_RSCR_MNGR_DB_RSRC_ID_MAX
-						  + 1];
-	union ipa_hwio_def_ipa_rsrc_mngr_db_list_read_u
-		ipa_rsrc_mngr_db_list_read_arr[IPA_RSCR_MNGR_DB_RSRC_TYPE_MAX +
-					       1][IPA_RSCR_MNGR_DB_RSRC_ID_MAX
-						  + 1];
-};
-
-/* Source Resource Group IPA register save data struct */
-struct ipa_reg_save_src_rsrc_grp_s {
-	struct ipa_hwio_def_ipa_src_rsrc_grp_01_rsrc_type_n_s
-	  ipa_src_rsrc_grp_01_rsrc_type_n;
-	struct ipa_hwio_def_ipa_src_rsrc_grp_23_rsrc_type_n_s
-	  ipa_src_rsrc_grp_23_rsrc_type_n;
-	struct ipa_hwio_def_ipa_src_rsrc_grp_45_rsrc_type_n_s
-	  ipa_src_rsrc_grp_45_rsrc_type_n;
-};
-
-/* Source Resource Group IPA register save data struct */
-struct ipa_reg_save_dst_rsrc_grp_s {
-	struct ipa_hwio_def_ipa_dst_rsrc_grp_01_rsrc_type_n_s
-	  ipa_dst_rsrc_grp_01_rsrc_type_n;
-	struct ipa_hwio_def_ipa_dst_rsrc_grp_23_rsrc_type_n_s
-	  ipa_dst_rsrc_grp_23_rsrc_type_n;
-	struct ipa_hwio_def_ipa_dst_rsrc_grp_45_rsrc_type_n_s
-	  ipa_dst_rsrc_grp_45_rsrc_type_n;
-};
-
-/* Source Resource Group Count IPA register save data struct */
-struct ipa_reg_save_src_rsrc_cnt_s {
-	struct ipa_hwio_def_ipa_src_rsrc_grp_0123_rsrc_type_cnt_n_s
-	  ipa_src_rsrc_grp_0123_rsrc_type_cnt_n;
-	struct ipa_hwio_def_ipa_src_rsrc_grp_4567_rsrc_type_cnt_n_s
-	  ipa_src_rsrc_grp_4567_rsrc_type_cnt_n;
-};
-
-/* Destination Resource Group Count IPA register save data struct */
-struct ipa_reg_save_dst_rsrc_cnt_s {
-	struct ipa_hwio_def_ipa_dst_rsrc_grp_0123_rsrc_type_cnt_n_s
-	  ipa_dst_rsrc_grp_0123_rsrc_type_cnt_n;
-	struct ipa_hwio_def_ipa_dst_rsrc_grp_4567_rsrc_type_cnt_n_s
-	  ipa_dst_rsrc_grp_4567_rsrc_type_cnt_n;
-};
-
-/* GSI General register save data struct */
-struct ipa_reg_save_gsi_gen_s {
-	struct gsi_hwio_def_gsi_cfg_s
-	  gsi_cfg;
-	struct gsi_hwio_def_gsi_ree_cfg_s
-	  gsi_ree_cfg;
-	struct ipa_hwio_def_ipa_gsi_top_gsi_inst_ram_n_s
-	  ipa_gsi_top_gsi_inst_ram_n;
-};
-
-/* GSI General EE register save data struct */
-struct ipa_reg_save_gsi_gen_ee_s {
-	struct gsi_hwio_def_gsi_manager_ee_qos_n_s
-	  gsi_manager_ee_qos_n;
-	struct gsi_hwio_def_ee_n_gsi_status_s
-	  ee_n_gsi_status;
-	struct gsi_hwio_def_ee_n_cntxt_type_irq_s
-	  ee_n_cntxt_type_irq;
-	struct gsi_hwio_def_ee_n_cntxt_type_irq_msk_s
-	  ee_n_cntxt_type_irq_msk;
-	struct gsi_hwio_def_ee_n_cntxt_src_gsi_ch_irq_s
-	  ee_n_cntxt_src_gsi_ch_irq;
-	struct gsi_hwio_def_ee_n_cntxt_src_ev_ch_irq_s
-	  ee_n_cntxt_src_ev_ch_irq;
-	struct gsi_hwio_def_ee_n_cntxt_src_gsi_ch_irq_msk_s
-	  ee_n_cntxt_src_gsi_ch_irq_msk;
-	struct gsi_hwio_def_ee_n_cntxt_src_ev_ch_irq_msk_s
-	  ee_n_cntxt_src_ev_ch_irq_msk;
-	struct gsi_hwio_def_ee_n_cntxt_src_ieob_irq_s
-	  ee_n_cntxt_src_ieob_irq;
-	struct gsi_hwio_def_ee_n_cntxt_src_ieob_irq_msk_s
-	  ee_n_cntxt_src_ieob_irq_msk;
-	struct gsi_hwio_def_ee_n_cntxt_gsi_irq_stts_s
-	  ee_n_cntxt_gsi_irq_stts;
-	struct gsi_hwio_def_ee_n_cntxt_glob_irq_stts_s
-	  ee_n_cntxt_glob_irq_stts;
-	struct gsi_hwio_def_ee_n_error_log_s
-	  ee_n_error_log;
-	struct gsi_hwio_def_ee_n_cntxt_scratch_0_s
-	  ee_n_cntxt_scratch_0;
-	struct gsi_hwio_def_ee_n_cntxt_scratch_1_s
-	  ee_n_cntxt_scratch_1;
-	struct gsi_hwio_def_ee_n_cntxt_intset_s
-	  ee_n_cntxt_intset;
-	struct gsi_hwio_def_ee_n_cntxt_msi_base_lsb_s
-	  ee_n_cntxt_msi_base_lsb;
-	struct gsi_hwio_def_ee_n_cntxt_msi_base_msb_s
-	  ee_n_cntxt_msi_base_msb;
-};
-
-/* GSI QSB debug register save data struct */
-struct ipa_reg_save_gsi_qsb_debug_s {
-	struct gsi_hwio_def_gsi_debug_qsb_log_last_misc_idn_s
-	  qsb_log_last_misc[GSI_HW_QSB_LOG_MISC_MAX];
-};
-
-static u32 ipa_reg_save_gsi_ch_test_bus_selector_array[] = {
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_ZEROS,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_MCS_0,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_MCS_1,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_MCS_2,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_MCS_3,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_MCS_4,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_DB_ENG,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_REE_0,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_REE_1,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_REE_2,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_REE_3,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_REE_4,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_REE_5,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_REE_6,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_REE_7,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_EVE_0,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_EVE_1,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_EVE_2,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_EVE_3,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_EVE_4,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_EVE_5,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_IE_0,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_IE_1,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_IC_0,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_IC_1,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_IC_2,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_IC_3,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_IC_4,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_MOQA_0,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_MOQA_1,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_MOQA_2,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_MOQA_3,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_TMR_0,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_TMR_1,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_TMR_2,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_TMR_3,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_RD_WR_0,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_RD_WR_1,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_RD_WR_2,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_RD_WR_3,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_CSR,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_SDMA_0,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_SDMA_1,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_IE_2,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_CSR_1,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_CSR_2,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_MCS_5,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_IC_5,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_CSR_3,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_TLV_0,
-	HWIO_GSI_DEBUG_TEST_BUS_SELECTOR_REE_8,
-};
-
-/*
- * GSI QSB debug bus register save data struct
- */
-struct ipa_reg_save_gsi_test_bus_s {
-	u32 test_bus_selector[
-		ARRAY_SIZE(ipa_reg_save_gsi_ch_test_bus_selector_array)];
-	struct
-	  gsi_hwio_def_gsi_test_bus_reg_s
-	  test_bus_reg[ARRAY_SIZE(ipa_reg_save_gsi_ch_test_bus_selector_array)];
-};
-
-/* GSI debug MCS registers save data struct */
-struct ipa_reg_save_gsi_mcs_regs_s {
-	struct
-	  gsi_hwio_def_gsi_debug_sw_rf_n_read_s
-		mcs_reg[HWIO_GSI_DEBUG_SW_RF_n_READ_MAXn + 1];
-};
-
-/* GSI debug counters save data struct */
-struct ipa_reg_save_gsi_debug_cnt_s {
-	struct
-	  gsi_hwio_def_gsi_debug_countern_s
-		cnt[HWIO_GSI_DEBUG_COUNTERn_MAXn + 1];
-};
-
-/* GSI IRAM pointers (IEP) save data struct */
-struct ipa_reg_save_gsi_iram_ptr_regs_s {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_ch_cmd_s
-	  ipa_gsi_top_gsi_iram_ptr_ch_cmd;
-	struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_ee_generic_cmd_s
-	  ipa_gsi_top_gsi_iram_ptr_ee_generic_cmd;
-	struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_ch_db_s
-	  ipa_gsi_top_gsi_iram_ptr_ch_db;
-	struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_ev_db_s
-	  ipa_gsi_top_gsi_iram_ptr_ev_db;
-	struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_new_re_s
-	  ipa_gsi_top_gsi_iram_ptr_new_re;
-	struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_ch_dis_comp_s
-	  ipa_gsi_top_gsi_iram_ptr_ch_dis_comp;
-	struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_ch_empty_s
-	  ipa_gsi_top_gsi_iram_ptr_ch_empty;
-	struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_event_gen_comp_s
-	  ipa_gsi_top_gsi_iram_ptr_event_gen_comp;
-	struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_timer_expired_s
-	  ipa_gsi_top_gsi_iram_ptr_timer_expired;
-	struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_write_eng_comp_s
-	  ipa_gsi_top_gsi_iram_ptr_write_eng_comp;
-	struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_read_eng_comp_s
-	  ipa_gsi_top_gsi_iram_ptr_read_eng_comp;
-	struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_uc_gp_int_s
-	  ipa_gsi_top_gsi_iram_ptr_uc_gp_int;
-	struct ipa_hwio_def_ipa_gsi_top_gsi_iram_ptr_int_mod_stopped_s
-	  ipa_gsi_top_gsi_iram_ptr_int_mod_stopped;
-};
-
-/* GSI SHRAM pointers save data struct */
-struct ipa_reg_save_gsi_shram_ptr_regs_s {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_shram_ptr_ch_cntxt_base_addr_s
-	  ipa_gsi_top_gsi_shram_ptr_ch_cntxt_base_addr;
-	struct ipa_hwio_def_ipa_gsi_top_gsi_shram_ptr_ev_cntxt_base_addr_s
-	  ipa_gsi_top_gsi_shram_ptr_ev_cntxt_base_addr;
-	struct ipa_hwio_def_ipa_gsi_top_gsi_shram_ptr_re_storage_base_addr_s
-	  ipa_gsi_top_gsi_shram_ptr_re_storage_base_addr;
-	struct ipa_hwio_def_ipa_gsi_top_gsi_shram_ptr_re_esc_buf_base_addr_s
-	  ipa_gsi_top_gsi_shram_ptr_re_esc_buf_base_addr;
-	struct ipa_hwio_def_ipa_gsi_top_gsi_shram_ptr_ee_scrach_base_addr_s
-	  ipa_gsi_top_gsi_shram_ptr_ee_scrach_base_addr;
-	struct ipa_hwio_def_ipa_gsi_top_gsi_shram_ptr_func_stack_base_addr_s
-	  ipa_gsi_top_gsi_shram_ptr_func_stack_base_addr;
-};
-
-/* GSI debug register save data struct */
-struct ipa_reg_save_gsi_debug_s {
-	struct ipa_hwio_def_ipa_gsi_top_gsi_debug_busy_reg_s
-	  ipa_gsi_top_gsi_debug_busy_reg;
-	struct ipa_hwio_def_ipa_gsi_top_gsi_debug_event_pending_s
-	  ipa_gsi_top_gsi_debug_event_pending;
-	struct ipa_hwio_def_ipa_gsi_top_gsi_debug_timer_pending_s
-	  ipa_gsi_top_gsi_debug_timer_pending;
-	struct ipa_hwio_def_ipa_gsi_top_gsi_debug_rd_wr_pending_s
-	  ipa_gsi_top_gsi_debug_rd_wr_pending;
-	struct ipa_hwio_def_ipa_gsi_top_gsi_debug_pc_from_sw_s
-	  ipa_gsi_top_gsi_debug_pc_from_sw;
-	struct ipa_hwio_def_ipa_gsi_top_gsi_debug_sw_stall_s
-	  ipa_gsi_top_gsi_debug_sw_stall;
-	struct ipa_hwio_def_ipa_gsi_top_gsi_debug_pc_for_debug_s
-	  ipa_gsi_top_gsi_debug_pc_for_debug;
-	struct ipa_hwio_def_ipa_gsi_top_gsi_debug_qsb_log_err_trns_id_s
-	  ipa_gsi_top_gsi_debug_qsb_log_err_trns_id;
-	struct ipa_reg_save_gsi_qsb_debug_s	gsi_qsb_debug;
-	struct ipa_reg_save_gsi_test_bus_s		gsi_test_bus;
-	struct ipa_reg_save_gsi_mcs_regs_s		gsi_mcs_regs;
-	struct ipa_reg_save_gsi_debug_cnt_s		gsi_cnt_regs;
-	struct ipa_reg_save_gsi_iram_ptr_regs_s		gsi_iram_ptrs;
-	struct ipa_reg_save_gsi_shram_ptr_regs_s	gsi_shram_ptrs;
-};
-
-/* GSI MCS channel scratch registers save data struct */
-struct ipa_reg_save_gsi_mcs_channel_scratch_regs_s {
-	struct gsi_hwio_def_gsi_shram_n_s
-	  scratch4;
-	struct gsi_hwio_def_gsi_shram_n_s
-	  scratch5;
-};
-
-/* GSI Channel Context register save data struct */
-struct ipa_reg_save_gsi_ch_cntxt_per_ep_s {
-	struct gsi_hwio_def_ee_n_gsi_ch_k_cntxt_0_s
-	  ee_n_gsi_ch_k_cntxt_0;
-	struct gsi_hwio_def_ee_n_gsi_ch_k_cntxt_1_s
-	  ee_n_gsi_ch_k_cntxt_1;
-	struct gsi_hwio_def_ee_n_gsi_ch_k_cntxt_2_s
-	  ee_n_gsi_ch_k_cntxt_2;
-	struct gsi_hwio_def_ee_n_gsi_ch_k_cntxt_3_s
-	  ee_n_gsi_ch_k_cntxt_3;
-	struct gsi_hwio_def_ee_n_gsi_ch_k_cntxt_4_s
-	  ee_n_gsi_ch_k_cntxt_4;
-	struct gsi_hwio_def_ee_n_gsi_ch_k_cntxt_5_s
-	  ee_n_gsi_ch_k_cntxt_5;
-	struct gsi_hwio_def_ee_n_gsi_ch_k_cntxt_6_s
-	  ee_n_gsi_ch_k_cntxt_6;
-	struct gsi_hwio_def_ee_n_gsi_ch_k_cntxt_7_s
-	  ee_n_gsi_ch_k_cntxt_7;
-	struct gsi_hwio_def_ee_n_gsi_ch_k_re_fetch_read_ptr_s
-	  ee_n_gsi_ch_k_re_fetch_read_ptr;
-	struct gsi_hwio_def_ee_n_gsi_ch_k_re_fetch_write_ptr_s
-	  ee_n_gsi_ch_k_re_fetch_write_ptr;
-	struct gsi_hwio_def_ee_n_gsi_ch_k_qos_s
-	  ee_n_gsi_ch_k_qos;
-	struct gsi_hwio_def_ee_n_gsi_ch_k_scratch_0_s
-	  ee_n_gsi_ch_k_scratch_0;
-	struct gsi_hwio_def_ee_n_gsi_ch_k_scratch_1_s
-	  ee_n_gsi_ch_k_scratch_1;
-	struct gsi_hwio_def_ee_n_gsi_ch_k_scratch_2_s
-	  ee_n_gsi_ch_k_scratch_2;
-	struct gsi_hwio_def_ee_n_gsi_ch_k_scratch_3_s
-	  ee_n_gsi_ch_k_scratch_3;
-	struct gsi_hwio_def_gsi_map_ee_n_ch_k_vp_table_s
-	  gsi_map_ee_n_ch_k_vp_table;
-	struct ipa_reg_save_gsi_mcs_channel_scratch_regs_s mcs_channel_scratch;
-};
-
-/* GSI Event Context register save data struct */
-struct ipa_reg_save_gsi_evt_cntxt_per_ep_s {
-	struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_0_s
-	  ee_n_ev_ch_k_cntxt_0;
-	struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_1_s
-	  ee_n_ev_ch_k_cntxt_1;
-	struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_2_s
-	  ee_n_ev_ch_k_cntxt_2;
-	struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_3_s
-	  ee_n_ev_ch_k_cntxt_3;
-	struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_4_s
-	  ee_n_ev_ch_k_cntxt_4;
-	struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_5_s
-	  ee_n_ev_ch_k_cntxt_5;
-	struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_6_s
-	  ee_n_ev_ch_k_cntxt_6;
-	struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_7_s
-	  ee_n_ev_ch_k_cntxt_7;
-	struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_8_s
-	  ee_n_ev_ch_k_cntxt_8;
-	struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_9_s
-	  ee_n_ev_ch_k_cntxt_9;
-	struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_10_s
-	  ee_n_ev_ch_k_cntxt_10;
-	struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_11_s
-	  ee_n_ev_ch_k_cntxt_11;
-	struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_12_s
-	  ee_n_ev_ch_k_cntxt_12;
-	struct gsi_hwio_def_ee_n_ev_ch_k_cntxt_13_s
-	  ee_n_ev_ch_k_cntxt_13;
-	struct gsi_hwio_def_ee_n_ev_ch_k_scratch_0_s
-	  ee_n_ev_ch_k_scratch_0;
-	struct gsi_hwio_def_ee_n_ev_ch_k_scratch_1_s
-	  ee_n_ev_ch_k_scratch_1;
-	struct gsi_hwio_def_gsi_debug_ee_n_ev_k_vp_table_s
-	  gsi_debug_ee_n_ev_k_vp_table;
-};
-
-/* GSI FIFO status register save data struct */
-struct ipa_reg_save_gsi_fifo_status_s {
-	union ipa_hwio_def_ipa_gsi_fifo_status_ctrl_u
-		gsi_fifo_status_ctrl;
-	union ipa_hwio_def_ipa_gsi_tlv_fifo_status_u
-		gsi_tlv_fifo_status;
-	union ipa_hwio_def_ipa_gsi_aos_fifo_status_u
-		gsi_aos_fifo_status;
-};
-
-/* GSI Channel Context register save top level data struct */
-struct ipa_reg_save_gsi_ch_cntxt_s {
-	struct ipa_reg_save_gsi_ch_cntxt_per_ep_s
-		a7[IPA_HW_REG_SAVE_GSI_NUM_CH_CNTXT_A7];
-	struct ipa_reg_save_gsi_ch_cntxt_per_ep_s
-		uc[IPA_HW_REG_SAVE_GSI_NUM_CH_CNTXT_UC];
-};
-
-/* GSI Event Context register save top level data struct */
-struct ipa_reg_save_gsi_evt_cntxt_s {
-	struct ipa_reg_save_gsi_evt_cntxt_per_ep_s
-		a7[IPA_HW_REG_SAVE_GSI_NUM_EVT_CNTXT_A7];
-	struct ipa_reg_save_gsi_evt_cntxt_per_ep_s
-		uc[IPA_HW_REG_SAVE_GSI_NUM_EVT_CNTXT_UC];
-};
-
-/* Top level IPA register save data struct */
-struct ipa_regs_save_hierarchy_s {
-	struct ipa_gen_regs_s
-		gen;
-	struct ipa_reg_save_gen_ee_s
-		gen_ee[IPA_HW_EE_MAX];
-	struct ipa_reg_save_hwp_s
-		hwp;
-	struct ipa_reg_save_dbg_s
-		dbg;
-	struct ipa_reg_save_ipa_testbus_s
-		*testbus;
-	struct ipa_reg_save_pipe_s
-		pipes[IPA_HW_PIPE_ID_MAX];
-	struct ipa_reg_save_src_rsrc_grp_s
-		src_rsrc_grp[IPA_HW_SRC_RSRP_TYPE_MAX];
-	struct ipa_reg_save_dst_rsrc_grp_s
-		dst_rsrc_grp[IPA_HW_DST_RSRP_TYPE_MAX];
-	struct ipa_reg_save_src_rsrc_cnt_s
-		src_rsrc_cnt[IPA_HW_SRC_RSRP_TYPE_MAX];
-	struct ipa_reg_save_dst_rsrc_cnt_s
-		dst_rsrc_cnt[IPA_HW_DST_RSRP_TYPE_MAX];
-	u32 *ipa_iu_ptr;
-	u32 *ipa_sram_ptr;
-	u32 *ipa_mbox_ptr;
-	u32 *ipa_hram_ptr;
-	u32 *ipa_seq_ptr;
-	u32 *ipa_gsi_ptr;
-};
-
-/* Top level GSI register save data struct */
-struct gsi_regs_save_hierarchy_s {
-	u32 fw_ver;
-	struct ipa_reg_save_gsi_gen_s		gen;
-	struct ipa_reg_save_gsi_gen_ee_s	gen_ee[IPA_REG_SAVE_GSI_NUM_EE];
-	struct ipa_reg_save_gsi_ch_cntxt_s	ch_cntxt;
-	struct ipa_reg_save_gsi_evt_cntxt_s	evt_cntxt;
-	struct ipa_reg_save_gsi_debug_s		debug;
-};
-
-/* Source resources for a resource group */
-struct ipa_reg_save_src_rsrc_cnts_s {
-	u8 pkt_cntxt;
-	u8 descriptor_list;
-	u8 data_descriptor_buffer;
-	u8 hps_dmars;
-	u8 reserved_acks;
-};
-
-/* Destination resources for a resource group */
-struct ipa_reg_save_dst_rsrc_cnts_s {
-	u8 reserved_sectors;
-	u8 dps_dmars;
-};
-
-/* Resource count structure for a resource group */
-struct ipa_reg_save_rsrc_cnts_per_grp_s {
-	/* Resource group number */
-	u8 resource_group;
-	/* Source resources for a resource group */
-	struct ipa_reg_save_src_rsrc_cnts_s src;
-	/* Destination resources for a resource group */
-	struct ipa_reg_save_dst_rsrc_cnts_s dst;
-};
-
-/* Top level resource count structure */
-struct ipa_reg_save_rsrc_cnts_s {
-	/* Resource count structure for PCIE group */
-	struct ipa_reg_save_rsrc_cnts_per_grp_s pcie;
-	/* Resource count structure for DDR group */
-	struct ipa_reg_save_rsrc_cnts_per_grp_s ddr;
-};
-
-/*
- * Top level IPA and GSI registers save data struct
- */
-struct regs_save_hierarchy_s {
-	struct ipa_regs_save_hierarchy_s
-		ipa;
-	struct gsi_regs_save_hierarchy_s
-		gsi;
-	bool
-		pkt_ctntx_active[IPA_HW_PKT_CTNTX_MAX];
-	union ipa_hwio_def_ipa_ctxh_ctrl_u
-		pkt_ctntxt_lock;
-	enum ipa_hw_pkt_cntxt_state_e
-		pkt_cntxt_state[IPA_HW_PKT_CTNTX_MAX];
-	struct ipa_pkt_ctntx_s
-		pkt_ctntx[IPA_HW_PKT_CTNTX_MAX];
-	struct ipa_reg_save_rsrc_cnts_s
-		rsrc_cnts;
-	struct ipa_reg_save_gsi_fifo_status_s
-		gsi_fifo_status[IPA_HW_PIPE_ID_MAX];
-};
-
-/*
- * The following section deals with handling IPA registers' memory
- * access relative to pre-defined memory protection schemes
- * (ie. "access control").
- *
- * In a nut shell, the intent of the data stuctures below is to allow
- * higher level register accessors to be unaware of what really is
- * going on at the lowest level (ie. real vs non-real access).  This
- * methodology is also designed to allow for platform specific "access
- * maps."
- */
-
-/*
- * Function for doing an actual read
- */
-static inline u32
-act_read(void __iomem *addr)
-{
-	u32 val = ioread32(addr);
-
-	return val;
-}
-
-/*
- * Function for doing an actual write
- */
-static inline void
-act_write(void __iomem *addr, u32 val)
-{
-	iowrite32(val, addr);
-}
-
-/*
- * Function that pretends to do a read
- */
-static inline u32
-nop_read(void __iomem *addr)
-{
-	return IPA_MEM_INIT_VAL;
-}
-
-/*
- * Function that pretends to do a write
- */
-static inline void
-nop_write(void __iomem *addr, u32 val)
-{
-}
-
-/*
- * The following are used to define struct reg_access_funcs_s below...
- */
-typedef u32 (*reg_read_func_t)(
-	void __iomem *addr);
-typedef void (*reg_write_func_t)(
-	void __iomem *addr,
-	u32 val);
-
-/*
- * The following in used to define io_matrix[] below...
- */
-struct reg_access_funcs_s {
-	reg_read_func_t  read;
-	reg_write_func_t write;
-};
-
-/*
- * The following will be used to appropriately index into the
- * read/write combos defined in io_matrix[] below...
- */
-#define AA_COMBO 0 /* actual read, actual write */
-#define AN_COMBO 1 /* actual read, no-op write  */
-#define NA_COMBO 2 /* no-op read,  actual write */
-#define NN_COMBO 3 /* no-op read,  no-op write  */
-
-/*
- * The following will be used to dictate registers' access methods
- * relative to the state of secure debug...whether it's enabled or
- * disabled.
- *
- * NOTE: The table below defines all access combinations.
- */
-static struct reg_access_funcs_s io_matrix[] = {
-	{ act_read, act_write }, /* the AA_COMBO */
-	{ act_read, nop_write }, /* the AN_COMBO */
-	{ nop_read, act_write }, /* the NA_COMBO */
-	{ nop_read, nop_write }, /* the NN_COMBO */
-};
-
-/*
- * The following will be used to define and drive IPA's register
- * access rules.
- */
-struct reg_mem_access_map_t {
-	u32 addr_range_begin;
-	u32 addr_range_end;
-	struct reg_access_funcs_s *access[2];
-};
-
-#endif /* #if !defined(_IPA_REG_DUMP_H_) */

ipa/ipa_v3/ipa.c

@@ -1,8284 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2012-2019, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/clk.h>
-#include <linux/compat.h>
-#include <linux/device.h>
-#include <linux/dmapool.h>
-#include <linux/fs.h>
-#include <linux/genalloc.h>
-#include <linux/init.h>
-#include <linux/kernel.h>
-#include <linux/mm.h>
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/of_platform.h>
-#include <linux/platform_device.h>
-#include <linux/rbtree.h>
-#include <linux/uaccess.h>
-#include <linux/interrupt.h>
-#include <linux/msm-bus.h>
-#include <linux/msm-bus-board.h>
-#include <linux/netdevice.h>
-#include <linux/delay.h>
-#include <linux/msm_gsi.h>
-#include <linux/time.h>
-#include <linux/hashtable.h>
-#include <linux/jhash.h>
-#include <linux/pci.h>
-#include <soc/qcom/subsystem_restart.h>
-#include <linux/soc/qcom/smem.h>
-#include <soc/qcom/scm.h>
-#include <asm/cacheflush.h>
-#include <linux/soc/qcom/smem_state.h>
-#include <linux/of_irq.h>
-
-#ifdef CONFIG_ARM64
-
-/* Outer caches unsupported on ARM64 platforms */
-#define outer_flush_range(x, y)
-#define __cpuc_flush_dcache_area __flush_dcache_area
-
-#endif
-
-#define IPA_SUBSYSTEM_NAME "ipa_fws"
-#define IPA_UC_SUBSYSTEM_NAME "ipa_uc"
-
-#include "ipa_i.h"
-#include "../ipa_rm_i.h"
-#include "ipahal/ipahal.h"
-#include "ipahal/ipahal_fltrt.h"
-
-#define CREATE_TRACE_POINTS
-#include "ipa_trace.h"
-#include "ipa_odl.h"
-
-/*
- * The following for adding code (ie. for EMULATION) not found on x86.
- */
-#if defined(CONFIG_IPA_EMULATION)
-# include "ipa_emulation_stubs.h"
-#endif
-
-#ifdef CONFIG_COMPAT
-/**
- * struct ipa3_ioc_nat_alloc_mem32 - nat table memory allocation
- * properties
- * @dev_name: input parameter, the name of table
- * @size: input parameter, size of table in bytes
- * @offset: output parameter, offset into page in case of system memory
- */
-struct ipa3_ioc_nat_alloc_mem32 {
-	char dev_name[IPA_RESOURCE_NAME_MAX];
-	compat_size_t size;
-	compat_off_t offset;
-};
-
-/**
- * struct ipa_ioc_nat_ipv6ct_table_alloc32 - table memory allocation
- * properties
- * @size: input parameter, size of table in bytes
- * @offset: output parameter, offset into page in case of system memory
- */
-struct ipa_ioc_nat_ipv6ct_table_alloc32 {
-	compat_size_t size;
-	compat_off_t offset;
-};
-#endif /* #ifdef CONFIG_COMPAT */
-
-#define IPA_TZ_UNLOCK_ATTRIBUTE 0x0C0311
-#define TZ_MEM_PROTECT_REGION_ID 0x10
-
-struct tz_smmu_ipa_protect_region_iovec_s {
-	u64 input_addr;
-	u64 output_addr;
-	u64 size;
-	u32 attr;
-} __packed;
-
-struct tz_smmu_ipa_protect_region_s {
-	phys_addr_t iovec_buf;
-	u32 size_bytes;
-} __packed;
-
-static void ipa3_start_tag_process(struct work_struct *work);
-static DECLARE_WORK(ipa3_tag_work, ipa3_start_tag_process);
-
-static void ipa3_transport_release_resource(struct work_struct *work);
-static DECLARE_DELAYED_WORK(ipa3_transport_release_resource_work,
-	ipa3_transport_release_resource);
-static void ipa_gsi_notify_cb(struct gsi_per_notify *notify);
-
-static int ipa3_attach_to_smmu(void);
-static int ipa3_alloc_pkt_init(void);
-
-static void ipa3_load_ipa_fw(struct work_struct *work);
-static DECLARE_WORK(ipa3_fw_loading_work, ipa3_load_ipa_fw);
-
-static void ipa_dec_clients_disable_clks_on_wq(struct work_struct *work);
-static DECLARE_WORK(ipa_dec_clients_disable_clks_on_wq_work,
-	ipa_dec_clients_disable_clks_on_wq);
-
-static int ipa3_ioctl_add_rt_rule_v2(unsigned long arg);
-static int ipa3_ioctl_add_rt_rule_ext_v2(unsigned long arg);
-static int ipa3_ioctl_add_rt_rule_after_v2(unsigned long arg);
-static int ipa3_ioctl_mdfy_rt_rule_v2(unsigned long arg);
-static int ipa3_ioctl_add_flt_rule_v2(unsigned long arg);
-static int ipa3_ioctl_add_flt_rule_after_v2(unsigned long arg);
-static int ipa3_ioctl_mdfy_flt_rule_v2(unsigned long arg);
-static int ipa3_ioctl_fnr_counter_alloc(unsigned long arg);
-static int ipa3_ioctl_fnr_counter_query(unsigned long arg);
-
-static struct ipa3_plat_drv_res ipa3_res = {0, };
-
-static struct clk *ipa3_clk;
-
-struct ipa3_context *ipa3_ctx;
-
-static struct {
-	bool present[IPA_SMMU_CB_MAX];
-	bool arm_smmu;
-	bool use_64_bit_dma_mask;
-	u32 ipa_base;
-	u32 ipa_size;
-} smmu_info;
-
-static char *active_clients_table_buf;
-
-int ipa3_active_clients_log_print_buffer(char *buf, int size)
-{
-	int i;
-	int nbytes;
-	int cnt = 0;
-	int start_idx;
-	int end_idx;
-	unsigned long flags;
-
-	spin_lock_irqsave(&ipa3_ctx->ipa3_active_clients_logging.lock, flags);
-	start_idx = (ipa3_ctx->ipa3_active_clients_logging.log_tail + 1) %
-			IPA3_ACTIVE_CLIENTS_LOG_BUFFER_SIZE_LINES;
-	end_idx = ipa3_ctx->ipa3_active_clients_logging.log_head;
-	for (i = start_idx; i != end_idx;
-		i = (i + 1) % IPA3_ACTIVE_CLIENTS_LOG_BUFFER_SIZE_LINES) {
-		nbytes = scnprintf(buf + cnt, size - cnt, "%s\n",
-				ipa3_ctx->ipa3_active_clients_logging
-				.log_buffer[i]);
-		cnt += nbytes;
-	}
-	spin_unlock_irqrestore(&ipa3_ctx->ipa3_active_clients_logging.lock,
-		flags);
-
-	return cnt;
-}
-
-int ipa3_active_clients_log_print_table(char *buf, int size)
-{
-	int i;
-	struct ipa3_active_client_htable_entry *iterator;
-	int cnt = 0;
-	unsigned long flags;
-
-	spin_lock_irqsave(&ipa3_ctx->ipa3_active_clients_logging.lock, flags);
-	cnt = scnprintf(buf, size, "\n---- Active Clients Table ----\n");
-	hash_for_each(ipa3_ctx->ipa3_active_clients_logging.htable, i,
-			iterator, list) {
-		switch (iterator->type) {
-		case IPA3_ACTIVE_CLIENT_LOG_TYPE_EP:
-			cnt += scnprintf(buf + cnt, size - cnt,
-					"%-40s %-3d ENDPOINT\n",
-					iterator->id_string, iterator->count);
-			break;
-		case IPA3_ACTIVE_CLIENT_LOG_TYPE_SIMPLE:
-			cnt += scnprintf(buf + cnt, size - cnt,
-					"%-40s %-3d SIMPLE\n",
-					iterator->id_string, iterator->count);
-			break;
-		case IPA3_ACTIVE_CLIENT_LOG_TYPE_RESOURCE:
-			cnt += scnprintf(buf + cnt, size - cnt,
-					"%-40s %-3d RESOURCE\n",
-					iterator->id_string, iterator->count);
-			break;
-		case IPA3_ACTIVE_CLIENT_LOG_TYPE_SPECIAL:
-			cnt += scnprintf(buf + cnt, size - cnt,
-					"%-40s %-3d SPECIAL\n",
-					iterator->id_string, iterator->count);
-			break;
-		default:
-			IPAERR("Trying to print illegal active_clients type");
-			break;
-		}
-	}
-	cnt += scnprintf(buf + cnt, size - cnt,
-			"\nTotal active clients count: %d\n",
-			atomic_read(&ipa3_ctx->ipa3_active_clients.cnt));
-
-	if (ipa3_is_mhip_offload_enabled())
-		cnt += ipa_mpm_panic_handler(buf + cnt, size - cnt);
-
-	spin_unlock_irqrestore(&ipa3_ctx->ipa3_active_clients_logging.lock,
-		flags);
-
-	return cnt;
-}
-
-static int ipa3_clean_modem_rule(void)
-{
-	struct ipa_install_fltr_rule_req_msg_v01 *req;
-	struct ipa_install_fltr_rule_req_ex_msg_v01 *req_ex;
-	int val = 0;
-
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v3_0) {
-		req = kzalloc(
-			sizeof(struct ipa_install_fltr_rule_req_msg_v01),
-			GFP_KERNEL);
-		if (!req) {
-			IPAERR("mem allocated failed!\n");
-			return -ENOMEM;
-		}
-		req->filter_spec_list_valid = false;
-		req->filter_spec_list_len = 0;
-		req->source_pipe_index_valid = 0;
-		val = ipa3_qmi_filter_request_send(req);
-		kfree(req);
-	} else {
-		req_ex = kzalloc(
-			sizeof(struct ipa_install_fltr_rule_req_ex_msg_v01),
-			GFP_KERNEL);
-		if (!req_ex) {
-			IPAERR("mem allocated failed!\n");
-			return -ENOMEM;
-		}
-		req_ex->filter_spec_ex_list_valid = false;
-		req_ex->filter_spec_ex_list_len = 0;
-		req_ex->source_pipe_index_valid = 0;
-		val = ipa3_qmi_filter_request_ex_send(req_ex);
-		kfree(req_ex);
-	}
-
-	return val;
-}
-
-static int ipa3_active_clients_panic_notifier(struct notifier_block *this,
-		unsigned long event, void *ptr)
-{
-	ipa3_active_clients_log_print_table(active_clients_table_buf,
-			IPA3_ACTIVE_CLIENTS_TABLE_BUF_SIZE);
-	IPAERR("%s\n", active_clients_table_buf);
-
-	return NOTIFY_DONE;
-}
-
-static struct notifier_block ipa3_active_clients_panic_blk = {
-	.notifier_call  = ipa3_active_clients_panic_notifier,
-};
-
-#ifdef CONFIG_IPA_DEBUG
-static int ipa3_active_clients_log_insert(const char *string)
-{
-	int head;
-	int tail;
-
-	if (!ipa3_ctx->ipa3_active_clients_logging.log_rdy)
-		return -EPERM;
-
-	head = ipa3_ctx->ipa3_active_clients_logging.log_head;
-	tail = ipa3_ctx->ipa3_active_clients_logging.log_tail;
-
-	memset(ipa3_ctx->ipa3_active_clients_logging.log_buffer[head], '_',
-			IPA3_ACTIVE_CLIENTS_LOG_LINE_LEN);
-	strlcpy(ipa3_ctx->ipa3_active_clients_logging.log_buffer[head], string,
-			(size_t)IPA3_ACTIVE_CLIENTS_LOG_LINE_LEN);
-	head = (head + 1) % IPA3_ACTIVE_CLIENTS_LOG_BUFFER_SIZE_LINES;
-	if (tail == head)
-		tail = (tail + 1) % IPA3_ACTIVE_CLIENTS_LOG_BUFFER_SIZE_LINES;
-
-	ipa3_ctx->ipa3_active_clients_logging.log_tail = tail;
-	ipa3_ctx->ipa3_active_clients_logging.log_head = head;
-
-	return 0;
-}
-#endif
-
-static int ipa3_active_clients_log_init(void)
-{
-	int i;
-
-	spin_lock_init(&ipa3_ctx->ipa3_active_clients_logging.lock);
-	ipa3_ctx->ipa3_active_clients_logging.log_buffer[0] = kcalloc(
-			IPA3_ACTIVE_CLIENTS_LOG_BUFFER_SIZE_LINES,
-			sizeof(char[IPA3_ACTIVE_CLIENTS_LOG_LINE_LEN]),
-			GFP_KERNEL);
-	active_clients_table_buf = kzalloc(sizeof(
-			char[IPA3_ACTIVE_CLIENTS_TABLE_BUF_SIZE]), GFP_KERNEL);
-	if (ipa3_ctx->ipa3_active_clients_logging.log_buffer == NULL) {
-		pr_err("Active Clients Logging memory allocation failed\n");
-		goto bail;
-	}
-	for (i = 0; i < IPA3_ACTIVE_CLIENTS_LOG_BUFFER_SIZE_LINES; i++) {
-		ipa3_ctx->ipa3_active_clients_logging.log_buffer[i] =
-			ipa3_ctx->ipa3_active_clients_logging.log_buffer[0] +
-			(IPA3_ACTIVE_CLIENTS_LOG_LINE_LEN * i);
-	}
-	ipa3_ctx->ipa3_active_clients_logging.log_head = 0;
-	ipa3_ctx->ipa3_active_clients_logging.log_tail =
-			IPA3_ACTIVE_CLIENTS_LOG_BUFFER_SIZE_LINES - 1;
-	hash_init(ipa3_ctx->ipa3_active_clients_logging.htable);
-	atomic_notifier_chain_register(&panic_notifier_list,
-			&ipa3_active_clients_panic_blk);
-	ipa3_ctx->ipa3_active_clients_logging.log_rdy = true;
-
-	return 0;
-
-bail:
-	return -ENOMEM;
-}
-
-void ipa3_active_clients_log_clear(void)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&ipa3_ctx->ipa3_active_clients_logging.lock, flags);
-	ipa3_ctx->ipa3_active_clients_logging.log_head = 0;
-	ipa3_ctx->ipa3_active_clients_logging.log_tail =
-			IPA3_ACTIVE_CLIENTS_LOG_BUFFER_SIZE_LINES - 1;
-	spin_unlock_irqrestore(&ipa3_ctx->ipa3_active_clients_logging.lock,
-		flags);
-}
-
-static void ipa3_active_clients_log_destroy(void)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&ipa3_ctx->ipa3_active_clients_logging.lock, flags);
-	ipa3_ctx->ipa3_active_clients_logging.log_rdy = false;
-	kfree(active_clients_table_buf);
-	active_clients_table_buf = NULL;
-	kfree(ipa3_ctx->ipa3_active_clients_logging.log_buffer[0]);
-	ipa3_ctx->ipa3_active_clients_logging.log_head = 0;
-	ipa3_ctx->ipa3_active_clients_logging.log_tail =
-			IPA3_ACTIVE_CLIENTS_LOG_BUFFER_SIZE_LINES - 1;
-	spin_unlock_irqrestore(&ipa3_ctx->ipa3_active_clients_logging.lock,
-		flags);
-}
-
-static struct ipa_smmu_cb_ctx smmu_cb[IPA_SMMU_CB_MAX];
-
-struct iommu_domain *ipa3_get_smmu_domain_by_type(enum ipa_smmu_cb_type cb_type)
-{
-	if (VALID_IPA_SMMU_CB_TYPE(cb_type) && smmu_cb[cb_type].valid)
-		return smmu_cb[cb_type].iommu_domain;
-
-	IPAERR("cb_type(%d) not valid\n", cb_type);
-
-	return NULL;
-}
-
-struct iommu_domain *ipa3_get_smmu_domain(void)
-{
-	return ipa3_get_smmu_domain_by_type(IPA_SMMU_CB_AP);
-}
-
-struct iommu_domain *ipa3_get_uc_smmu_domain(void)
-{
-	return ipa3_get_smmu_domain_by_type(IPA_SMMU_CB_UC);
-}
-
-struct iommu_domain *ipa3_get_wlan_smmu_domain(void)
-{
-	return ipa3_get_smmu_domain_by_type(IPA_SMMU_CB_WLAN);
-}
-
-struct iommu_domain *ipa3_get_11ad_smmu_domain(void)
-{
-	return ipa3_get_smmu_domain_by_type(IPA_SMMU_CB_11AD);
-}
-
-struct device *ipa3_get_dma_dev(void)
-{
-	return ipa3_ctx->pdev;
-}
-
-/**
- * ipa3_get_smmu_ctx()- Return smmu context for the given cb_type
- *
- * Return value: pointer to smmu context address
- */
-struct ipa_smmu_cb_ctx *ipa3_get_smmu_ctx(enum ipa_smmu_cb_type cb_type)
-{
-	return &smmu_cb[cb_type];
-}
-
-static int ipa3_open(struct inode *inode, struct file *filp)
-{
-	IPADBG_LOW("ENTER\n");
-	filp->private_data = ipa3_ctx;
-
-	return 0;
-}
-
-static void ipa3_wan_msg_free_cb(void *buff, u32 len, u32 type)
-{
-	if (!buff) {
-		IPAERR("Null buffer\n");
-		return;
-	}
-
-	if (type != WAN_UPSTREAM_ROUTE_ADD &&
-	    type != WAN_UPSTREAM_ROUTE_DEL &&
-	    type != WAN_EMBMS_CONNECT) {
-		IPAERR("Wrong type given. buff %pK type %d\n", buff, type);
-		return;
-	}
-
-	kfree(buff);
-}
-
-static int ipa3_send_wan_msg(unsigned long usr_param,
-	uint8_t msg_type, bool is_cache)
-{
-	int retval;
-	struct ipa_wan_msg *wan_msg;
-	struct ipa_msg_meta msg_meta;
-	struct ipa_wan_msg cache_wan_msg;
-
-	wan_msg = kzalloc(sizeof(*wan_msg), GFP_KERNEL);
-	if (!wan_msg)
-		return -ENOMEM;
-
-	if (copy_from_user(wan_msg, (const void __user *)usr_param,
-		sizeof(struct ipa_wan_msg))) {
-		kfree(wan_msg);
-		return -EFAULT;
-	}
-
-	memcpy(&cache_wan_msg, wan_msg, sizeof(cache_wan_msg));
-
-	memset(&msg_meta, 0, sizeof(struct ipa_msg_meta));
-	msg_meta.msg_type = msg_type;
-	msg_meta.msg_len = sizeof(struct ipa_wan_msg);
-	retval = ipa3_send_msg(&msg_meta, wan_msg, ipa3_wan_msg_free_cb);
-	if (retval) {
-		IPAERR_RL("ipa3_send_msg failed: %d\n", retval);
-		kfree(wan_msg);
-		return retval;
-	}
-
-	if (is_cache) {
-		mutex_lock(&ipa3_ctx->ipa_cne_evt_lock);
-
-		/* cache the cne event */
-		memcpy(&ipa3_ctx->ipa_cne_evt_req_cache[
-			ipa3_ctx->num_ipa_cne_evt_req].wan_msg,
-			&cache_wan_msg,
-			sizeof(cache_wan_msg));
-
-		memcpy(&ipa3_ctx->ipa_cne_evt_req_cache[
-			ipa3_ctx->num_ipa_cne_evt_req].msg_meta,
-			&msg_meta,
-			sizeof(struct ipa_msg_meta));
-
-		ipa3_ctx->num_ipa_cne_evt_req++;
-		ipa3_ctx->num_ipa_cne_evt_req %= IPA_MAX_NUM_REQ_CACHE;
-		mutex_unlock(&ipa3_ctx->ipa_cne_evt_lock);
-	}
-
-	return 0;
-}
-
-static void ipa3_vlan_l2tp_msg_free_cb(void *buff, u32 len, u32 type)
-{
-	if (!buff) {
-		IPAERR("Null buffer\n");
-		return;
-	}
-
-	switch (type) {
-	case ADD_VLAN_IFACE:
-	case DEL_VLAN_IFACE:
-	case ADD_L2TP_VLAN_MAPPING:
-	case DEL_L2TP_VLAN_MAPPING:
-	case ADD_BRIDGE_VLAN_MAPPING:
-	case DEL_BRIDGE_VLAN_MAPPING:
-		break;
-	default:
-		IPAERR("Wrong type given. buff %pK type %d\n", buff, type);
-		return;
-	}
-
-	kfree(buff);
-}
-
-static int ipa3_send_vlan_l2tp_msg(unsigned long usr_param, uint8_t msg_type)
-{
-	int retval;
-	struct ipa_ioc_vlan_iface_info *vlan_info;
-	struct ipa_ioc_l2tp_vlan_mapping_info *mapping_info;
-	struct ipa_ioc_bridge_vlan_mapping_info *bridge_vlan_info;
-	struct ipa_msg_meta msg_meta;
-	void *buff;
-
-	IPADBG("type %d\n", msg_type);
-
-	memset(&msg_meta, 0, sizeof(msg_meta));
-	msg_meta.msg_type = msg_type;
-
-	if ((msg_type == ADD_VLAN_IFACE) ||
-		(msg_type == DEL_VLAN_IFACE)) {
-		vlan_info = kzalloc(sizeof(struct ipa_ioc_vlan_iface_info),
-			GFP_KERNEL);
-		if (!vlan_info)
-			return -ENOMEM;
-
-		if (copy_from_user((u8 *)vlan_info, (void __user *)usr_param,
-			sizeof(struct ipa_ioc_vlan_iface_info))) {
-			kfree(vlan_info);
-			return -EFAULT;
-		}
-
-		msg_meta.msg_len = sizeof(struct ipa_ioc_vlan_iface_info);
-		buff = vlan_info;
-	} else if ((msg_type == ADD_L2TP_VLAN_MAPPING) ||
-		(msg_type == DEL_L2TP_VLAN_MAPPING)) {
-		mapping_info = kzalloc(sizeof(struct
-			ipa_ioc_l2tp_vlan_mapping_info), GFP_KERNEL);
-		if (!mapping_info)
-			return -ENOMEM;
-
-		if (copy_from_user((u8 *)mapping_info,
-			(void __user *)usr_param,
-			sizeof(struct ipa_ioc_l2tp_vlan_mapping_info))) {
-			kfree(mapping_info);
-			return -EFAULT;
-		}
-
-		msg_meta.msg_len = sizeof(struct
-			ipa_ioc_l2tp_vlan_mapping_info);
-		buff = mapping_info;
-	} else if ((msg_type == ADD_BRIDGE_VLAN_MAPPING) ||
-		(msg_type == DEL_BRIDGE_VLAN_MAPPING)) {
-		bridge_vlan_info = kzalloc(
-			sizeof(struct ipa_ioc_bridge_vlan_mapping_info),
-			GFP_KERNEL);
-		if (!bridge_vlan_info)
-			return -ENOMEM;
-
-		if (copy_from_user((u8 *)bridge_vlan_info,
-			(void __user *)usr_param,
-			sizeof(struct ipa_ioc_bridge_vlan_mapping_info))) {
-			kfree(bridge_vlan_info);
-			IPAERR("copy from user failed\n");
-			return -EFAULT;
-		}
-
-		msg_meta.msg_len = sizeof(struct
-			ipa_ioc_bridge_vlan_mapping_info);
-		buff = bridge_vlan_info;
-	} else {
-		IPAERR("Unexpected event\n");
-		return -EFAULT;
-	}
-
-	retval = ipa3_send_msg(&msg_meta, buff,
-		ipa3_vlan_l2tp_msg_free_cb);
-	if (retval) {
-		IPAERR("ipa3_send_msg failed: %d, msg_type %d\n",
-			retval,
-			msg_type);
-		kfree(buff);
-		return retval;
-	}
-	IPADBG("exit\n");
-
-	return 0;
-}
-
-static void ipa3_gsb_msg_free_cb(void *buff, u32 len, u32 type)
-{
-	if (!buff) {
-		IPAERR("Null buffer\n");
-		return;
-	}
-
-	switch (type) {
-	case IPA_GSB_CONNECT:
-	case IPA_GSB_DISCONNECT:
-		break;
-	default:
-		IPAERR("Wrong type given. buff %pK type %d\n", buff, type);
-		return;
-	}
-
-	kfree(buff);
-}
-
-static int ipa3_send_gsb_msg(unsigned long usr_param, uint8_t msg_type)
-{
-	int retval;
-	struct ipa_ioc_gsb_info *gsb_info;
-	struct ipa_msg_meta msg_meta;
-	void *buff;
-
-	IPADBG("type %d\n", msg_type);
-
-	memset(&msg_meta, 0, sizeof(msg_meta));
-	msg_meta.msg_type = msg_type;
-
-	if ((msg_type == IPA_GSB_CONNECT) ||
-		(msg_type == IPA_GSB_DISCONNECT)) {
-		gsb_info = kzalloc(sizeof(struct ipa_ioc_gsb_info),
-			GFP_KERNEL);
-		if (!gsb_info) {
-			IPAERR("no memory\n");
-			return -ENOMEM;
-		}
-
-		if (copy_from_user((u8 *)gsb_info, (void __user *)usr_param,
-			sizeof(struct ipa_ioc_gsb_info))) {
-			kfree(gsb_info);
-			return -EFAULT;
-		}
-
-		msg_meta.msg_len = sizeof(struct ipa_ioc_gsb_info);
-		buff = gsb_info;
-	} else {
-		IPAERR("Unexpected event\n");
-		return -EFAULT;
-	}
-
-	retval = ipa3_send_msg(&msg_meta, buff,
-		ipa3_gsb_msg_free_cb);
-	if (retval) {
-		IPAERR("ipa3_send_msg failed: %d, msg_type %d\n",
-			retval,
-			msg_type);
-		kfree(buff);
-		return retval;
-	}
-	IPADBG("exit\n");
-
-	return 0;
-}
-
-static int ipa3_ioctl_add_rt_rule_v2(unsigned long arg)
-{
-	int retval = 0;
-	int i;
-	u8 header[128] = { 0 };
-	int pre_entry;
-	u32 usr_pyld_sz;
-	u32 pyld_sz;
-	u64 uptr = 0;
-	u8 *param = NULL;
-	u8 *kptr = NULL;
-
-	if (copy_from_user(header, (const void __user *)arg,
-		sizeof(struct ipa_ioc_add_rt_rule_v2))) {
-		IPAERR_RL("copy_from_user fails\n");
-		retval = -EFAULT;
-		goto free_param_kptr;
-	}
-	pre_entry =
-		((struct ipa_ioc_add_rt_rule_v2 *)header)->num_rules;
-	if (unlikely(((struct ipa_ioc_add_rt_rule_v2 *)
-		header)->rule_add_size >
-		sizeof(struct ipa_rt_rule_add_i))) {
-		IPAERR_RL("unexpected rule_add_size %d\n",
-		((struct ipa_ioc_add_rt_rule_v2 *)
-		header)->rule_add_size);
-		retval = -EPERM;
-		goto free_param_kptr;
-	}
-	/* user payload size */
-	usr_pyld_sz = ((struct ipa_ioc_add_rt_rule_v2 *)
-		header)->rule_add_size * pre_entry;
-	/* actual payload structure size in kernel */
-	pyld_sz = sizeof(struct ipa_rt_rule_add_i) * pre_entry;
-	uptr = ((struct ipa_ioc_add_rt_rule_v2 *)
-		header)->rules;
-	if (unlikely(!uptr)) {
-		IPAERR_RL("unexpected NULL rules\n");
-		retval = -EPERM;
-		goto free_param_kptr;
-	}
-	/* alloc param with same payload size as user payload */
-	param = memdup_user((const void __user *)uptr,
-		usr_pyld_sz);
-	if (IS_ERR(param)) {
-		retval = -EFAULT;
-		goto free_param_kptr;
-	}
-	/* alloc kernel pointer with actual payload size */
-	kptr = kzalloc(pyld_sz, GFP_KERNEL);
-	if (!kptr) {
-		retval = -ENOMEM;
-		goto free_param_kptr;
-	}
-	for (i = 0; i < pre_entry; i++)
-		memcpy(kptr + i * sizeof(struct ipa_rt_rule_add_i),
-			(void *)param + i *
-			((struct ipa_ioc_add_rt_rule_v2 *)
-			header)->rule_add_size,
-			((struct ipa_ioc_add_rt_rule_v2 *)
-			header)->rule_add_size);
-	/* modify the rule pointer to the kernel pointer */
-	((struct ipa_ioc_add_rt_rule_v2 *)header)->rules =
-		(u64)kptr;
-	if (ipa3_add_rt_rule_usr_v2(
-		(struct ipa_ioc_add_rt_rule_v2 *)header, true)) {
-		IPAERR_RL("ipa3_add_rt_rule_usr_v2 fails\n");
-		retval = -EPERM;
-		goto free_param_kptr;
-	}
-	for (i = 0; i < pre_entry; i++)
-		memcpy((void *)param + i *
-			((struct ipa_ioc_add_rt_rule_v2 *)
-			header)->rule_add_size,
-			kptr + i * sizeof(struct ipa_rt_rule_add_i),
-			((struct ipa_ioc_add_rt_rule_v2 *)
-			header)->rule_add_size);
-	if (copy_to_user((void __user *)uptr, param,
-		usr_pyld_sz)) {
-		IPAERR_RL("copy_to_user fails\n");
-		retval = -EFAULT;
-		goto free_param_kptr;
-	}
-
-free_param_kptr:
-	if (!IS_ERR(param))
-		kfree(param);
-	kfree(kptr);
-
-	return retval;
-}
-
-static int ipa3_ioctl_add_rt_rule_ext_v2(unsigned long arg)
-{
-	int retval = 0;
-	int i;
-	u8 header[128] = { 0 };
-	int pre_entry;
-	u32 usr_pyld_sz;
-	u32 pyld_sz;
-	u64 uptr = 0;
-	u8 *param = NULL;
-	u8 *kptr = NULL;
-
-	if (copy_from_user(header,
-			(const void __user *)arg,
-			sizeof(struct ipa_ioc_add_rt_rule_ext_v2))) {
-		IPAERR_RL("copy_from_user fails\n");
-		retval = -EFAULT;
-		goto free_param_kptr;
-	}
-	pre_entry =
-		((struct ipa_ioc_add_rt_rule_ext_v2 *)
-		header)->num_rules;
-	if (unlikely(((struct ipa_ioc_add_rt_rule_ext_v2 *)
-		header)->rule_add_ext_size >
-		sizeof(struct ipa_rt_rule_add_ext_i))) {
-		IPAERR_RL("unexpected rule_add_size %d\n",
-		((struct ipa_ioc_add_rt_rule_ext_v2 *)
-		header)->rule_add_ext_size);
-		retval = -EPERM;
-		goto free_param_kptr;
-	}
-	/* user payload size */
-	usr_pyld_sz = ((struct ipa_ioc_add_rt_rule_ext_v2 *)
-		header)->rule_add_ext_size * pre_entry;
-	/* actual payload structure size in kernel */
-	pyld_sz = sizeof(struct ipa_rt_rule_add_ext_i)
-		* pre_entry;
-	uptr = ((struct ipa_ioc_add_rt_rule_ext_v2 *)
-		header)->rules;
-	if (unlikely(!uptr)) {
-		IPAERR_RL("unexpected NULL rules\n");
-		retval = -EPERM;
-		goto free_param_kptr;
-	}
-	/* alloc param with same payload size as user payload */
-	param = memdup_user((const void __user *)uptr,
-		usr_pyld_sz);
-	if (IS_ERR(param)) {
-		retval = -EFAULT;
-		goto free_param_kptr;
-	}
-	/* alloc kernel pointer with actual payload size */
-	kptr = kzalloc(pyld_sz, GFP_KERNEL);
-	if (!kptr) {
-		retval = -ENOMEM;
-		goto free_param_kptr;
-	}
-	for (i = 0; i < pre_entry; i++)
-		memcpy(kptr + i *
-			sizeof(struct ipa_rt_rule_add_ext_i),
-			(void *)param + i *
-			((struct ipa_ioc_add_rt_rule_ext_v2 *)
-			header)->rule_add_ext_size,
-			((struct ipa_ioc_add_rt_rule_ext_v2 *)
-			header)->rule_add_ext_size);
-	/* modify the rule pointer to the kernel pointer */
-	((struct ipa_ioc_add_rt_rule_ext_v2 *)header)->rules =
-		(u64)kptr;
-	if (ipa3_add_rt_rule_ext_v2(
-		(struct ipa_ioc_add_rt_rule_ext_v2 *)header)) {
-		IPAERR_RL("ipa3_add_rt_rule_ext_v2 fails\n");
-		retval = -EPERM;
-		goto free_param_kptr;
-	}
-	for (i = 0; i < pre_entry; i++)
-		memcpy((void *)param + i *
-			((struct ipa_ioc_add_rt_rule_ext_v2 *)
-			header)->rule_add_ext_size,
-			kptr + i *
-			sizeof(struct ipa_rt_rule_add_ext_i),
-			((struct ipa_ioc_add_rt_rule_ext_v2 *)
-			header)->rule_add_ext_size);
-	if (copy_to_user((void __user *)uptr, param,
-		usr_pyld_sz)) {
-		IPAERR_RL("copy_to_user fails\n");
-		retval = -EFAULT;
-		goto free_param_kptr;
-	}
-
-free_param_kptr:
-	if (!IS_ERR(param))
-		kfree(param);
-	kfree(kptr);
-
-	return retval;
-}
-
-static int ipa3_ioctl_add_rt_rule_after_v2(unsigned long arg)
-{
-	int retval = 0;
-	int i;
-	u8 header[128] = { 0 };
-	int pre_entry;
-	u32 usr_pyld_sz;
-	u32 pyld_sz;
-	u64 uptr = 0;
-	u8 *param = NULL;
-	u8 *kptr = NULL;
-
-	if (copy_from_user(header, (const void __user *)arg,
-		sizeof(struct ipa_ioc_add_rt_rule_after_v2))) {
-		IPAERR_RL("copy_from_user fails\n");
-		retval = -EFAULT;
-		goto free_param_kptr;
-	}
-	pre_entry =
-		((struct ipa_ioc_add_rt_rule_after_v2 *)
-		header)->num_rules;
-	if (unlikely(((struct ipa_ioc_add_rt_rule_after_v2 *)
-		header)->rule_add_size >
-		sizeof(struct ipa_rt_rule_add_i))) {
-		IPAERR_RL("unexpected rule_add_size %d\n",
-		((struct ipa_ioc_add_rt_rule_after_v2 *)
-		header)->rule_add_size);
-		retval = -EPERM;
-		goto free_param_kptr;
-	}
-	/* user payload size */
-	usr_pyld_sz = ((struct ipa_ioc_add_rt_rule_after_v2 *)
-		header)->rule_add_size * pre_entry;
-	/* actual payload structure size in kernel */
-	pyld_sz = sizeof(struct ipa_rt_rule_add_i)
-		* pre_entry;
-	uptr = ((struct ipa_ioc_add_rt_rule_after_v2 *)
-		header)->rules;
-	if (unlikely(!uptr)) {
-		IPAERR_RL("unexpected NULL rules\n");
-		retval = -EPERM;
-		goto free_param_kptr;
-	}
-	/* alloc param with same payload size as user payload */
-	param = memdup_user((const void __user *)uptr,
-		usr_pyld_sz);
-	if (IS_ERR(param)) {
-		retval = -EFAULT;
-		goto free_param_kptr;
-	}
-	/* alloc kernel pointer with actual payload size */
-	kptr = kzalloc(pyld_sz, GFP_KERNEL);
-	if (!kptr) {
-		retval = -ENOMEM;
-		goto free_param_kptr;
-	}
-	for (i = 0; i < pre_entry; i++)
-		memcpy(kptr + i * sizeof(struct ipa_rt_rule_add_i),
-			(void *)param + i *
-			((struct ipa_ioc_add_rt_rule_after_v2 *)
-			header)->rule_add_size,
-			((struct ipa_ioc_add_rt_rule_after_v2 *)
-			header)->rule_add_size);
-	/* modify the rule pointer to the kernel pointer */
-	((struct ipa_ioc_add_rt_rule_after_v2 *)header)->rules =
-		(u64)kptr;
-	if (ipa3_add_rt_rule_after_v2(
-		(struct ipa_ioc_add_rt_rule_after_v2 *)header)) {
-		IPAERR_RL("ipa3_add_rt_rule_after_v2 fails\n");
-		retval = -EPERM;
-		goto free_param_kptr;
-	}
-	for (i = 0; i < pre_entry; i++)
-		memcpy((void *)param + i *
-			((struct ipa_ioc_add_rt_rule_after_v2 *)
-			header)->rule_add_size,
-			kptr + i * sizeof(struct ipa_rt_rule_add_i),
-			((struct ipa_ioc_add_rt_rule_after_v2 *)
-			header)->rule_add_size);
-	if (copy_to_user((void __user *)uptr, param,
-		usr_pyld_sz)) {
-		IPAERR_RL("copy_to_user fails\n");
-		retval = -EFAULT;
-		goto free_param_kptr;
-	}
-
-free_param_kptr:
-	if (!IS_ERR(param))
-		kfree(param);
-	kfree(kptr);
-
-	return retval;
-}
-
-static int ipa3_ioctl_mdfy_rt_rule_v2(unsigned long arg)
-{
-	int retval = 0;
-	int i;
-	u8 header[128] = { 0 };
-	int pre_entry;
-	u32 usr_pyld_sz;
-	u32 pyld_sz;
-	u64 uptr = 0;
-	u8 *param = NULL;
-	u8 *kptr = NULL;
-
-	if (copy_from_user(header, (const void __user *)arg,
-		sizeof(struct ipa_ioc_mdfy_rt_rule_v2))) {
-		IPAERR_RL("copy_from_user fails\n");
-		retval = -EFAULT;
-		goto free_param_kptr;
-	}
-	pre_entry =
-		((struct ipa_ioc_mdfy_rt_rule_v2 *)
-		header)->num_rules;
-	if (unlikely(((struct ipa_ioc_mdfy_rt_rule_v2 *)
-		header)->rule_mdfy_size >
-		sizeof(struct ipa_rt_rule_mdfy_i))) {
-		IPAERR_RL("unexpected rule_add_size %d\n",
-		((struct ipa_ioc_mdfy_rt_rule_v2 *)
-		header)->rule_mdfy_size);
-		retval = -EPERM;
-		goto free_param_kptr;
-	}
-	/* user payload size */
-	usr_pyld_sz = ((struct ipa_ioc_mdfy_rt_rule_v2 *)
-		header)->rule_mdfy_size * pre_entry;
-	/* actual payload structure size in kernel */
-	pyld_sz = sizeof(struct ipa_rt_rule_mdfy_i)
-		* pre_entry;
-	uptr = ((struct ipa_ioc_mdfy_rt_rule_v2 *)
-		header)->rules;
-	if (unlikely(!uptr)) {
-		IPAERR_RL("unexpected NULL rules\n");
-		retval = -EPERM;
-		goto free_param_kptr;
-	}
-	/* alloc param with same payload size as user payload */
-	param = memdup_user((const void __user *)uptr,
-		usr_pyld_sz);
-	if (IS_ERR(param)) {
-		retval = -EFAULT;
-		goto free_param_kptr;
-	}
-	/* alloc kernel pointer with actual payload size */
-	kptr = kzalloc(pyld_sz, GFP_KERNEL);
-	if (!kptr) {
-		retval = -ENOMEM;
-		goto free_param_kptr;
-	}
-	for (i = 0; i < pre_entry; i++)
-		memcpy(kptr + i * sizeof(struct ipa_rt_rule_mdfy_i),
-			(void *)param + i *
-			((struct ipa_ioc_mdfy_rt_rule_v2 *)
-			header)->rule_mdfy_size,
-			((struct ipa_ioc_mdfy_rt_rule_v2 *)
-			header)->rule_mdfy_size);
-	/* modify the rule pointer to the kernel pointer */
-	((struct ipa_ioc_mdfy_rt_rule_v2 *)header)->rules =
-		(u64)kptr;
-	if (ipa3_mdfy_rt_rule_v2((struct ipa_ioc_mdfy_rt_rule_v2 *)
-		header)) {
-		IPAERR_RL("ipa3_mdfy_rt_rule_v2 fails\n");
-		retval = -EPERM;
-		goto free_param_kptr;
-	}
-	for (i = 0; i < pre_entry; i++)
-		memcpy((void *)param + i *
-			((struct ipa_ioc_mdfy_rt_rule_v2 *)
-			header)->rule_mdfy_size,
-			kptr + i * sizeof(struct ipa_rt_rule_mdfy_i),
-			((struct ipa_ioc_mdfy_rt_rule_v2 *)
-			header)->rule_mdfy_size);
-	if (copy_to_user((void __user *)uptr, param,
-		usr_pyld_sz)) {
-		IPAERR_RL("copy_to_user fails\n");
-		retval = -EFAULT;
-		goto free_param_kptr;
-	}
-
-free_param_kptr:
-	if (!IS_ERR(param))
-		kfree(param);
-	kfree(kptr);
-
-	return retval;
-}
-
-static int ipa3_ioctl_add_flt_rule_v2(unsigned long arg)
-{
-	int retval = 0;
-	int i;
-	u8 header[128] = { 0 };
-	int pre_entry;
-	u32 usr_pyld_sz;
-	u32 pyld_sz;
-	u64 uptr = 0;
-	u8 *param = NULL;
-	u8 *kptr = NULL;
-
-	if (copy_from_user(header, (const void __user *)arg,
-		sizeof(struct ipa_ioc_add_flt_rule_v2))) {
-		IPAERR_RL("copy_from_user fails\n");
-		retval = -EFAULT;
-		goto free_param_kptr;
-	}
-	pre_entry =
-		((struct ipa_ioc_add_flt_rule_v2 *)header)->num_rules;
-	if (unlikely(((struct ipa_ioc_add_flt_rule_v2 *)
-		header)->flt_rule_size >
-		sizeof(struct ipa_flt_rule_add_i))) {
-		IPAERR_RL("unexpected rule_add_size %d\n",
-		((struct ipa_ioc_add_flt_rule_v2 *)
-		header)->flt_rule_size);
-		retval = -EPERM;
-		goto free_param_kptr;
-	}
-	/* user payload size */
-	usr_pyld_sz = ((struct ipa_ioc_add_flt_rule_v2 *)
-		header)->flt_rule_size * pre_entry;
-	/* actual payload structure size in kernel */
-	pyld_sz = sizeof(struct ipa_flt_rule_add_i)
-		* pre_entry;
-	uptr = ((struct ipa_ioc_add_flt_rule_v2 *)
-		header)->rules;
-	if (unlikely(!uptr)) {
-		IPAERR_RL("unexpected NULL rules\n");
-		retval = -EPERM;
-		goto free_param_kptr;
-	}
-	/* alloc param with same payload size as user payload */
-	param = memdup_user((const void __user *)uptr,
-		usr_pyld_sz);
-	if (IS_ERR(param)) {
-		retval = -EFAULT;
-		goto free_param_kptr;
-	}
-	/* alloc kernel pointer with actual payload size */
-	kptr = kzalloc(pyld_sz, GFP_KERNEL);
-	if (!kptr) {
-		retval = -ENOMEM;
-		goto free_param_kptr;
-	}
-	for (i = 0; i < pre_entry; i++)
-		memcpy(kptr + i * sizeof(struct ipa_flt_rule_add_i),
-			(void *)param + i *
-			((struct ipa_ioc_add_flt_rule_v2 *)
-			header)->flt_rule_size,
-			((struct ipa_ioc_add_flt_rule_v2 *)
-			header)->flt_rule_size);
-	/* modify the rule pointer to the kernel pointer */
-	((struct ipa_ioc_add_flt_rule_v2 *)header)->rules =
-		(u64)kptr;
-	if (ipa3_add_flt_rule_usr_v2((struct ipa_ioc_add_flt_rule_v2 *)
-			header, true)) {
-		IPAERR_RL("ipa3_add_flt_rule_usr_v2 fails\n");
-		retval = -EPERM;
-		goto free_param_kptr;
-	}
-	for (i = 0; i < pre_entry; i++)
-		memcpy((void *)param + i *
-			((struct ipa_ioc_add_flt_rule_v2 *)
-			header)->flt_rule_size,
-			kptr + i * sizeof(struct ipa_flt_rule_add_i),
-			((struct ipa_ioc_add_flt_rule_v2 *)
-			header)->flt_rule_size);
-	if (copy_to_user((void __user *)uptr, param,
-		usr_pyld_sz)) {
-		IPAERR_RL("copy_to_user fails\n");
-		retval = -EFAULT;
-		goto free_param_kptr;
-	}
-free_param_kptr:
-	if (!IS_ERR(param))
-		kfree(param);
-	kfree(kptr);
-
-	return retval;
-}
-
-static int ipa3_ioctl_add_flt_rule_after_v2(unsigned long arg)
-{
-	int retval = 0;
-	int i;
-	u8 header[128] = { 0 };
-	int pre_entry;
-	u32 usr_pyld_sz;
-	u32 pyld_sz;
-	u64 uptr = 0;
-	u8 *param = NULL;
-	u8 *kptr = NULL;
-
-	if (copy_from_user(header, (const void __user *)arg,
-		sizeof(struct ipa_ioc_add_flt_rule_after_v2))) {
-		IPAERR_RL("copy_from_user fails\n");
-		retval = -EFAULT;
-		goto free_param_kptr;
-	}
-	pre_entry =
-		((struct ipa_ioc_add_flt_rule_after_v2 *)
-		 header)->num_rules;
-	if (unlikely(((struct ipa_ioc_add_flt_rule_after_v2 *)
-		header)->flt_rule_size >
-		sizeof(struct ipa_flt_rule_add_i))) {
-		IPAERR_RL("unexpected rule_add_size %d\n",
-		((struct ipa_ioc_add_flt_rule_after_v2 *)
-		header)->flt_rule_size);
-		retval = -EPERM;
-		goto free_param_kptr;
-	}
-	/* user payload size */
-	usr_pyld_sz = ((struct ipa_ioc_add_flt_rule_after_v2 *)
-		header)->flt_rule_size * pre_entry;
-	/* actual payload structure size in kernel */
-	pyld_sz = sizeof(struct ipa_flt_rule_add_i)
-		* pre_entry;
-	uptr = ((struct ipa_ioc_add_flt_rule_after_v2 *)
-		header)->rules;
-	if (unlikely(!uptr)) {
-		IPAERR_RL("unexpected NULL rules\n");
-		retval = -EPERM;
-		goto free_param_kptr;
-	}
-	/* alloc param with same payload size as user payload */
-	param = memdup_user((const void __user *)uptr,
-		usr_pyld_sz);
-	if (IS_ERR(param)) {
-		retval = -EFAULT;
-		goto free_param_kptr;
-	}
-	/* alloc kernel pointer with actual payload size */
-	kptr = kzalloc(pyld_sz, GFP_KERNEL);
-	if (!kptr) {
-		retval = -ENOMEM;
-		goto free_param_kptr;
-	}
-	for (i = 0; i < pre_entry; i++)
-		memcpy(kptr + i * sizeof(struct ipa_flt_rule_add_i),
-			(void *)param + i *
-			((struct ipa_ioc_add_flt_rule_after_v2 *)
-			header)->flt_rule_size,
-			((struct ipa_ioc_add_flt_rule_after_v2 *)
-			header)->flt_rule_size);
-	/* modify the rule pointer to the kernel pointer */
-	((struct ipa_ioc_add_flt_rule_after_v2 *)header)->rules =
-		(u64)kptr;
-	if (ipa3_add_flt_rule_after_v2(
-		(struct ipa_ioc_add_flt_rule_after_v2 *)header)) {
-		IPAERR_RL("ipa3_add_flt_rule_after_v2 fails\n");
-		retval = -EPERM;
-		goto free_param_kptr;
-	}
-	for (i = 0; i < pre_entry; i++)
-		memcpy((void *)param + i *
-			((struct ipa_ioc_add_flt_rule_after_v2 *)
-			header)->flt_rule_size,
-			kptr + i * sizeof(struct ipa_flt_rule_add_i),
-			((struct ipa_ioc_add_flt_rule_after_v2 *)
-			header)->flt_rule_size);
-	if (copy_to_user((void __user *)uptr, param,
-		usr_pyld_sz)) {
-		IPAERR_RL("copy_to_user fails\n");
-		retval = -EFAULT;
-		goto free_param_kptr;
-	}
-
-free_param_kptr:
-	if (!IS_ERR(param))
-		kfree(param);
-	kfree(kptr);
-
-	return retval;
-}
-
-static int ipa3_ioctl_mdfy_flt_rule_v2(unsigned long arg)
-{
-	int retval = 0;
-	int i;
-	u8 header[128] = { 0 };
-	int pre_entry;
-	u32 usr_pyld_sz;
-	u32 pyld_sz;
-	u64 uptr = 0;
-	u8 *param = NULL;
-	u8 *kptr = NULL;
-
-	if (copy_from_user(header, (const void __user *)arg,
-		sizeof(struct ipa_ioc_mdfy_flt_rule_v2))) {
-		IPAERR_RL("copy_from_user fails\n");
-		retval = -EFAULT;
-		goto free_param_kptr;
-	}
-	pre_entry =
-		((struct ipa_ioc_mdfy_flt_rule_v2 *)
-		 header)->num_rules;
-	if (unlikely(((struct ipa_ioc_mdfy_flt_rule_v2 *)
-		header)->rule_mdfy_size >
-		sizeof(struct ipa_flt_rule_mdfy_i))) {
-		IPAERR_RL("unexpected rule_add_size %d\n",
-		((struct ipa_ioc_mdfy_flt_rule_v2 *)
-		header)->rule_mdfy_size);
-		retval = -EPERM;
-		goto free_param_kptr;
-	}
-	/* user payload size */
-	usr_pyld_sz = ((struct ipa_ioc_mdfy_flt_rule_v2 *)
-		header)->rule_mdfy_size * pre_entry;
-	/* actual payload structure size in kernel */
-	pyld_sz = sizeof(struct ipa_flt_rule_mdfy_i)
-		* pre_entry;
-	uptr = ((struct ipa_ioc_mdfy_flt_rule_v2 *)
-		header)->rules;
-	if (unlikely(!uptr)) {
-		IPAERR_RL("unexpected NULL rules\n");
-		retval = -EPERM;
-		goto free_param_kptr;
-	}
-	/* alloc param with same payload size as user payload */
-	param = memdup_user((const void __user *)uptr,
-		usr_pyld_sz);
-	if (IS_ERR(param)) {
-		retval = -EFAULT;
-		goto free_param_kptr;
-	}
-	/* alloc kernel pointer with actual payload size */
-	kptr = kzalloc(pyld_sz, GFP_KERNEL);
-	if (!kptr) {
-		retval = -ENOMEM;
-		goto free_param_kptr;
-	}
-	for (i = 0; i < pre_entry; i++)
-		memcpy(kptr + i * sizeof(struct ipa_flt_rule_mdfy_i),
-			(void *)param + i *
-			((struct ipa_ioc_mdfy_flt_rule_v2 *)
-			header)->rule_mdfy_size,
-			((struct ipa_ioc_mdfy_flt_rule_v2 *)
-			header)->rule_mdfy_size);
-	/* modify the rule pointer to the kernel pointer */
-	((struct ipa_ioc_add_flt_rule_after_v2 *)header)->rules =
-		(u64)kptr;
-	if (ipa3_mdfy_flt_rule_v2
-		((struct ipa_ioc_mdfy_flt_rule_v2 *)header)) {
-		IPAERR_RL("ipa3_mdfy_flt_rule_v2 fails\n");
-		retval = -EPERM;
-		goto free_param_kptr;
-	}
-	for (i = 0; i < pre_entry; i++)
-		memcpy((void *)param + i *
-			((struct ipa_ioc_mdfy_flt_rule_v2 *)
-			header)->rule_mdfy_size,
-			kptr + i * sizeof(struct ipa_flt_rule_mdfy_i),
-			((struct ipa_ioc_mdfy_flt_rule_v2 *)
-			header)->rule_mdfy_size);
-	if (copy_to_user((void __user *)uptr, param,
-		usr_pyld_sz)) {
-		IPAERR_RL("copy_to_user fails\n");
-		retval = -EFAULT;
-		goto free_param_kptr;
-	}
-
-free_param_kptr:
-	if (!IS_ERR(param))
-		kfree(param);
-	kfree(kptr);
-
-	return retval;
-}
-
-static int ipa3_ioctl_fnr_counter_alloc(unsigned long arg)
-{
-	int retval = 0;
-	u8 header[128] = { 0 };
-
-	if (copy_from_user(header, (const void __user *)arg,
-		sizeof(struct ipa_ioc_flt_rt_counter_alloc))) {
-		IPAERR("copy_from_user fails\n");
-		return -EFAULT;
-	}
-	if (((struct ipa_ioc_flt_rt_counter_alloc *)
-		header)->hw_counter.num_counters >
-		IPA_FLT_RT_HW_COUNTER ||
-		((struct ipa_ioc_flt_rt_counter_alloc *)
-		header)->sw_counter.num_counters >
-		IPA_FLT_RT_SW_COUNTER) {
-		IPAERR("failed: wrong sw/hw num_counters\n");
-		return -EPERM;
-	}
-	if (((struct ipa_ioc_flt_rt_counter_alloc *)
-		header)->hw_counter.num_counters == 0 &&
-		((struct ipa_ioc_flt_rt_counter_alloc *)
-		header)->sw_counter.num_counters == 0) {
-		IPAERR("failed: both sw/hw num_counters 0\n");
-		return -EPERM;
-	}
-	retval = ipa3_alloc_counter_id
-		((struct ipa_ioc_flt_rt_counter_alloc *)header);
-	if (retval < 0) {
-		IPAERR("ipa3_alloc_counter_id failed\n");
-		return retval;
-	}
-	if (copy_to_user((void __user *)arg, header,
-		sizeof(struct ipa_ioc_flt_rt_counter_alloc))) {
-		IPAERR("copy_to_user fails\n");
-		ipa3_counter_remove_hdl(
-		((struct ipa_ioc_flt_rt_counter_alloc *)
-		header)->hdl);
-		return -EFAULT;
-	}
-	return 0;
-}
-
-static int ipa3_ioctl_fnr_counter_query(unsigned long arg)
-{
-	int retval = 0;
-	int i;
-	u8 header[128] = { 0 };
-	int pre_entry;
-	u32 usr_pyld_sz;
-	u32 pyld_sz;
-	u64 uptr = 0;
-	u8 *param = NULL;
-	u8 *kptr = NULL;
-
-	if (copy_from_user(header, (const void __user *)arg,
-		sizeof(struct ipa_ioc_flt_rt_query))) {
-		IPAERR_RL("copy_from_user fails\n");
-		retval = -EFAULT;
-		goto free_param_kptr;
-	}
-	pre_entry =
-		((struct ipa_ioc_flt_rt_query *)
-		header)->end_id - ((struct ipa_ioc_flt_rt_query *)
-		header)->start_id + 1;
-	if (pre_entry <= 0 || pre_entry > IPA_MAX_FLT_RT_CNT_INDEX) {
-		IPAERR("IPA_IOC_FNR_COUNTER_QUERY failed: num %d\n",
-			pre_entry);
-		retval = -EPERM;
-		goto free_param_kptr;
-	}
-	if (((struct ipa_ioc_flt_rt_query *)header)->stats_size
-		> sizeof(struct ipa_flt_rt_stats)) {
-		IPAERR_RL("unexpected stats_size %d\n",
-		((struct ipa_ioc_flt_rt_query *)header)->stats_size);
-		retval = -EPERM;
-		goto free_param_kptr;
-	}
-	/* user payload size */
-	usr_pyld_sz = ((struct ipa_ioc_flt_rt_query *)
-		header)->stats_size * pre_entry;
-	/* actual payload structure size in kernel */
-	pyld_sz = sizeof(struct ipa_flt_rt_stats) * pre_entry;
-	uptr = ((struct ipa_ioc_flt_rt_query *)
-		header)->stats;
-	if (unlikely(!uptr)) {
-		IPAERR_RL("unexpected NULL rules\n");
-		retval = -EPERM;
-		goto free_param_kptr;
-	}
-	/* alloc param with same payload size as user payload */
-	param = memdup_user((const void __user *)uptr,
-		usr_pyld_sz);
-	if (IS_ERR(param)) {
-		retval = -EFAULT;
-		goto free_param_kptr;
-	}
-	/* alloc kernel pointer with actual payload size */
-	kptr = kzalloc(pyld_sz, GFP_KERNEL);
-	if (!kptr) {
-		retval = -ENOMEM;
-		goto free_param_kptr;
-	}
-	for (i = 0; i < pre_entry; i++)
-		memcpy(kptr + i * sizeof(struct ipa_flt_rt_stats),
-			(void *)param + i *
-			((struct ipa_ioc_flt_rt_query *)
-			header)->stats_size,
-			((struct ipa_ioc_flt_rt_query *)
-			header)->stats_size);
-	/* modify the rule pointer to the kernel pointer */
-	((struct ipa_ioc_flt_rt_query *)
-		header)->stats = (u64)kptr;
-	retval = ipa_get_flt_rt_stats
-		((struct ipa_ioc_flt_rt_query *)header);
-	if (retval < 0) {
-		IPAERR("ipa_get_flt_rt_stats failed\n");
-		retval = -EPERM;
-		goto free_param_kptr;
-	}
-	for (i = 0; i < pre_entry; i++)
-		memcpy((void *)param + i *
-			((struct ipa_ioc_flt_rt_query *)
-			header)->stats_size,
-			kptr + i * sizeof(struct ipa_flt_rt_stats),
-			((struct ipa_ioc_flt_rt_query *)
-			header)->stats_size);
-	if (copy_to_user((void __user *)uptr, param,
-		usr_pyld_sz)) {
-		IPAERR_RL("copy_to_user fails\n");
-		retval = -EFAULT;
-		goto free_param_kptr;
-	}
-
-free_param_kptr:
-	if (!IS_ERR(param))
-		kfree(param);
-	kfree(kptr);
-
-	return retval;
-}
-
-static long ipa3_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
-{
-	int retval = 0;
-	u32 pyld_sz;
-	u8 header[128] = { 0 };
-	u8 *param = NULL;
-	bool is_vlan_mode;
-	struct ipa_ioc_nat_alloc_mem nat_mem;
-	struct ipa_ioc_nat_ipv6ct_table_alloc table_alloc;
-	struct ipa_ioc_v4_nat_init nat_init;
-	struct ipa_ioc_ipv6ct_init ipv6ct_init;
-	struct ipa_ioc_v4_nat_del nat_del;
-	struct ipa_ioc_nat_ipv6ct_table_del table_del;
-	struct ipa_ioc_nat_pdn_entry mdfy_pdn;
-	struct ipa_ioc_nat_dma_cmd *table_dma_cmd;
-	struct ipa_ioc_get_vlan_mode vlan_mode;
-	struct ipa_ioc_wigig_fst_switch fst_switch;
-	size_t sz;
-	int pre_entry;
-	int hdl;
-
-	IPADBG("cmd=%x nr=%d\n", cmd, _IOC_NR(cmd));
-
-	if (_IOC_TYPE(cmd) != IPA_IOC_MAGIC)
-		return -ENOTTY;
-
-	if (!ipa3_is_ready()) {
-		IPAERR("IPA not ready, waiting for init completion\n");
-		wait_for_completion(&ipa3_ctx->init_completion_obj);
-	}
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	switch (cmd) {
-	case IPA_IOC_ALLOC_NAT_MEM:
-		if (copy_from_user(&nat_mem, (const void __user *)arg,
-			sizeof(struct ipa_ioc_nat_alloc_mem))) {
-			retval = -EFAULT;
-			break;
-		}
-		/* null terminate the string */
-		nat_mem.dev_name[IPA_RESOURCE_NAME_MAX - 1] = '\0';
-
-		if (ipa3_allocate_nat_device(&nat_mem)) {
-			retval = -EFAULT;
-			break;
-		}
-		if (copy_to_user((void __user *)arg, &nat_mem,
-			sizeof(struct ipa_ioc_nat_alloc_mem))) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-	case IPA_IOC_ALLOC_NAT_TABLE:
-		if (copy_from_user(&table_alloc, (const void __user *)arg,
-			sizeof(struct ipa_ioc_nat_ipv6ct_table_alloc))) {
-			retval = -EFAULT;
-			break;
-		}
-
-		if (ipa3_allocate_nat_table(&table_alloc)) {
-			retval = -EFAULT;
-			break;
-		}
-		if (table_alloc.offset &&
-			copy_to_user((void __user *)arg, &table_alloc, sizeof(
-				struct ipa_ioc_nat_ipv6ct_table_alloc))) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-
-	case IPA_IOC_ALLOC_IPV6CT_TABLE:
-		if (copy_from_user(&table_alloc, (const void __user *)arg,
-			sizeof(struct ipa_ioc_nat_ipv6ct_table_alloc))) {
-			retval = -EFAULT;
-			break;
-		}
-
-		if (ipa3_allocate_ipv6ct_table(&table_alloc)) {
-			retval = -EFAULT;
-			break;
-		}
-		if (table_alloc.offset &&
-			copy_to_user((void __user *)arg, &table_alloc, sizeof(
-				struct ipa_ioc_nat_ipv6ct_table_alloc))) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-
-	case IPA_IOC_V4_INIT_NAT:
-		if (copy_from_user(&nat_init, (const void __user *)arg,
-			sizeof(struct ipa_ioc_v4_nat_init))) {
-			retval = -EFAULT;
-			break;
-		}
-		if (ipa3_nat_init_cmd(&nat_init)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-
-	case IPA_IOC_INIT_IPV6CT_TABLE:
-		if (copy_from_user(&ipv6ct_init, (const void __user *)arg,
-			sizeof(struct ipa_ioc_ipv6ct_init))) {
-			retval = -EFAULT;
-			break;
-		}
-		if (ipa3_ipv6ct_init_cmd(&ipv6ct_init)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-
-	case IPA_IOC_TABLE_DMA_CMD:
-		table_dma_cmd = (struct ipa_ioc_nat_dma_cmd *)header;
-		if (copy_from_user(header, (const void __user *)arg,
-			sizeof(struct ipa_ioc_nat_dma_cmd))) {
-			retval = -EFAULT;
-			break;
-		}
-		pre_entry = table_dma_cmd->entries;
-		pyld_sz = sizeof(struct ipa_ioc_nat_dma_cmd) +
-			pre_entry * sizeof(struct ipa_ioc_nat_dma_one);
-		param = memdup_user((const void __user *)arg, pyld_sz);
-		if (IS_ERR(param)) {
-			retval = PTR_ERR(param);
-			break;
-		}
-		table_dma_cmd = (struct ipa_ioc_nat_dma_cmd *)param;
-		/* add check in case user-space module compromised */
-		if (unlikely(table_dma_cmd->entries != pre_entry)) {
-			IPAERR_RL("current %d pre %d\n",
-				table_dma_cmd->entries, pre_entry);
-			retval = -EFAULT;
-			break;
-		}
-		if (ipa3_table_dma_cmd(table_dma_cmd)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-
-	case IPA_IOC_V4_DEL_NAT:
-		if (copy_from_user(&nat_del, (const void __user *)arg,
-			sizeof(struct ipa_ioc_v4_nat_del))) {
-			retval = -EFAULT;
-			break;
-		}
-		if (ipa3_nat_del_cmd(&nat_del)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-
-	case IPA_IOC_DEL_NAT_TABLE:
-		if (copy_from_user(&table_del, (const void __user *)arg,
-			sizeof(struct ipa_ioc_nat_ipv6ct_table_del))) {
-			retval = -EFAULT;
-			break;
-		}
-		if (ipa3_del_nat_table(&table_del)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-
-	case IPA_IOC_DEL_IPV6CT_TABLE:
-		if (copy_from_user(&table_del, (const void __user *)arg,
-			sizeof(struct ipa_ioc_nat_ipv6ct_table_del))) {
-			retval = -EFAULT;
-			break;
-		}
-		if (ipa3_del_ipv6ct_table(&table_del)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-
-	case IPA_IOC_NAT_MODIFY_PDN:
-		if (copy_from_user(&mdfy_pdn, (const void __user *)arg,
-			sizeof(struct ipa_ioc_nat_pdn_entry))) {
-			retval = -EFAULT;
-			break;
-		}
-		if (ipa3_nat_mdfy_pdn(&mdfy_pdn)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-
-	case IPA_IOC_ADD_HDR:
-		if (copy_from_user(header, (const void __user *)arg,
-			sizeof(struct ipa_ioc_add_hdr))) {
-			retval = -EFAULT;
-			break;
-		}
-		pre_entry =
-			((struct ipa_ioc_add_hdr *)header)->num_hdrs;
-		pyld_sz =
-		   sizeof(struct ipa_ioc_add_hdr) +
-		   pre_entry * sizeof(struct ipa_hdr_add);
-		param = memdup_user((const void __user *)arg, pyld_sz);
-		if (IS_ERR(param)) {
-			retval = PTR_ERR(param);
-			break;
-		}
-		/* add check in case user-space module compromised */
-		if (unlikely(((struct ipa_ioc_add_hdr *)param)->num_hdrs
-			!= pre_entry)) {
-			IPAERR_RL("current %d pre %d\n",
-				((struct ipa_ioc_add_hdr *)param)->num_hdrs,
-				pre_entry);
-			retval = -EFAULT;
-			break;
-		}
-		if (ipa3_add_hdr_usr((struct ipa_ioc_add_hdr *)param,
-			true)) {
-			retval = -EFAULT;
-			break;
-		}
-		if (copy_to_user((void __user *)arg, param, pyld_sz)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-
-	case IPA_IOC_DEL_HDR:
-		if (copy_from_user(header, (const void __user *)arg,
-			sizeof(struct ipa_ioc_del_hdr))) {
-			retval = -EFAULT;
-			break;
-		}
-		pre_entry =
-			((struct ipa_ioc_del_hdr *)header)->num_hdls;
-		pyld_sz =
-		   sizeof(struct ipa_ioc_del_hdr) +
-		   pre_entry * sizeof(struct ipa_hdr_del);
-		param = memdup_user((const void __user *)arg, pyld_sz);
-		if (IS_ERR(param)) {
-			retval = PTR_ERR(param);
-			break;
-		}
-		/* add check in case user-space module compromised */
-		if (unlikely(((struct ipa_ioc_del_hdr *)param)->num_hdls
-			!= pre_entry)) {
-			IPAERR_RL("current %d pre %d\n",
-				((struct ipa_ioc_del_hdr *)param)->num_hdls,
-				pre_entry);
-			retval = -EFAULT;
-			break;
-		}
-		if (ipa3_del_hdr_by_user((struct ipa_ioc_del_hdr *)param,
-			true)) {
-			retval = -EFAULT;
-			break;
-		}
-		if (copy_to_user((void __user *)arg, param, pyld_sz)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-
-	case IPA_IOC_ADD_RT_RULE:
-		if (copy_from_user(header, (const void __user *)arg,
-			sizeof(struct ipa_ioc_add_rt_rule))) {
-			retval = -EFAULT;
-			break;
-		}
-		pre_entry =
-			((struct ipa_ioc_add_rt_rule *)header)->num_rules;
-		pyld_sz =
-		   sizeof(struct ipa_ioc_add_rt_rule) +
-		   pre_entry * sizeof(struct ipa_rt_rule_add);
-		param = memdup_user((const void __user *)arg, pyld_sz);
-		if (IS_ERR(param)) {
-			retval = PTR_ERR(param);
-			break;
-		}
-		/* add check in case user-space module compromised */
-		if (unlikely(((struct ipa_ioc_add_rt_rule *)param)->num_rules
-			!= pre_entry)) {
-			IPAERR_RL("current %d pre %d\n",
-				((struct ipa_ioc_add_rt_rule *)param)->
-				num_rules,
-				pre_entry);
-			retval = -EFAULT;
-			break;
-		}
-		if (ipa3_add_rt_rule_usr((struct ipa_ioc_add_rt_rule *)param,
-				true)) {
-			retval = -EFAULT;
-			break;
-		}
-		if (copy_to_user((void __user *)arg, param, pyld_sz)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-
-	case IPA_IOC_ADD_RT_RULE_EXT:
-		if (copy_from_user(header,
-				(const void __user *)arg,
-				sizeof(struct ipa_ioc_add_rt_rule_ext))) {
-			retval = -EFAULT;
-			break;
-		}
-		pre_entry =
-			((struct ipa_ioc_add_rt_rule_ext *)header)->num_rules;
-		pyld_sz =
-		   sizeof(struct ipa_ioc_add_rt_rule_ext) +
-		   pre_entry * sizeof(struct ipa_rt_rule_add_ext);
-		param = memdup_user((const void __user *)arg, pyld_sz);
-		if (IS_ERR(param)) {
-			retval = PTR_ERR(param);
-			break;
-		}
-		/* add check in case user-space module compromised */
-		if (unlikely(
-			((struct ipa_ioc_add_rt_rule_ext *)param)->num_rules
-			!= pre_entry)) {
-			IPAERR(" prevent memory corruption(%d not match %d)\n",
-				((struct ipa_ioc_add_rt_rule_ext *)param)->
-				num_rules,
-				pre_entry);
-			retval = -EINVAL;
-			break;
-		}
-		if (ipa3_add_rt_rule_ext(
-			(struct ipa_ioc_add_rt_rule_ext *)param)) {
-			retval = -EFAULT;
-			break;
-		}
-		if (copy_to_user((void __user *)arg, param, pyld_sz)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-
-	case IPA_IOC_ADD_RT_RULE_AFTER:
-		if (copy_from_user(header, (const void __user *)arg,
-			sizeof(struct ipa_ioc_add_rt_rule_after))) {
-
-			retval = -EFAULT;
-			break;
-		}
-		pre_entry =
-			((struct ipa_ioc_add_rt_rule_after *)header)->num_rules;
-		pyld_sz =
-		   sizeof(struct ipa_ioc_add_rt_rule_after) +
-		   pre_entry * sizeof(struct ipa_rt_rule_add);
-		param = memdup_user((const void __user *)arg, pyld_sz);
-		if (IS_ERR(param)) {
-			retval = PTR_ERR(param);
-			break;
-		}
-		/* add check in case user-space module compromised */
-		if (unlikely(((struct ipa_ioc_add_rt_rule_after *)param)->
-			num_rules != pre_entry)) {
-			IPAERR_RL("current %d pre %d\n",
-				((struct ipa_ioc_add_rt_rule_after *)param)->
-				num_rules,
-				pre_entry);
-			retval = -EFAULT;
-			break;
-		}
-		if (ipa3_add_rt_rule_after(
-			(struct ipa_ioc_add_rt_rule_after *)param)) {
-
-			retval = -EFAULT;
-			break;
-		}
-		if (copy_to_user((void __user *)arg, param, pyld_sz)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-
-	case IPA_IOC_MDFY_RT_RULE:
-		if (copy_from_user(header, (const void __user *)arg,
-			sizeof(struct ipa_ioc_mdfy_rt_rule))) {
-			retval = -EFAULT;
-			break;
-		}
-		pre_entry =
-			((struct ipa_ioc_mdfy_rt_rule *)header)->num_rules;
-		pyld_sz =
-		   sizeof(struct ipa_ioc_mdfy_rt_rule) +
-		   pre_entry * sizeof(struct ipa_rt_rule_mdfy);
-		param = memdup_user((const void __user *)arg, pyld_sz);
-		if (IS_ERR(param)) {
-			retval = PTR_ERR(param);
-			break;
-		}
-		/* add check in case user-space module compromised */
-		if (unlikely(((struct ipa_ioc_mdfy_rt_rule *)param)->num_rules
-			!= pre_entry)) {
-			IPAERR_RL("current %d pre %d\n",
-				((struct ipa_ioc_mdfy_rt_rule *)param)->
-				num_rules,
-				pre_entry);
-			retval = -EFAULT;
-			break;
-		}
-		if (ipa3_mdfy_rt_rule((struct ipa_ioc_mdfy_rt_rule *)param)) {
-			retval = -EFAULT;
-			break;
-		}
-		if (copy_to_user((void __user *)arg, param, pyld_sz)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-
-	case IPA_IOC_DEL_RT_RULE:
-		if (copy_from_user(header, (const void __user *)arg,
-			sizeof(struct ipa_ioc_del_rt_rule))) {
-			retval = -EFAULT;
-			break;
-		}
-		pre_entry =
-			((struct ipa_ioc_del_rt_rule *)header)->num_hdls;
-		pyld_sz =
-		   sizeof(struct ipa_ioc_del_rt_rule) +
-		   pre_entry * sizeof(struct ipa_rt_rule_del);
-		param = memdup_user((const void __user *)arg, pyld_sz);
-		if (IS_ERR(param)) {
-			retval = PTR_ERR(param);
-			break;
-		}
-		/* add check in case user-space module compromised */
-		if (unlikely(((struct ipa_ioc_del_rt_rule *)param)->num_hdls
-			!= pre_entry)) {
-			IPAERR_RL("current %d pre %d\n",
-				((struct ipa_ioc_del_rt_rule *)param)->num_hdls,
-				pre_entry);
-			retval = -EFAULT;
-			break;
-		}
-		if (ipa3_del_rt_rule((struct ipa_ioc_del_rt_rule *)param)) {
-			retval = -EFAULT;
-			break;
-		}
-		if (copy_to_user((void __user *)arg, param, pyld_sz)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-
-	case IPA_IOC_ADD_FLT_RULE:
-		if (copy_from_user(header, (const void __user *)arg,
-			sizeof(struct ipa_ioc_add_flt_rule))) {
-			retval = -EFAULT;
-			break;
-		}
-		pre_entry =
-			((struct ipa_ioc_add_flt_rule *)header)->num_rules;
-		pyld_sz =
-		   sizeof(struct ipa_ioc_add_flt_rule) +
-		   pre_entry * sizeof(struct ipa_flt_rule_add);
-		param = memdup_user((const void __user *)arg, pyld_sz);
-		if (IS_ERR(param)) {
-			retval = PTR_ERR(param);
-			break;
-		}
-		/* add check in case user-space module compromised */
-		if (unlikely(((struct ipa_ioc_add_flt_rule *)param)->num_rules
-			!= pre_entry)) {
-			IPAERR_RL("current %d pre %d\n",
-				((struct ipa_ioc_add_flt_rule *)param)->
-				num_rules,
-				pre_entry);
-			retval = -EFAULT;
-			break;
-		}
-		if (ipa3_add_flt_rule_usr((struct ipa_ioc_add_flt_rule *)param,
-				true)) {
-			retval = -EFAULT;
-			break;
-		}
-		if (copy_to_user((void __user *)arg, param, pyld_sz)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-
-	case IPA_IOC_ADD_FLT_RULE_AFTER:
-		if (copy_from_user(header, (const void __user *)arg,
-			sizeof(struct ipa_ioc_add_flt_rule_after))) {
-
-			retval = -EFAULT;
-			break;
-		}
-		pre_entry =
-			((struct ipa_ioc_add_flt_rule_after *)header)->
-			num_rules;
-		pyld_sz =
-		   sizeof(struct ipa_ioc_add_flt_rule_after) +
-		   pre_entry * sizeof(struct ipa_flt_rule_add);
-		param = memdup_user((const void __user *)arg, pyld_sz);
-		if (IS_ERR(param)) {
-			retval = PTR_ERR(param);
-			break;
-		}
-		/* add check in case user-space module compromised */
-		if (unlikely(((struct ipa_ioc_add_flt_rule_after *)param)->
-			num_rules != pre_entry)) {
-			IPAERR_RL("current %d pre %d\n",
-				((struct ipa_ioc_add_flt_rule_after *)param)->
-				num_rules,
-				pre_entry);
-			retval = -EFAULT;
-			break;
-		}
-		if (ipa3_add_flt_rule_after(
-				(struct ipa_ioc_add_flt_rule_after *)param)) {
-			retval = -EFAULT;
-			break;
-		}
-		if (copy_to_user((void __user *)arg, param, pyld_sz)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-
-	case IPA_IOC_DEL_FLT_RULE:
-		if (copy_from_user(header, (const void __user *)arg,
-			sizeof(struct ipa_ioc_del_flt_rule))) {
-			retval = -EFAULT;
-			break;
-		}
-		pre_entry =
-			((struct ipa_ioc_del_flt_rule *)header)->num_hdls;
-		pyld_sz =
-		   sizeof(struct ipa_ioc_del_flt_rule) +
-		   pre_entry * sizeof(struct ipa_flt_rule_del);
-		param = memdup_user((const void __user *)arg, pyld_sz);
-		if (IS_ERR(param)) {
-			retval = PTR_ERR(param);
-			break;
-		}
-		/* add check in case user-space module compromised */
-		if (unlikely(((struct ipa_ioc_del_flt_rule *)param)->num_hdls
-			!= pre_entry)) {
-			IPAERR_RL("current %d pre %d\n",
-				((struct ipa_ioc_del_flt_rule *)param)->
-				num_hdls,
-				pre_entry);
-			retval = -EFAULT;
-			break;
-		}
-		if (ipa3_del_flt_rule((struct ipa_ioc_del_flt_rule *)param)) {
-			retval = -EFAULT;
-			break;
-		}
-		if (copy_to_user((void __user *)arg, param, pyld_sz)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-
-	case IPA_IOC_MDFY_FLT_RULE:
-		if (copy_from_user(header, (const void __user *)arg,
-			sizeof(struct ipa_ioc_mdfy_flt_rule))) {
-			retval = -EFAULT;
-			break;
-		}
-		pre_entry =
-			((struct ipa_ioc_mdfy_flt_rule *)header)->num_rules;
-		pyld_sz =
-		   sizeof(struct ipa_ioc_mdfy_flt_rule) +
-		   pre_entry * sizeof(struct ipa_flt_rule_mdfy);
-		param = memdup_user((const void __user *)arg, pyld_sz);
-		if (IS_ERR(param)) {
-			retval = PTR_ERR(param);
-			break;
-		}
-		/* add check in case user-space module compromised */
-		if (unlikely(((struct ipa_ioc_mdfy_flt_rule *)param)->num_rules
-			!= pre_entry)) {
-			IPAERR_RL("current %d pre %d\n",
-				((struct ipa_ioc_mdfy_flt_rule *)param)->
-				num_rules,
-				pre_entry);
-			retval = -EFAULT;
-			break;
-		}
-		if (ipa3_mdfy_flt_rule((struct ipa_ioc_mdfy_flt_rule *)param)) {
-			retval = -EFAULT;
-			break;
-		}
-		if (copy_to_user((void __user *)arg, param, pyld_sz)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-
-	case IPA_IOC_COMMIT_HDR:
-		retval = ipa3_commit_hdr();
-		break;
-	case IPA_IOC_RESET_HDR:
-		retval = ipa3_reset_hdr(false);
-		break;
-	case IPA_IOC_COMMIT_RT:
-		retval = ipa3_commit_rt(arg);
-		break;
-	case IPA_IOC_RESET_RT:
-		retval = ipa3_reset_rt(arg, false);
-		break;
-	case IPA_IOC_COMMIT_FLT:
-		retval = ipa3_commit_flt(arg);
-		break;
-	case IPA_IOC_RESET_FLT:
-		retval = ipa3_reset_flt(arg, false);
-		break;
-	case IPA_IOC_GET_RT_TBL:
-		if (copy_from_user(header, (const void __user *)arg,
-			sizeof(struct ipa_ioc_get_rt_tbl))) {
-			retval = -EFAULT;
-			break;
-		}
-		if (ipa3_get_rt_tbl((struct ipa_ioc_get_rt_tbl *)header)) {
-			retval = -EFAULT;
-			break;
-		}
-		if (copy_to_user((void __user *)arg, header,
-					sizeof(struct ipa_ioc_get_rt_tbl))) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-	case IPA_IOC_PUT_RT_TBL:
-		retval = ipa3_put_rt_tbl(arg);
-		break;
-	case IPA_IOC_GET_HDR:
-		if (copy_from_user(header, (const void __user *)arg,
-			sizeof(struct ipa_ioc_get_hdr))) {
-			retval = -EFAULT;
-			break;
-		}
-		if (ipa3_get_hdr((struct ipa_ioc_get_hdr *)header)) {
-			retval = -EFAULT;
-			break;
-		}
-		if (copy_to_user((void __user *)arg, header,
-			sizeof(struct ipa_ioc_get_hdr))) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-	case IPA_IOC_PUT_HDR:
-		retval = ipa3_put_hdr(arg);
-		break;
-	case IPA_IOC_SET_FLT:
-		retval = ipa3_cfg_filter(arg);
-		break;
-	case IPA_IOC_COPY_HDR:
-		if (copy_from_user(header, (const void __user *)arg,
-			sizeof(struct ipa_ioc_copy_hdr))) {
-			retval = -EFAULT;
-			break;
-		}
-		if (ipa3_copy_hdr((struct ipa_ioc_copy_hdr *)header)) {
-			retval = -EFAULT;
-			break;
-		}
-		if (copy_to_user((void __user *)arg, header,
-			sizeof(struct ipa_ioc_copy_hdr))) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-	case IPA_IOC_QUERY_INTF:
-		if (copy_from_user(header, (const void __user *)arg,
-			sizeof(struct ipa_ioc_query_intf))) {
-			retval = -EFAULT;
-			break;
-		}
-		if (ipa3_query_intf((struct ipa_ioc_query_intf *)header)) {
-			retval = -1;
-			break;
-		}
-		if (copy_to_user((void __user *)arg, header,
-			sizeof(struct ipa_ioc_query_intf))) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-	case IPA_IOC_QUERY_INTF_TX_PROPS:
-		sz = sizeof(struct ipa_ioc_query_intf_tx_props);
-		if (copy_from_user(header, (const void __user *)arg, sz)) {
-			retval = -EFAULT;
-			break;
-		}
-
-		if (((struct ipa_ioc_query_intf_tx_props *)header)->num_tx_props
-			> IPA_NUM_PROPS_MAX) {
-			retval = -EFAULT;
-			break;
-		}
-		pre_entry =
-			((struct ipa_ioc_query_intf_tx_props *)
-			header)->num_tx_props;
-		pyld_sz = sz + pre_entry *
-			sizeof(struct ipa_ioc_tx_intf_prop);
-		param = memdup_user((const void __user *)arg, pyld_sz);
-		if (IS_ERR(param)) {
-			retval = PTR_ERR(param);
-			break;
-		}
-		/* add check in case user-space module compromised */
-		if (unlikely(((struct ipa_ioc_query_intf_tx_props *)
-			param)->num_tx_props
-			!= pre_entry)) {
-			IPAERR_RL("current %d pre %d\n",
-				((struct ipa_ioc_query_intf_tx_props *)
-				param)->num_tx_props, pre_entry);
-			retval = -EFAULT;
-			break;
-		}
-		if (ipa3_query_intf_tx_props(
-			(struct ipa_ioc_query_intf_tx_props *)param)) {
-			retval = -1;
-			break;
-		}
-		if (copy_to_user((void __user *)arg, param, pyld_sz)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-	case IPA_IOC_QUERY_INTF_RX_PROPS:
-		sz = sizeof(struct ipa_ioc_query_intf_rx_props);
-		if (copy_from_user(header, (const void __user *)arg, sz)) {
-			retval = -EFAULT;
-			break;
-		}
-
-		if (((struct ipa_ioc_query_intf_rx_props *)header)->num_rx_props
-			> IPA_NUM_PROPS_MAX) {
-			retval = -EFAULT;
-			break;
-		}
-		pre_entry =
-			((struct ipa_ioc_query_intf_rx_props *)
-			header)->num_rx_props;
-		pyld_sz = sz + pre_entry *
-			sizeof(struct ipa_ioc_rx_intf_prop);
-		param = memdup_user((const void __user *)arg, pyld_sz);
-		if (IS_ERR(param)) {
-			retval = PTR_ERR(param);
-			break;
-		}
-		/* add check in case user-space module compromised */
-		if (unlikely(((struct ipa_ioc_query_intf_rx_props *)
-			param)->num_rx_props != pre_entry)) {
-			IPAERR_RL("current %d pre %d\n",
-				((struct ipa_ioc_query_intf_rx_props *)
-				param)->num_rx_props, pre_entry);
-			retval = -EFAULT;
-			break;
-		}
-		if (ipa3_query_intf_rx_props(
-			(struct ipa_ioc_query_intf_rx_props *)param)) {
-			retval = -1;
-			break;
-		}
-		if (copy_to_user((void __user *)arg, param, pyld_sz)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-	case IPA_IOC_QUERY_INTF_EXT_PROPS:
-		sz = sizeof(struct ipa_ioc_query_intf_ext_props);
-		if (copy_from_user(header, (const void __user *)arg, sz)) {
-			retval = -EFAULT;
-			break;
-		}
-
-		if (((struct ipa_ioc_query_intf_ext_props *)
-			header)->num_ext_props > IPA_NUM_PROPS_MAX) {
-			retval = -EFAULT;
-			break;
-		}
-		pre_entry =
-			((struct ipa_ioc_query_intf_ext_props *)
-			header)->num_ext_props;
-		pyld_sz = sz + pre_entry *
-			sizeof(struct ipa_ioc_ext_intf_prop);
-		param = memdup_user((const void __user *)arg, pyld_sz);
-		if (IS_ERR(param)) {
-			retval = PTR_ERR(param);
-			break;
-		}
-		/* add check in case user-space module compromised */
-		if (unlikely(((struct ipa_ioc_query_intf_ext_props *)
-			param)->num_ext_props != pre_entry)) {
-			IPAERR_RL("current %d pre %d\n",
-				((struct ipa_ioc_query_intf_ext_props *)
-				param)->num_ext_props, pre_entry);
-			retval = -EFAULT;
-			break;
-		}
-		if (ipa3_query_intf_ext_props(
-			(struct ipa_ioc_query_intf_ext_props *)param)) {
-			retval = -1;
-			break;
-		}
-		if (copy_to_user((void __user *)arg, param, pyld_sz)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-	case IPA_IOC_PULL_MSG:
-		if (copy_from_user(header, (const void __user *)arg,
-			sizeof(struct ipa_msg_meta))) {
-			retval = -EFAULT;
-			break;
-		}
-		pre_entry =
-		   ((struct ipa_msg_meta *)header)->msg_len;
-		pyld_sz = sizeof(struct ipa_msg_meta) +
-		   pre_entry;
-		param = memdup_user((const void __user *)arg, pyld_sz);
-		if (IS_ERR(param)) {
-			retval = PTR_ERR(param);
-			break;
-		}
-		/* add check in case user-space module compromised */
-		if (unlikely(((struct ipa_msg_meta *)param)->msg_len
-			!= pre_entry)) {
-			IPAERR_RL("current %d pre %d\n",
-				((struct ipa_msg_meta *)param)->msg_len,
-				pre_entry);
-			retval = -EFAULT;
-			break;
-		}
-		if (ipa3_pull_msg((struct ipa_msg_meta *)param,
-			(char *)param + sizeof(struct ipa_msg_meta),
-			((struct ipa_msg_meta *)param)->msg_len) !=
-			((struct ipa_msg_meta *)param)->msg_len) {
-			retval = -1;
-			break;
-		}
-		if (copy_to_user((void __user *)arg, param, pyld_sz)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-	case IPA_IOC_RM_ADD_DEPENDENCY:
-		/* IPA RM is deprecate because IPA PM is used */
-		IPAERR("using obselete command: IPA_IOC_RM_ADD_DEPENDENCY");
-		return -EINVAL;
-
-	case IPA_IOC_RM_DEL_DEPENDENCY:
-		/* IPA RM is deprecate because IPA PM is used */
-		IPAERR("using obselete command: IPA_IOC_RM_DEL_DEPENDENCY");
-		return -EINVAL;
-
-	case IPA_IOC_GENERATE_FLT_EQ:
-		{
-			struct ipa_ioc_generate_flt_eq flt_eq;
-
-			if (copy_from_user(&flt_eq, (const void __user *)arg,
-				sizeof(struct ipa_ioc_generate_flt_eq))) {
-				retval = -EFAULT;
-				break;
-			}
-			if (ipahal_flt_generate_equation(flt_eq.ip,
-				&flt_eq.attrib, &flt_eq.eq_attrib)) {
-				retval = -EFAULT;
-				break;
-			}
-			if (copy_to_user((void __user *)arg, &flt_eq,
-				sizeof(struct ipa_ioc_generate_flt_eq))) {
-				retval = -EFAULT;
-				break;
-			}
-			break;
-		}
-	case IPA_IOC_QUERY_EP_MAPPING:
-		{
-			retval = ipa3_get_ep_mapping(arg);
-			break;
-		}
-	case IPA_IOC_QUERY_RT_TBL_INDEX:
-		if (copy_from_user(header, (const void __user *)arg,
-			sizeof(struct ipa_ioc_get_rt_tbl_indx))) {
-			retval = -EFAULT;
-			break;
-		}
-		if (ipa3_query_rt_index(
-			(struct ipa_ioc_get_rt_tbl_indx *)header)) {
-			retval = -EFAULT;
-			break;
-		}
-		if (copy_to_user((void __user *)arg, header,
-			sizeof(struct ipa_ioc_get_rt_tbl_indx))) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-	case IPA_IOC_WRITE_QMAPID:
-		if (copy_from_user(header, (const void __user *)arg,
-			sizeof(struct ipa_ioc_write_qmapid))) {
-			retval = -EFAULT;
-			break;
-		}
-		if (ipa3_write_qmap_id((struct ipa_ioc_write_qmapid *)header)) {
-			retval = -EFAULT;
-			break;
-		}
-		if (copy_to_user((void __user *)arg, header,
-			sizeof(struct ipa_ioc_write_qmapid))) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-	case IPA_IOC_NOTIFY_WAN_UPSTREAM_ROUTE_ADD:
-		retval = ipa3_send_wan_msg(arg, WAN_UPSTREAM_ROUTE_ADD, true);
-		if (retval) {
-			IPAERR("ipa3_send_wan_msg failed: %d\n", retval);
-			break;
-		}
-		break;
-	case IPA_IOC_NOTIFY_WAN_UPSTREAM_ROUTE_DEL:
-		retval = ipa3_send_wan_msg(arg, WAN_UPSTREAM_ROUTE_DEL, true);
-		if (retval) {
-			IPAERR("ipa3_send_wan_msg failed: %d\n", retval);
-			break;
-		}
-		break;
-	case IPA_IOC_NOTIFY_WAN_EMBMS_CONNECTED:
-		retval = ipa3_send_wan_msg(arg, WAN_EMBMS_CONNECT, false);
-		if (retval) {
-			IPAERR("ipa3_send_wan_msg failed: %d\n", retval);
-			break;
-		}
-		break;
-	case IPA_IOC_ADD_HDR_PROC_CTX:
-		if (copy_from_user(header, (const void __user *)arg,
-			sizeof(struct ipa_ioc_add_hdr_proc_ctx))) {
-			retval = -EFAULT;
-			break;
-		}
-		pre_entry =
-			((struct ipa_ioc_add_hdr_proc_ctx *)
-			header)->num_proc_ctxs;
-		pyld_sz =
-		   sizeof(struct ipa_ioc_add_hdr_proc_ctx) +
-		   pre_entry * sizeof(struct ipa_hdr_proc_ctx_add);
-		param = memdup_user((const void __user *)arg, pyld_sz);
-		if (IS_ERR(param)) {
-			retval = PTR_ERR(param);
-			break;
-		}
-		/* add check in case user-space module compromised */
-		if (unlikely(((struct ipa_ioc_add_hdr_proc_ctx *)
-			param)->num_proc_ctxs != pre_entry)) {
-			IPAERR_RL("current %d pre %d\n",
-				((struct ipa_ioc_add_hdr_proc_ctx *)
-				param)->num_proc_ctxs, pre_entry);
-			retval = -EFAULT;
-			break;
-		}
-		if (ipa3_add_hdr_proc_ctx(
-			(struct ipa_ioc_add_hdr_proc_ctx *)param, true)) {
-			retval = -EFAULT;
-			break;
-		}
-		if (copy_to_user((void __user *)arg, param, pyld_sz)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-	case IPA_IOC_DEL_HDR_PROC_CTX:
-		if (copy_from_user(header, (const void __user *)arg,
-			sizeof(struct ipa_ioc_del_hdr_proc_ctx))) {
-			retval = -EFAULT;
-			break;
-		}
-		pre_entry =
-			((struct ipa_ioc_del_hdr_proc_ctx *)header)->num_hdls;
-		pyld_sz =
-		   sizeof(struct ipa_ioc_del_hdr_proc_ctx) +
-		   pre_entry * sizeof(struct ipa_hdr_proc_ctx_del);
-		param = memdup_user((const void __user *)arg, pyld_sz);
-		if (IS_ERR(param)) {
-			retval = PTR_ERR(param);
-			break;
-		}
-		/* add check in case user-space module compromised */
-		if (unlikely(((struct ipa_ioc_del_hdr_proc_ctx *)
-			param)->num_hdls != pre_entry)) {
-			IPAERR_RL("current %d pre %d\n",
-				((struct ipa_ioc_del_hdr_proc_ctx *)param)->
-				num_hdls,
-				pre_entry);
-			retval = -EFAULT;
-			break;
-		}
-		if (ipa3_del_hdr_proc_ctx_by_user(
-			(struct ipa_ioc_del_hdr_proc_ctx *)param, true)) {
-			retval = -EFAULT;
-			break;
-		}
-		if (copy_to_user((void __user *)arg, param, pyld_sz)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-
-	case IPA_IOC_GET_HW_VERSION:
-		pyld_sz = sizeof(enum ipa_hw_type);
-		param = kmemdup(&ipa3_ctx->ipa_hw_type, pyld_sz, GFP_KERNEL);
-		if (!param) {
-			retval = -ENOMEM;
-			break;
-		}
-		if (copy_to_user((void __user *)arg, param, pyld_sz)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-
-	case IPA_IOC_GET_VLAN_MODE:
-		if (copy_from_user(&vlan_mode, (const void __user *)arg,
-			sizeof(struct ipa_ioc_get_vlan_mode))) {
-			retval = -EFAULT;
-			break;
-		}
-		retval = ipa3_is_vlan_mode(
-			vlan_mode.iface,
-			&is_vlan_mode);
-		if (retval)
-			break;
-
-		vlan_mode.is_vlan_mode = is_vlan_mode;
-
-		if (copy_to_user((void __user *)arg,
-			&vlan_mode,
-			sizeof(struct ipa_ioc_get_vlan_mode))) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-
-	case IPA_IOC_ADD_VLAN_IFACE:
-		if (ipa3_send_vlan_l2tp_msg(arg, ADD_VLAN_IFACE)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-
-	case IPA_IOC_DEL_VLAN_IFACE:
-		if (ipa3_send_vlan_l2tp_msg(arg, DEL_VLAN_IFACE)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-	case IPA_IOC_ADD_BRIDGE_VLAN_MAPPING:
-		if (ipa3_send_vlan_l2tp_msg(arg, ADD_BRIDGE_VLAN_MAPPING)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-	case IPA_IOC_DEL_BRIDGE_VLAN_MAPPING:
-		if (ipa3_send_vlan_l2tp_msg(arg, DEL_BRIDGE_VLAN_MAPPING)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-	case IPA_IOC_ADD_L2TP_VLAN_MAPPING:
-		if (ipa3_send_vlan_l2tp_msg(arg, ADD_L2TP_VLAN_MAPPING)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-
-	case IPA_IOC_DEL_L2TP_VLAN_MAPPING:
-		if (ipa3_send_vlan_l2tp_msg(arg, DEL_L2TP_VLAN_MAPPING)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-
-	case IPA_IOC_CLEANUP:
-		/*Route and filter rules will also be clean*/
-		IPADBG("Got IPA_IOC_CLEANUP\n");
-		retval = ipa3_reset_hdr(true);
-		memset(&nat_del, 0, sizeof(nat_del));
-		nat_del.table_index = 0;
-		retval = ipa3_nat_del_cmd(&nat_del);
-		retval = ipa3_clean_modem_rule();
-		ipa3_counter_id_remove_all();
-		break;
-
-	case IPA_IOC_QUERY_WLAN_CLIENT:
-		IPADBG("Got IPA_IOC_QUERY_WLAN_CLIENT\n");
-		retval = ipa3_resend_wlan_msg();
-		break;
-
-	case IPA_IOC_GSB_CONNECT:
-		IPADBG("Got IPA_IOC_GSB_CONNECT\n");
-		if (ipa3_send_gsb_msg(arg, IPA_GSB_CONNECT)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-
-	case IPA_IOC_GSB_DISCONNECT:
-		IPADBG("Got IPA_IOC_GSB_DISCONNECT\n");
-		if (ipa3_send_gsb_msg(arg, IPA_GSB_DISCONNECT)) {
-			retval = -EFAULT;
-			break;
-		}
-		break;
-
-	case IPA_IOC_ADD_RT_RULE_V2:
-		retval = ipa3_ioctl_add_rt_rule_v2(arg);
-		break;
-
-	case IPA_IOC_ADD_RT_RULE_EXT_V2:
-		retval = ipa3_ioctl_add_rt_rule_ext_v2(arg);
-		break;
-
-	case IPA_IOC_ADD_RT_RULE_AFTER_V2:
-		retval = ipa3_ioctl_add_rt_rule_after_v2(arg);
-		break;
-
-	case IPA_IOC_MDFY_RT_RULE_V2:
-		retval = ipa3_ioctl_mdfy_rt_rule_v2(arg);
-		break;
-
-	case IPA_IOC_ADD_FLT_RULE_V2:
-		retval = ipa3_ioctl_add_flt_rule_v2(arg);
-		break;
-
-	case IPA_IOC_ADD_FLT_RULE_AFTER_V2:
-		retval = ipa3_ioctl_add_flt_rule_after_v2(arg);
-		break;
-
-	case IPA_IOC_MDFY_FLT_RULE_V2:
-		retval = ipa3_ioctl_mdfy_flt_rule_v2(arg);
-		break;
-
-	case IPA_IOC_FNR_COUNTER_ALLOC:
-		retval = ipa3_ioctl_fnr_counter_alloc(arg);
-		break;
-
-	case IPA_IOC_FNR_COUNTER_DEALLOC:
-		hdl = (int)arg;
-		if (hdl < 0) {
-			IPAERR("IPA_FNR_COUNTER_DEALLOC failed: hdl %d\n",
-				hdl);
-			retval = -EPERM;
-			break;
-		}
-		ipa3_counter_remove_hdl(hdl);
-		break;
-
-	case IPA_IOC_FNR_COUNTER_QUERY:
-		retval = ipa3_ioctl_fnr_counter_query(arg);
-		break;
-
-	case IPA_IOC_WIGIG_FST_SWITCH:
-		IPADBG("Got IPA_IOCTL_WIGIG_FST_SWITCH\n");
-		if (copy_from_user(&fst_switch, (const void __user *)arg,
-			sizeof(struct ipa_ioc_wigig_fst_switch))) {
-			retval = -EFAULT;
-			break;
-		}
-		retval = ipa_wigig_send_msg(WIGIG_FST_SWITCH,
-			fst_switch.netdev_name,
-			fst_switch.client_mac_addr,
-			IPA_CLIENT_MAX,
-			fst_switch.to_wigig);
-		break;
-	default:
-		IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-		return -ENOTTY;
-	}
-	if (!IS_ERR(param))
-		kfree(param);
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-
-	return retval;
-}
-
-/**
- * ipa3_setup_dflt_rt_tables() - Setup default routing tables
- *
- * Return codes:
- * 0: success
- * -ENOMEM: failed to allocate memory
- * -EPERM: failed to add the tables
- */
-int ipa3_setup_dflt_rt_tables(void)
-{
-	struct ipa_ioc_add_rt_rule *rt_rule;
-	struct ipa_rt_rule_add *rt_rule_entry;
-
-	rt_rule =
-		kzalloc(sizeof(struct ipa_ioc_add_rt_rule) + 1 *
-			sizeof(struct ipa_rt_rule_add), GFP_KERNEL);
-	if (!rt_rule)
-		return -ENOMEM;
-
-	/* setup a default v4 route to point to Apps */
-	rt_rule->num_rules = 1;
-	rt_rule->commit = 1;
-	rt_rule->ip = IPA_IP_v4;
-	strlcpy(rt_rule->rt_tbl_name, IPA_DFLT_RT_TBL_NAME,
-		IPA_RESOURCE_NAME_MAX);
-
-	rt_rule_entry = &rt_rule->rules[0];
-	rt_rule_entry->at_rear = 1;
-	rt_rule_entry->rule.dst = IPA_CLIENT_APPS_LAN_CONS;
-	rt_rule_entry->rule.hdr_hdl = ipa3_ctx->excp_hdr_hdl;
-	rt_rule_entry->rule.retain_hdr = 1;
-
-	if (ipa3_add_rt_rule(rt_rule)) {
-		IPAERR("fail to add dflt v4 rule\n");
-		kfree(rt_rule);
-		return -EPERM;
-	}
-	IPADBG("dflt v4 rt rule hdl=%x\n", rt_rule_entry->rt_rule_hdl);
-	ipa3_ctx->dflt_v4_rt_rule_hdl = rt_rule_entry->rt_rule_hdl;
-
-	/* setup a default v6 route to point to A5 */
-	rt_rule->ip = IPA_IP_v6;
-	if (ipa3_add_rt_rule(rt_rule)) {
-		IPAERR("fail to add dflt v6 rule\n");
-		kfree(rt_rule);
-		return -EPERM;
-	}
-	IPADBG("dflt v6 rt rule hdl=%x\n", rt_rule_entry->rt_rule_hdl);
-	ipa3_ctx->dflt_v6_rt_rule_hdl = rt_rule_entry->rt_rule_hdl;
-
-	/*
-	 * because these tables are the very first to be added, they will both
-	 * have the same index (0) which is essential for programming the
-	 * "route" end-point config
-	 */
-
-	kfree(rt_rule);
-
-	return 0;
-}
-
-static int ipa3_setup_exception_path(void)
-{
-	struct ipa_ioc_add_hdr *hdr;
-	struct ipa_hdr_add *hdr_entry;
-	struct ipahal_reg_route route = { 0 };
-	int ret;
-
-	/* install the basic exception header */
-	hdr = kzalloc(sizeof(struct ipa_ioc_add_hdr) + 1 *
-		      sizeof(struct ipa_hdr_add), GFP_KERNEL);
-	if (!hdr)
-		return -ENOMEM;
-
-	hdr->num_hdrs = 1;
-	hdr->commit = 1;
-	hdr_entry = &hdr->hdr[0];
-
-	strlcpy(hdr_entry->name, IPA_LAN_RX_HDR_NAME, IPA_RESOURCE_NAME_MAX);
-	hdr_entry->hdr_len = IPA_LAN_RX_HEADER_LENGTH;
-
-	if (ipa3_add_hdr(hdr)) {
-		IPAERR("fail to add exception hdr\n");
-		ret = -EPERM;
-		goto bail;
-	}
-
-	if (hdr_entry->status) {
-		IPAERR("fail to add exception hdr\n");
-		ret = -EPERM;
-		goto bail;
-	}
-
-	ipa3_ctx->excp_hdr_hdl = hdr_entry->hdr_hdl;
-
-	/* set the route register to pass exception packets to Apps */
-	route.route_def_pipe = ipa3_get_ep_mapping(IPA_CLIENT_APPS_LAN_CONS);
-	route.route_frag_def_pipe = ipa3_get_ep_mapping(
-		IPA_CLIENT_APPS_LAN_CONS);
-	route.route_def_hdr_table = !ipa3_ctx->hdr_tbl_lcl;
-	route.route_def_retain_hdr = 1;
-
-	if (ipa3_cfg_route(&route)) {
-		IPAERR("fail to add exception hdr\n");
-		ret = -EPERM;
-		goto bail;
-	}
-
-	ret = 0;
-bail:
-	kfree(hdr);
-	return ret;
-}
-
-static int ipa3_init_smem_region(int memory_region_size,
-				int memory_region_offset)
-{
-	struct ipahal_imm_cmd_dma_shared_mem cmd;
-	struct ipahal_imm_cmd_pyld *cmd_pyld;
-	struct ipa3_desc desc;
-	struct ipa_mem_buffer mem;
-	int rc;
-
-	if (memory_region_size == 0)
-		return 0;
-
-	memset(&desc, 0, sizeof(desc));
-	memset(&cmd, 0, sizeof(cmd));
-	memset(&mem, 0, sizeof(mem));
-
-	mem.size = memory_region_size;
-	mem.base = dma_zalloc_coherent(ipa3_ctx->pdev, mem.size,
-		&mem.phys_base, GFP_KERNEL);
-	if (!mem.base) {
-		IPAERR("failed to alloc DMA buff of size %d\n", mem.size);
-		return -ENOMEM;
-	}
-
-	cmd.is_read = false;
-	cmd.skip_pipeline_clear = false;
-	cmd.pipeline_clear_options = IPAHAL_HPS_CLEAR;
-	cmd.size = mem.size;
-	cmd.system_addr = mem.phys_base;
-	cmd.local_addr = ipa3_ctx->smem_restricted_bytes +
-		memory_region_offset;
-	cmd_pyld = ipahal_construct_imm_cmd(
-		IPA_IMM_CMD_DMA_SHARED_MEM, &cmd, false);
-	if (!cmd_pyld) {
-		IPAERR("failed to construct dma_shared_mem imm cmd\n");
-		return -ENOMEM;
-	}
-	ipa3_init_imm_cmd_desc(&desc, cmd_pyld);
-
-	rc = ipa3_send_cmd(1, &desc);
-	if (rc) {
-		IPAERR("failed to send immediate command (error %d)\n", rc);
-		rc = -EFAULT;
-	}
-
-	ipahal_destroy_imm_cmd(cmd_pyld);
-	dma_free_coherent(ipa3_ctx->pdev, mem.size, mem.base,
-		mem.phys_base);
-
-	return rc;
-}
-
-/**
- * ipa3_init_q6_smem() - Initialize Q6 general memory and
- *                      header memory regions in IPA.
- *
- * Return codes:
- * 0: success
- * -ENOMEM: failed to allocate dma memory
- * -EFAULT: failed to send IPA command to initialize the memory
- */
-int ipa3_init_q6_smem(void)
-{
-	int rc;
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	rc = ipa3_init_smem_region(IPA_MEM_PART(modem_size),
-		IPA_MEM_PART(modem_ofst));
-	if (rc) {
-		IPAERR("failed to initialize Modem RAM memory\n");
-		IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-		return rc;
-	}
-
-	rc = ipa3_init_smem_region(IPA_MEM_PART(modem_hdr_size),
-		IPA_MEM_PART(modem_hdr_ofst));
-	if (rc) {
-		IPAERR("failed to initialize Modem HDRs RAM memory\n");
-		IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-		return rc;
-	}
-
-	rc = ipa3_init_smem_region(IPA_MEM_PART(modem_hdr_proc_ctx_size),
-		IPA_MEM_PART(modem_hdr_proc_ctx_ofst));
-	if (rc) {
-		IPAERR("failed to initialize Modem proc ctx RAM memory\n");
-		IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-		return rc;
-	}
-
-	rc = ipa3_init_smem_region(IPA_MEM_PART(modem_comp_decomp_size),
-		IPA_MEM_PART(modem_comp_decomp_ofst));
-	if (rc) {
-		IPAERR("failed to initialize Modem Comp/Decomp RAM memory\n");
-		IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-		return rc;
-	}
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-
-	return rc;
-}
-
-static void ipa3_destroy_imm(void *user1, int user2)
-{
-	ipahal_destroy_imm_cmd(user1);
-}
-
-static void ipa3_q6_pipe_delay(bool delay)
-{
-	int client_idx;
-	int ep_idx;
-	struct ipa_ep_cfg_ctrl ep_ctrl;
-
-	memset(&ep_ctrl, 0, sizeof(struct ipa_ep_cfg_ctrl));
-	ep_ctrl.ipa_ep_delay = delay;
-
-	for (client_idx = 0; client_idx < IPA_CLIENT_MAX; client_idx++) {
-		if (IPA_CLIENT_IS_Q6_PROD(client_idx)) {
-			ep_idx = ipa3_get_ep_mapping(client_idx);
-			if (ep_idx == -1)
-				continue;
-
-			ipahal_write_reg_n_fields(IPA_ENDP_INIT_CTRL_n,
-				ep_idx, &ep_ctrl);
-		}
-	}
-}
-
-static void ipa3_q6_avoid_holb(void)
-{
-	int ep_idx;
-	int client_idx;
-	struct ipa_ep_cfg_ctrl ep_suspend;
-	struct ipa_ep_cfg_holb ep_holb;
-
-	memset(&ep_suspend, 0, sizeof(ep_suspend));
-	memset(&ep_holb, 0, sizeof(ep_holb));
-
-	ep_suspend.ipa_ep_suspend = true;
-	ep_holb.tmr_val = 0;
-	ep_holb.en = 1;
-
-	if (ipa3_ctx->ipa_hw_type == IPA_HW_v4_2)
-		ipa3_cal_ep_holb_scale_base_val(ep_holb.tmr_val, &ep_holb);
-
-	for (client_idx = 0; client_idx < IPA_CLIENT_MAX; client_idx++) {
-		if (IPA_CLIENT_IS_Q6_CONS(client_idx)) {
-			ep_idx = ipa3_get_ep_mapping(client_idx);
-			if (ep_idx == -1)
-				continue;
-
-			/* from IPA 4.0 pipe suspend is not supported */
-			if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_0)
-				ipahal_write_reg_n_fields(
-				IPA_ENDP_INIT_CTRL_n,
-				ep_idx, &ep_suspend);
-
-			/*
-			 * ipa3_cfg_ep_holb is not used here because we are
-			 * setting HOLB on Q6 pipes, and from APPS perspective
-			 * they are not valid, therefore, the above function
-			 * will fail.
-			 */
-			ipahal_write_reg_n_fields(
-				IPA_ENDP_INIT_HOL_BLOCK_TIMER_n,
-				ep_idx, &ep_holb);
-			ipahal_write_reg_n_fields(
-				IPA_ENDP_INIT_HOL_BLOCK_EN_n,
-				ep_idx, &ep_holb);
-
-			/* IPA4.5 issue requires HOLB_EN to be written twice */
-			if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_5)
-				ipahal_write_reg_n_fields(
-					IPA_ENDP_INIT_HOL_BLOCK_EN_n,
-					ep_idx, &ep_holb);
-		}
-	}
-}
-
-static void ipa3_halt_q6_gsi_channels(bool prod)
-{
-	int ep_idx;
-	int client_idx;
-	const struct ipa_gsi_ep_config *gsi_ep_cfg;
-	int i;
-	int ret;
-	int code = 0;
-
-	/* if prod flag is true, then we halt the producer channels also */
-	for (client_idx = 0; client_idx < IPA_CLIENT_MAX; client_idx++) {
-		if (IPA_CLIENT_IS_Q6_CONS(client_idx)
-			|| (IPA_CLIENT_IS_Q6_PROD(client_idx) && prod)) {
-			ep_idx = ipa3_get_ep_mapping(client_idx);
-			if (ep_idx == -1)
-				continue;
-
-			gsi_ep_cfg = ipa3_get_gsi_ep_info(client_idx);
-			if (!gsi_ep_cfg) {
-				IPAERR("failed to get GSI config\n");
-				ipa_assert();
-				return;
-			}
-
-			ret = gsi_halt_channel_ee(
-				gsi_ep_cfg->ipa_gsi_chan_num, gsi_ep_cfg->ee,
-				&code);
-			for (i = 0; i < IPA_GSI_CHANNEL_STOP_MAX_RETRY &&
-				ret == -GSI_STATUS_AGAIN; i++) {
-				IPADBG(
-				"ch %d ee %d with code %d\n is busy try again",
-					gsi_ep_cfg->ipa_gsi_chan_num,
-					gsi_ep_cfg->ee,
-					code);
-				usleep_range(IPA_GSI_CHANNEL_HALT_MIN_SLEEP,
-					IPA_GSI_CHANNEL_HALT_MAX_SLEEP);
-				ret = gsi_halt_channel_ee(
-					gsi_ep_cfg->ipa_gsi_chan_num,
-					gsi_ep_cfg->ee, &code);
-			}
-			if (ret == GSI_STATUS_SUCCESS)
-				IPADBG("halted gsi ch %d ee %d with code %d\n",
-				gsi_ep_cfg->ipa_gsi_chan_num,
-				gsi_ep_cfg->ee,
-				code);
-			else
-				IPAERR("failed to halt ch %d ee %d code %d\n",
-				gsi_ep_cfg->ipa_gsi_chan_num,
-				gsi_ep_cfg->ee,
-				code);
-		}
-	}
-}
-
-static int ipa3_q6_clean_q6_flt_tbls(enum ipa_ip_type ip,
-	enum ipa_rule_type rlt)
-{
-	struct ipa3_desc *desc;
-	struct ipahal_imm_cmd_dma_shared_mem cmd = {0};
-	struct ipahal_imm_cmd_pyld **cmd_pyld;
-	int retval = 0;
-	int pipe_idx;
-	int flt_idx = 0;
-	int num_cmds = 0;
-	int index;
-	u32 lcl_addr_mem_part;
-	u32 lcl_hdr_sz;
-	struct ipa_mem_buffer mem;
-
-	IPADBG("Entry\n");
-
-	if ((ip >= IPA_IP_MAX) || (rlt >= IPA_RULE_TYPE_MAX)) {
-		IPAERR("Input Err: ip=%d ; rlt=%d\n", ip, rlt);
-		return -EINVAL;
-	}
-
-	/*
-	 * SRAM memory not allocated to hash tables. Cleaning the of hash table
-	 * operation not supported.
-	 */
-	if (rlt == IPA_RULE_HASHABLE && ipa3_ctx->ipa_fltrt_not_hashable) {
-		IPADBG("Clean hashable rules not supported\n");
-		return retval;
-	}
-
-	/* Up to filtering pipes we have filtering tables */
-	desc = kcalloc(ipa3_ctx->ep_flt_num, sizeof(struct ipa3_desc),
-		GFP_KERNEL);
-	if (!desc)
-		return -ENOMEM;
-
-	cmd_pyld = kcalloc(ipa3_ctx->ep_flt_num,
-		sizeof(struct ipahal_imm_cmd_pyld *), GFP_KERNEL);
-	if (!cmd_pyld) {
-		retval = -ENOMEM;
-		goto free_desc;
-	}
-
-	if (ip == IPA_IP_v4) {
-		if (rlt == IPA_RULE_HASHABLE) {
-			lcl_addr_mem_part = IPA_MEM_PART(v4_flt_hash_ofst);
-			lcl_hdr_sz = IPA_MEM_PART(v4_flt_hash_size);
-		} else {
-			lcl_addr_mem_part = IPA_MEM_PART(v4_flt_nhash_ofst);
-			lcl_hdr_sz = IPA_MEM_PART(v4_flt_nhash_size);
-		}
-	} else {
-		if (rlt == IPA_RULE_HASHABLE) {
-			lcl_addr_mem_part = IPA_MEM_PART(v6_flt_hash_ofst);
-			lcl_hdr_sz = IPA_MEM_PART(v6_flt_hash_size);
-		} else {
-			lcl_addr_mem_part = IPA_MEM_PART(v6_flt_nhash_ofst);
-			lcl_hdr_sz = IPA_MEM_PART(v6_flt_nhash_size);
-		}
-	}
-
-	retval = ipahal_flt_generate_empty_img(1, lcl_hdr_sz, lcl_hdr_sz,
-		0, &mem, true);
-	if (retval) {
-		IPAERR("failed to generate flt single tbl empty img\n");
-		goto free_cmd_pyld;
-	}
-
-	for (pipe_idx = 0; pipe_idx < ipa3_ctx->ipa_num_pipes; pipe_idx++) {
-		if (!ipa_is_ep_support_flt(pipe_idx))
-			continue;
-
-		/*
-		 * Iterating over all the filtering pipes which are either
-		 * invalid but connected or connected but not configured by AP.
-		 */
-		if (!ipa3_ctx->ep[pipe_idx].valid ||
-		    ipa3_ctx->ep[pipe_idx].skip_ep_cfg) {
-
-			if (num_cmds >= ipa3_ctx->ep_flt_num) {
-				IPAERR("number of commands is out of range\n");
-				retval = -ENOBUFS;
-				goto free_empty_img;
-			}
-
-			cmd.is_read = false;
-			cmd.skip_pipeline_clear = false;
-			cmd.pipeline_clear_options = IPAHAL_HPS_CLEAR;
-			cmd.size = mem.size;
-			cmd.system_addr = mem.phys_base;
-			cmd.local_addr =
-				ipa3_ctx->smem_restricted_bytes +
-				lcl_addr_mem_part +
-				ipahal_get_hw_tbl_hdr_width() +
-				flt_idx * ipahal_get_hw_tbl_hdr_width();
-			cmd_pyld[num_cmds] = ipahal_construct_imm_cmd(
-				IPA_IMM_CMD_DMA_SHARED_MEM, &cmd, false);
-			if (!cmd_pyld[num_cmds]) {
-				IPAERR("fail construct dma_shared_mem cmd\n");
-				retval = -ENOMEM;
-				goto free_empty_img;
-			}
-			ipa3_init_imm_cmd_desc(&desc[num_cmds],
-				cmd_pyld[num_cmds]);
-			++num_cmds;
-		}
-
-		++flt_idx;
-	}
-
-	IPADBG("Sending %d descriptors for flt tbl clearing\n", num_cmds);
-	retval = ipa3_send_cmd(num_cmds, desc);
-	if (retval) {
-		IPAERR("failed to send immediate command (err %d)\n", retval);
-		retval = -EFAULT;
-	}
-
-free_empty_img:
-	ipahal_free_dma_mem(&mem);
-free_cmd_pyld:
-	for (index = 0; index < num_cmds; index++)
-		ipahal_destroy_imm_cmd(cmd_pyld[index]);
-	kfree(cmd_pyld);
-free_desc:
-	kfree(desc);
-	return retval;
-}
-
-static int ipa3_q6_clean_q6_rt_tbls(enum ipa_ip_type ip,
-	enum ipa_rule_type rlt)
-{
-	struct ipa3_desc *desc;
-	struct ipahal_imm_cmd_dma_shared_mem cmd = {0};
-	struct ipahal_imm_cmd_pyld *cmd_pyld = NULL;
-	int retval = 0;
-	u32 modem_rt_index_lo;
-	u32 modem_rt_index_hi;
-	u32 lcl_addr_mem_part;
-	u32 lcl_hdr_sz;
-	struct ipa_mem_buffer mem;
-
-	IPADBG("Entry\n");
-
-	if ((ip >= IPA_IP_MAX) || (rlt >= IPA_RULE_TYPE_MAX)) {
-		IPAERR("Input Err: ip=%d ; rlt=%d\n", ip, rlt);
-		return -EINVAL;
-	}
-
-	/*
-	 * SRAM memory not allocated to hash tables. Cleaning the of hash table
-	 * operation not supported.
-	 */
-	if (rlt == IPA_RULE_HASHABLE && ipa3_ctx->ipa_fltrt_not_hashable) {
-		IPADBG("Clean hashable rules not supported\n");
-		return retval;
-	}
-
-	if (ip == IPA_IP_v4) {
-		modem_rt_index_lo = IPA_MEM_PART(v4_modem_rt_index_lo);
-		modem_rt_index_hi = IPA_MEM_PART(v4_modem_rt_index_hi);
-		if (rlt == IPA_RULE_HASHABLE) {
-			lcl_addr_mem_part = IPA_MEM_PART(v4_rt_hash_ofst);
-			lcl_hdr_sz =  IPA_MEM_PART(v4_flt_hash_size);
-		} else {
-			lcl_addr_mem_part = IPA_MEM_PART(v4_rt_nhash_ofst);
-			lcl_hdr_sz = IPA_MEM_PART(v4_flt_nhash_size);
-		}
-	} else {
-		modem_rt_index_lo = IPA_MEM_PART(v6_modem_rt_index_lo);
-		modem_rt_index_hi = IPA_MEM_PART(v6_modem_rt_index_hi);
-		if (rlt == IPA_RULE_HASHABLE) {
-			lcl_addr_mem_part = IPA_MEM_PART(v6_rt_hash_ofst);
-			lcl_hdr_sz =  IPA_MEM_PART(v6_flt_hash_size);
-		} else {
-			lcl_addr_mem_part = IPA_MEM_PART(v6_rt_nhash_ofst);
-			lcl_hdr_sz = IPA_MEM_PART(v6_flt_nhash_size);
-		}
-	}
-
-	retval = ipahal_rt_generate_empty_img(
-		modem_rt_index_hi - modem_rt_index_lo + 1,
-		lcl_hdr_sz, lcl_hdr_sz, &mem, true);
-	if (retval) {
-		IPAERR("fail generate empty rt img\n");
-		return -ENOMEM;
-	}
-
-	desc = kzalloc(sizeof(struct ipa3_desc), GFP_KERNEL);
-	if (!desc) {
-		IPAERR("failed to allocate memory\n");
-		goto free_empty_img;
-	}
-
-	cmd.is_read = false;
-	cmd.skip_pipeline_clear = false;
-	cmd.pipeline_clear_options = IPAHAL_HPS_CLEAR;
-	cmd.size = mem.size;
-	cmd.system_addr =  mem.phys_base;
-	cmd.local_addr = ipa3_ctx->smem_restricted_bytes +
-		lcl_addr_mem_part +
-		modem_rt_index_lo * ipahal_get_hw_tbl_hdr_width();
-	cmd_pyld = ipahal_construct_imm_cmd(
-			IPA_IMM_CMD_DMA_SHARED_MEM, &cmd, false);
-	if (!cmd_pyld) {
-		IPAERR("failed to construct dma_shared_mem imm cmd\n");
-		retval = -ENOMEM;
-		goto free_desc;
-	}
-	ipa3_init_imm_cmd_desc(desc, cmd_pyld);
-
-	IPADBG("Sending 1 descriptor for rt tbl clearing\n");
-	retval = ipa3_send_cmd(1, desc);
-	if (retval) {
-		IPAERR("failed to send immediate command (err %d)\n", retval);
-		retval = -EFAULT;
-	}
-
-	ipahal_destroy_imm_cmd(cmd_pyld);
-free_desc:
-	kfree(desc);
-free_empty_img:
-	ipahal_free_dma_mem(&mem);
-	return retval;
-}
-
-static int ipa3_q6_clean_q6_tables(void)
-{
-	struct ipa3_desc *desc;
-	struct ipahal_imm_cmd_pyld *cmd_pyld = NULL;
-	struct ipahal_imm_cmd_register_write reg_write_cmd = {0};
-	int retval = 0;
-	struct ipahal_reg_fltrt_hash_flush flush;
-	struct ipahal_reg_valmask valmask;
-
-	IPADBG("Entry\n");
-
-
-	if (ipa3_q6_clean_q6_flt_tbls(IPA_IP_v4, IPA_RULE_HASHABLE)) {
-		IPAERR("failed to clean q6 flt tbls (v4/hashable)\n");
-		return -EFAULT;
-	}
-	if (ipa3_q6_clean_q6_flt_tbls(IPA_IP_v6, IPA_RULE_HASHABLE)) {
-		IPAERR("failed to clean q6 flt tbls (v6/hashable)\n");
-		return -EFAULT;
-	}
-	if (ipa3_q6_clean_q6_flt_tbls(IPA_IP_v4, IPA_RULE_NON_HASHABLE)) {
-		IPAERR("failed to clean q6 flt tbls (v4/non-hashable)\n");
-		return -EFAULT;
-	}
-	if (ipa3_q6_clean_q6_flt_tbls(IPA_IP_v6, IPA_RULE_NON_HASHABLE)) {
-		IPAERR("failed to clean q6 flt tbls (v6/non-hashable)\n");
-		return -EFAULT;
-	}
-
-	if (ipa3_q6_clean_q6_rt_tbls(IPA_IP_v4, IPA_RULE_HASHABLE)) {
-		IPAERR("failed to clean q6 rt tbls (v4/hashable)\n");
-		return -EFAULT;
-	}
-	if (ipa3_q6_clean_q6_rt_tbls(IPA_IP_v6, IPA_RULE_HASHABLE)) {
-		IPAERR("failed to clean q6 rt tbls (v6/hashable)\n");
-		return -EFAULT;
-	}
-	if (ipa3_q6_clean_q6_rt_tbls(IPA_IP_v4, IPA_RULE_NON_HASHABLE)) {
-		IPAERR("failed to clean q6 rt tbls (v4/non-hashable)\n");
-		return -EFAULT;
-	}
-	if (ipa3_q6_clean_q6_rt_tbls(IPA_IP_v6, IPA_RULE_NON_HASHABLE)) {
-		IPAERR("failed to clean q6 rt tbls (v6/non-hashable)\n");
-		return -EFAULT;
-	}
-
-	/*
-	 * SRAM memory not allocated to hash tables. Cleaning the of hash table
-	 * operation not supported.
-	 */
-	if (ipa3_ctx->ipa_fltrt_not_hashable)
-		return retval;
-	/* Flush rules cache */
-	desc = kzalloc(sizeof(struct ipa3_desc), GFP_KERNEL);
-	if (!desc)
-		return -ENOMEM;
-
-	flush.v4_flt = true;
-	flush.v4_rt = true;
-	flush.v6_flt = true;
-	flush.v6_rt = true;
-	ipahal_get_fltrt_hash_flush_valmask(&flush, &valmask);
-	reg_write_cmd.skip_pipeline_clear = false;
-	reg_write_cmd.pipeline_clear_options = IPAHAL_HPS_CLEAR;
-	reg_write_cmd.offset = ipahal_get_reg_ofst(IPA_FILT_ROUT_HASH_FLUSH);
-	reg_write_cmd.value = valmask.val;
-	reg_write_cmd.value_mask = valmask.mask;
-	cmd_pyld = ipahal_construct_imm_cmd(IPA_IMM_CMD_REGISTER_WRITE,
-		&reg_write_cmd, false);
-	if (!cmd_pyld) {
-		IPAERR("fail construct register_write imm cmd\n");
-		retval = -EFAULT;
-		goto bail_desc;
-	}
-	ipa3_init_imm_cmd_desc(desc, cmd_pyld);
-
-	IPADBG("Sending 1 descriptor for tbls flush\n");
-	retval = ipa3_send_cmd(1, desc);
-	if (retval) {
-		IPAERR("failed to send immediate command (err %d)\n", retval);
-		retval = -EFAULT;
-	}
-
-	ipahal_destroy_imm_cmd(cmd_pyld);
-
-bail_desc:
-	kfree(desc);
-	IPADBG("Done - retval = %d\n", retval);
-	return retval;
-}
-
-static int ipa3_q6_set_ex_path_to_apps(void)
-{
-	int ep_idx;
-	int client_idx;
-	struct ipa3_desc *desc;
-	int num_descs = 0;
-	int index;
-	struct ipahal_imm_cmd_register_write reg_write;
-	struct ipahal_imm_cmd_pyld *cmd_pyld;
-	int retval;
-
-	desc = kcalloc(ipa3_ctx->ipa_num_pipes, sizeof(struct ipa3_desc),
-			GFP_KERNEL);
-	if (!desc)
-		return -ENOMEM;
-
-	/* Set the exception path to AP */
-	for (client_idx = 0; client_idx < IPA_CLIENT_MAX; client_idx++) {
-		ep_idx = ipa3_get_ep_mapping(client_idx);
-		if (ep_idx == -1)
-			continue;
-
-		/* disable statuses for all modem controlled prod pipes */
-		if (!IPA_CLIENT_IS_TEST(client_idx) &&
-			(IPA_CLIENT_IS_Q6_PROD(client_idx) ||
-			(IPA_CLIENT_IS_PROD(client_idx) &&
-			ipa3_ctx->ep[ep_idx].valid &&
-			ipa3_ctx->ep[ep_idx].skip_ep_cfg) ||
-			(ipa3_ctx->ep[ep_idx].client == IPA_CLIENT_APPS_WAN_PROD
-			&& ipa3_ctx->modem_cfg_emb_pipe_flt))) {
-			ipa_assert_on(num_descs >= ipa3_ctx->ipa_num_pipes);
-
-			ipa3_ctx->ep[ep_idx].status.status_en = false;
-			reg_write.skip_pipeline_clear = false;
-			reg_write.pipeline_clear_options =
-				IPAHAL_HPS_CLEAR;
-			reg_write.offset =
-				ipahal_get_reg_n_ofst(IPA_ENDP_STATUS_n,
-					ep_idx);
-			reg_write.value = 0;
-			reg_write.value_mask = ~0;
-			cmd_pyld = ipahal_construct_imm_cmd(
-				IPA_IMM_CMD_REGISTER_WRITE, &reg_write, false);
-			if (!cmd_pyld) {
-				IPAERR("fail construct register_write cmd\n");
-				ipa_assert();
-				return -ENOMEM;
-			}
-
-			ipa3_init_imm_cmd_desc(&desc[num_descs], cmd_pyld);
-			desc[num_descs].callback = ipa3_destroy_imm;
-			desc[num_descs].user1 = cmd_pyld;
-			++num_descs;
-		}
-	}
-
-	/* Will wait 500msecs for IPA tag process completion */
-	retval = ipa3_tag_process(desc, num_descs,
-		msecs_to_jiffies(CLEANUP_TAG_PROCESS_TIMEOUT));
-	if (retval) {
-		IPAERR("TAG process failed! (error %d)\n", retval);
-		/* For timeout error ipa3_destroy_imm cb will destroy user1 */
-		if (retval != -ETIME) {
-			for (index = 0; index < num_descs; index++)
-				if (desc[index].callback)
-					desc[index].callback(desc[index].user1,
-						desc[index].user2);
-			retval = -EINVAL;
-		}
-	}
-
-	kfree(desc);
-
-	return retval;
-}
-
-/*
- * ipa3_update_ssr_state() - updating current SSR state
- * @is_ssr:	[in] Current SSR state
- */
-
-void ipa3_update_ssr_state(bool is_ssr)
-{
-	if (is_ssr)
-		atomic_set(&ipa3_ctx->is_ssr, 1);
-	else
-		atomic_set(&ipa3_ctx->is_ssr, 0);
-}
-
-/**
- * ipa3_q6_pre_shutdown_cleanup() - A cleanup for all Q6 related configuration
- *                    in IPA HW. This is performed in case of SSR.
- *
- * This is a mandatory procedure, in case one of the steps fails, the
- * AP needs to restart.
- */
-void ipa3_q6_pre_shutdown_cleanup(void)
-{
-	IPADBG_LOW("ENTER\n");
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	ipa3_update_ssr_state(true);
-	if (!ipa3_ctx->ipa_endp_delay_wa)
-		ipa3_q6_pipe_delay(true);
-	ipa3_q6_avoid_holb();
-	if (ipa3_ctx->ipa_config_is_mhi)
-		ipa3_set_reset_client_cons_pipe_sus_holb(true,
-		IPA_CLIENT_MHI_CONS);
-	if (ipa3_q6_clean_q6_tables()) {
-		IPAERR("Failed to clean Q6 tables\n");
-		/*
-		 * Indicates IPA hardware is stalled, unexpected
-		 * hardware state.
-		 */
-		ipa_assert();
-	}
-	if (ipa3_q6_set_ex_path_to_apps()) {
-		IPAERR("Failed to redirect exceptions to APPS\n");
-		/*
-		 * Indicates IPA hardware is stalled, unexpected
-		 * hardware state.
-		 */
-		ipa_assert();
-	}
-	/* Remove delay from Q6 PRODs to avoid pending descriptors
-	 * on pipe reset procedure
-	 */
-	if (!ipa3_ctx->ipa_endp_delay_wa) {
-		ipa3_q6_pipe_delay(false);
-		ipa3_set_reset_client_prod_pipe_delay(true,
-			IPA_CLIENT_USB_PROD);
-	} else {
-		ipa3_start_stop_client_prod_gsi_chnl(IPA_CLIENT_USB_PROD,
-						false);
-	}
-
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-	IPADBG_LOW("Exit with success\n");
-}
-
-/*
- * ipa3_q6_post_shutdown_cleanup() - As part of this cleanup
- * check if GSI channel related to Q6 producer client is empty.
- *
- * Q6 GSI channel emptiness is needed to garantee no descriptors with invalid
- *  info are injected into IPA RX from IPA_IF, while modem is restarting.
- */
-void ipa3_q6_post_shutdown_cleanup(void)
-{
-	int client_idx;
-	int ep_idx;
-	bool prod = false;
-
-	IPADBG_LOW("ENTER\n");
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	/* Handle the issue where SUSPEND was removed for some reason */
-	ipa3_q6_avoid_holb();
-
-	/* halt both prod and cons channels starting at IPAv4 */
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0) {
-		prod = true;
-		ipa3_halt_q6_gsi_channels(prod);
-		IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-		IPADBG("Exit without consumer check\n");
-		return;
-	}
-
-	ipa3_halt_q6_gsi_channels(prod);
-
-	if (!ipa3_ctx->uc_ctx.uc_loaded) {
-		IPAERR("uC is not loaded. Skipping\n");
-		return;
-	}
-
-	for (client_idx = 0; client_idx < IPA_CLIENT_MAX; client_idx++)
-		if (IPA_CLIENT_IS_Q6_PROD(client_idx)) {
-			ep_idx = ipa3_get_ep_mapping(client_idx);
-			if (ep_idx == -1)
-				continue;
-
-			if (ipa3_uc_is_gsi_channel_empty(client_idx)) {
-				IPAERR("fail to validate Q6 ch emptiness %d\n",
-					client_idx);
-				/*
-				 * Indicates GSI hardware is stalled, unexpected
-				 * hardware state.
-				 * Remove bug for adb reboot issue.
-				 */
-			}
-		}
-
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-	IPADBG_LOW("Exit with success\n");
-}
-
-/**
- * ipa3_q6_pre_powerup_cleanup() - A cleanup routine for pheripheral
- * configuration in IPA HW. This is performed in case of SSR.
- *
- * This is a mandatory procedure, in case one of the steps fails, the
- * AP needs to restart.
- */
-void ipa3_q6_pre_powerup_cleanup(void)
-{
-	IPADBG_LOW("ENTER\n");
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	if (ipa3_ctx->ipa_config_is_mhi)
-		ipa3_set_reset_client_prod_pipe_delay(true,
-			IPA_CLIENT_MHI_PROD);
-
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-	IPADBG_LOW("Exit with success\n");
-}
-
-/*
- * ipa3_client_prod_post_shutdown_cleanup () - As part of this function
- * set end point delay client producer pipes and starting corresponding
- * gsi channels
- */
-
-void ipa3_client_prod_post_shutdown_cleanup(void)
-{
-	IPADBG_LOW("ENTER\n");
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	ipa3_set_reset_client_prod_pipe_delay(true,
-				IPA_CLIENT_USB_PROD);
-	ipa3_start_stop_client_prod_gsi_chnl(IPA_CLIENT_USB_PROD, true);
-
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-	IPADBG_LOW("Exit with success\n");
-}
-
-static inline void ipa3_sram_set_canary(u32 *sram_mmio, int offset)
-{
-	/* Set 4 bytes of CANARY before the offset */
-	sram_mmio[(offset - 4) / 4] = IPA_MEM_CANARY_VAL;
-}
-
-/**
- * _ipa_init_sram_v3() - Initialize IPA local SRAM.
- *
- * Return codes: 0 for success, negative value for failure
- */
-int _ipa_init_sram_v3(void)
-{
-	u32 *ipa_sram_mmio;
-	unsigned long phys_addr;
-
-	IPADBG(
-	    "ipa_wrapper_base(0x%08X) ipa_reg_base_ofst(0x%08X) IPA_SW_AREA_RAM_DIRECT_ACCESS_n(0x%08X) smem_restricted_bytes(0x%08X) smem_sz(0x%08X)\n",
-	    ipa3_ctx->ipa_wrapper_base,
-	    ipa3_ctx->ctrl->ipa_reg_base_ofst,
-	    ipahal_get_reg_n_ofst(
-		IPA_SW_AREA_RAM_DIRECT_ACCESS_n,
-		ipa3_ctx->smem_restricted_bytes / 4),
-	    ipa3_ctx->smem_restricted_bytes,
-	    ipa3_ctx->smem_sz);
-
-	phys_addr = ipa3_ctx->ipa_wrapper_base +
-		ipa3_ctx->ctrl->ipa_reg_base_ofst +
-		ipahal_get_reg_n_ofst(IPA_SW_AREA_RAM_DIRECT_ACCESS_n,
-			ipa3_ctx->smem_restricted_bytes / 4);
-
-	ipa_sram_mmio = ioremap(phys_addr, ipa3_ctx->smem_sz);
-	if (!ipa_sram_mmio) {
-		IPAERR("fail to ioremap IPA SRAM\n");
-		return -ENOMEM;
-	}
-
-	/* Consult with ipa_i.h on the location of the CANARY values */
-	ipa3_sram_set_canary(ipa_sram_mmio, IPA_MEM_PART(v4_flt_hash_ofst) - 4);
-	ipa3_sram_set_canary(ipa_sram_mmio, IPA_MEM_PART(v4_flt_hash_ofst));
-	ipa3_sram_set_canary(ipa_sram_mmio,
-		IPA_MEM_PART(v4_flt_nhash_ofst) - 4);
-	ipa3_sram_set_canary(ipa_sram_mmio, IPA_MEM_PART(v4_flt_nhash_ofst));
-	ipa3_sram_set_canary(ipa_sram_mmio, IPA_MEM_PART(v6_flt_hash_ofst) - 4);
-	ipa3_sram_set_canary(ipa_sram_mmio, IPA_MEM_PART(v6_flt_hash_ofst));
-	ipa3_sram_set_canary(ipa_sram_mmio,
-		IPA_MEM_PART(v6_flt_nhash_ofst) - 4);
-	ipa3_sram_set_canary(ipa_sram_mmio, IPA_MEM_PART(v6_flt_nhash_ofst));
-	ipa3_sram_set_canary(ipa_sram_mmio, IPA_MEM_PART(v4_rt_hash_ofst) - 4);
-	ipa3_sram_set_canary(ipa_sram_mmio, IPA_MEM_PART(v4_rt_hash_ofst));
-	ipa3_sram_set_canary(ipa_sram_mmio, IPA_MEM_PART(v4_rt_nhash_ofst) - 4);
-	ipa3_sram_set_canary(ipa_sram_mmio, IPA_MEM_PART(v4_rt_nhash_ofst));
-	ipa3_sram_set_canary(ipa_sram_mmio, IPA_MEM_PART(v6_rt_hash_ofst) - 4);
-	ipa3_sram_set_canary(ipa_sram_mmio, IPA_MEM_PART(v6_rt_hash_ofst));
-	ipa3_sram_set_canary(ipa_sram_mmio, IPA_MEM_PART(v6_rt_nhash_ofst) - 4);
-	ipa3_sram_set_canary(ipa_sram_mmio, IPA_MEM_PART(v6_rt_nhash_ofst));
-	ipa3_sram_set_canary(ipa_sram_mmio, IPA_MEM_PART(modem_hdr_ofst) - 4);
-	ipa3_sram_set_canary(ipa_sram_mmio, IPA_MEM_PART(modem_hdr_ofst));
-	ipa3_sram_set_canary(ipa_sram_mmio,
-		IPA_MEM_PART(modem_hdr_proc_ctx_ofst) - 4);
-	ipa3_sram_set_canary(ipa_sram_mmio,
-		IPA_MEM_PART(modem_hdr_proc_ctx_ofst));
-	if (ipa_get_hw_type() >= IPA_HW_v4_5) {
-		ipa3_sram_set_canary(ipa_sram_mmio,
-			IPA_MEM_PART(nat_tbl_ofst) - 12);
-		ipa3_sram_set_canary(ipa_sram_mmio,
-			IPA_MEM_PART(nat_tbl_ofst) - 8);
-		ipa3_sram_set_canary(ipa_sram_mmio,
-			IPA_MEM_PART(nat_tbl_ofst) - 4);
-		ipa3_sram_set_canary(ipa_sram_mmio, IPA_MEM_PART(nat_tbl_ofst));
-	}
-	if (ipa_get_hw_type() >= IPA_HW_v4_0) {
-		ipa3_sram_set_canary(ipa_sram_mmio,
-			IPA_MEM_PART(pdn_config_ofst) - 4);
-		ipa3_sram_set_canary(ipa_sram_mmio,
-			IPA_MEM_PART(pdn_config_ofst));
-		ipa3_sram_set_canary(ipa_sram_mmio,
-			IPA_MEM_PART(stats_quota_ofst) - 4);
-		ipa3_sram_set_canary(ipa_sram_mmio,
-			IPA_MEM_PART(stats_quota_ofst));
-	}
-	if (ipa_get_hw_type() <= IPA_HW_v3_5 ||
-		ipa_get_hw_type() >= IPA_HW_v4_5) {
-		ipa3_sram_set_canary(ipa_sram_mmio,
-			IPA_MEM_PART(modem_ofst) - 4);
-		ipa3_sram_set_canary(ipa_sram_mmio, IPA_MEM_PART(modem_ofst));
-	}
-	ipa3_sram_set_canary(ipa_sram_mmio,
-		(ipa_get_hw_type() >= IPA_HW_v3_5) ?
-			IPA_MEM_PART(uc_descriptor_ram_ofst) :
-			IPA_MEM_PART(end_ofst));
-
-	iounmap(ipa_sram_mmio);
-
-	return 0;
-}
-
-/**
- * _ipa_init_hdr_v3_0() - Initialize IPA header block.
- *
- * Return codes: 0 for success, negative value for failure
- */
-int _ipa_init_hdr_v3_0(void)
-{
-	struct ipa3_desc desc;
-	struct ipa_mem_buffer mem;
-	struct ipahal_imm_cmd_hdr_init_local cmd = {0};
-	struct ipahal_imm_cmd_pyld *cmd_pyld;
-	struct ipahal_imm_cmd_dma_shared_mem dma_cmd = { 0 };
-
-	mem.size = IPA_MEM_PART(modem_hdr_size) + IPA_MEM_PART(apps_hdr_size);
-	mem.base = dma_zalloc_coherent(ipa3_ctx->pdev, mem.size, &mem.phys_base,
-		GFP_KERNEL);
-	if (!mem.base) {
-		IPAERR("fail to alloc DMA buff of size %d\n", mem.size);
-		return -ENOMEM;
-	}
-
-	cmd.hdr_table_addr = mem.phys_base;
-	cmd.size_hdr_table = mem.size;
-	cmd.hdr_addr = ipa3_ctx->smem_restricted_bytes +
-		IPA_MEM_PART(modem_hdr_ofst);
-	cmd_pyld = ipahal_construct_imm_cmd(
-		IPA_IMM_CMD_HDR_INIT_LOCAL, &cmd, false);
-	if (!cmd_pyld) {
-		IPAERR("fail to construct hdr_init_local imm cmd\n");
-		dma_free_coherent(ipa3_ctx->pdev,
-			mem.size, mem.base,
-			mem.phys_base);
-		return -EFAULT;
-	}
-	ipa3_init_imm_cmd_desc(&desc, cmd_pyld);
-	IPA_DUMP_BUFF(mem.base, mem.phys_base, mem.size);
-
-	if (ipa3_send_cmd(1, &desc)) {
-		IPAERR("fail to send immediate command\n");
-		ipahal_destroy_imm_cmd(cmd_pyld);
-		dma_free_coherent(ipa3_ctx->pdev,
-			mem.size, mem.base,
-			mem.phys_base);
-		return -EFAULT;
-	}
-
-	ipahal_destroy_imm_cmd(cmd_pyld);
-	dma_free_coherent(ipa3_ctx->pdev, mem.size, mem.base, mem.phys_base);
-
-	mem.size = IPA_MEM_PART(modem_hdr_proc_ctx_size) +
-		IPA_MEM_PART(apps_hdr_proc_ctx_size);
-	mem.base = dma_zalloc_coherent(ipa3_ctx->pdev, mem.size, &mem.phys_base,
-		GFP_KERNEL);
-	if (!mem.base) {
-		IPAERR("fail to alloc DMA buff of size %d\n", mem.size);
-		return -ENOMEM;
-	}
-
-	dma_cmd.is_read = false;
-	dma_cmd.skip_pipeline_clear = false;
-	dma_cmd.pipeline_clear_options = IPAHAL_HPS_CLEAR;
-	dma_cmd.system_addr = mem.phys_base;
-	dma_cmd.local_addr = ipa3_ctx->smem_restricted_bytes +
-		IPA_MEM_PART(modem_hdr_proc_ctx_ofst);
-	dma_cmd.size = mem.size;
-	cmd_pyld = ipahal_construct_imm_cmd(
-		IPA_IMM_CMD_DMA_SHARED_MEM, &dma_cmd, false);
-	if (!cmd_pyld) {
-		IPAERR("fail to construct dma_shared_mem imm\n");
-		dma_free_coherent(ipa3_ctx->pdev,
-			mem.size, mem.base,
-			mem.phys_base);
-		return -ENOMEM;
-	}
-	ipa3_init_imm_cmd_desc(&desc, cmd_pyld);
-	IPA_DUMP_BUFF(mem.base, mem.phys_base, mem.size);
-
-	if (ipa3_send_cmd(1, &desc)) {
-		IPAERR("fail to send immediate command\n");
-		ipahal_destroy_imm_cmd(cmd_pyld);
-		dma_free_coherent(ipa3_ctx->pdev,
-			mem.size,
-			mem.base,
-			mem.phys_base);
-		return -EBUSY;
-	}
-	ipahal_destroy_imm_cmd(cmd_pyld);
-
-	ipahal_write_reg(IPA_LOCAL_PKT_PROC_CNTXT_BASE, dma_cmd.local_addr);
-
-	dma_free_coherent(ipa3_ctx->pdev, mem.size, mem.base, mem.phys_base);
-
-	return 0;
-}
-
-/**
- * _ipa_init_rt4_v3() - Initialize IPA routing block for IPv4.
- *
- * Return codes: 0 for success, negative value for failure
- */
-int _ipa_init_rt4_v3(void)
-{
-	struct ipa3_desc desc;
-	struct ipa_mem_buffer mem;
-	struct ipahal_imm_cmd_ip_v4_routing_init v4_cmd;
-	struct ipahal_imm_cmd_pyld *cmd_pyld;
-	int i;
-	int rc = 0;
-
-	for (i = IPA_MEM_PART(v4_modem_rt_index_lo);
-		i <= IPA_MEM_PART(v4_modem_rt_index_hi);
-		i++)
-		ipa3_ctx->rt_idx_bitmap[IPA_IP_v4] |= (1 << i);
-	IPADBG("v4 rt bitmap 0x%lx\n", ipa3_ctx->rt_idx_bitmap[IPA_IP_v4]);
-
-	rc = ipahal_rt_generate_empty_img(IPA_MEM_PART(v4_rt_num_index),
-		IPA_MEM_PART(v4_rt_hash_size), IPA_MEM_PART(v4_rt_nhash_size),
-		&mem, false);
-	if (rc) {
-		IPAERR("fail generate empty v4 rt img\n");
-		return rc;
-	}
-
-	/*
-	 * SRAM memory not allocated to hash tables. Initializing/Sending
-	 * command to hash tables(filer/routing) operation not supported.
-	 */
-	if (ipa3_ctx->ipa_fltrt_not_hashable) {
-		v4_cmd.hash_rules_addr = 0;
-		v4_cmd.hash_rules_size = 0;
-		v4_cmd.hash_local_addr = 0;
-	} else {
-		v4_cmd.hash_rules_addr = mem.phys_base;
-		v4_cmd.hash_rules_size = mem.size;
-		v4_cmd.hash_local_addr = ipa3_ctx->smem_restricted_bytes +
-			IPA_MEM_PART(v4_rt_hash_ofst);
-	}
-
-	v4_cmd.nhash_rules_addr = mem.phys_base;
-	v4_cmd.nhash_rules_size = mem.size;
-	v4_cmd.nhash_local_addr = ipa3_ctx->smem_restricted_bytes +
-		IPA_MEM_PART(v4_rt_nhash_ofst);
-	IPADBG("putting hashable routing IPv4 rules to phys 0x%x\n",
-				v4_cmd.hash_local_addr);
-	IPADBG("putting non-hashable routing IPv4 rules to phys 0x%x\n",
-				v4_cmd.nhash_local_addr);
-	cmd_pyld = ipahal_construct_imm_cmd(
-		IPA_IMM_CMD_IP_V4_ROUTING_INIT, &v4_cmd, false);
-	if (!cmd_pyld) {
-		IPAERR("fail construct ip_v4_rt_init imm cmd\n");
-		rc = -EPERM;
-		goto free_mem;
-	}
-
-	ipa3_init_imm_cmd_desc(&desc, cmd_pyld);
-	IPA_DUMP_BUFF(mem.base, mem.phys_base, mem.size);
-
-	if (ipa3_send_cmd(1, &desc)) {
-		IPAERR("fail to send immediate command\n");
-		rc = -EFAULT;
-	}
-
-	ipahal_destroy_imm_cmd(cmd_pyld);
-
-free_mem:
-	ipahal_free_dma_mem(&mem);
-	return rc;
-}
-
-/**
- * _ipa_init_rt6_v3() - Initialize IPA routing block for IPv6.
- *
- * Return codes: 0 for success, negative value for failure
- */
-int _ipa_init_rt6_v3(void)
-{
-	struct ipa3_desc desc;
-	struct ipa_mem_buffer mem;
-	struct ipahal_imm_cmd_ip_v6_routing_init v6_cmd;
-	struct ipahal_imm_cmd_pyld *cmd_pyld;
-	int i;
-	int rc = 0;
-
-	for (i = IPA_MEM_PART(v6_modem_rt_index_lo);
-		i <= IPA_MEM_PART(v6_modem_rt_index_hi);
-		i++)
-		ipa3_ctx->rt_idx_bitmap[IPA_IP_v6] |= (1 << i);
-	IPADBG("v6 rt bitmap 0x%lx\n", ipa3_ctx->rt_idx_bitmap[IPA_IP_v6]);
-
-	rc = ipahal_rt_generate_empty_img(IPA_MEM_PART(v6_rt_num_index),
-		IPA_MEM_PART(v6_rt_hash_size), IPA_MEM_PART(v6_rt_nhash_size),
-		&mem, false);
-	if (rc) {
-		IPAERR("fail generate empty v6 rt img\n");
-		return rc;
-	}
-
-	/*
-	 * SRAM memory not allocated to hash tables. Initializing/Sending
-	 * command to hash tables(filer/routing) operation not supported.
-	 */
-	if (ipa3_ctx->ipa_fltrt_not_hashable) {
-		v6_cmd.hash_rules_addr = 0;
-		v6_cmd.hash_rules_size = 0;
-		v6_cmd.hash_local_addr = 0;
-	} else {
-		v6_cmd.hash_rules_addr = mem.phys_base;
-		v6_cmd.hash_rules_size = mem.size;
-		v6_cmd.hash_local_addr = ipa3_ctx->smem_restricted_bytes +
-			IPA_MEM_PART(v6_rt_hash_ofst);
-	}
-
-	v6_cmd.nhash_rules_addr = mem.phys_base;
-	v6_cmd.nhash_rules_size = mem.size;
-	v6_cmd.nhash_local_addr = ipa3_ctx->smem_restricted_bytes +
-		IPA_MEM_PART(v6_rt_nhash_ofst);
-	IPADBG("putting hashable routing IPv6 rules to phys 0x%x\n",
-				v6_cmd.hash_local_addr);
-	IPADBG("putting non-hashable routing IPv6 rules to phys 0x%x\n",
-				v6_cmd.nhash_local_addr);
-	cmd_pyld = ipahal_construct_imm_cmd(
-		IPA_IMM_CMD_IP_V6_ROUTING_INIT, &v6_cmd, false);
-	if (!cmd_pyld) {
-		IPAERR("fail construct ip_v6_rt_init imm cmd\n");
-		rc = -EPERM;
-		goto free_mem;
-	}
-
-	ipa3_init_imm_cmd_desc(&desc, cmd_pyld);
-	IPA_DUMP_BUFF(mem.base, mem.phys_base, mem.size);
-
-	if (ipa3_send_cmd(1, &desc)) {
-		IPAERR("fail to send immediate command\n");
-		rc = -EFAULT;
-	}
-
-	ipahal_destroy_imm_cmd(cmd_pyld);
-
-free_mem:
-	ipahal_free_dma_mem(&mem);
-	return rc;
-}
-
-/**
- * _ipa_init_flt4_v3() - Initialize IPA filtering block for IPv4.
- *
- * Return codes: 0 for success, negative value for failure
- */
-int _ipa_init_flt4_v3(void)
-{
-	struct ipa3_desc desc;
-	struct ipa_mem_buffer mem;
-	struct ipahal_imm_cmd_ip_v4_filter_init v4_cmd;
-	struct ipahal_imm_cmd_pyld *cmd_pyld;
-	int rc;
-
-	rc = ipahal_flt_generate_empty_img(ipa3_ctx->ep_flt_num,
-		IPA_MEM_PART(v4_flt_hash_size),
-		IPA_MEM_PART(v4_flt_nhash_size), ipa3_ctx->ep_flt_bitmap,
-		&mem, false);
-	if (rc) {
-		IPAERR("fail generate empty v4 flt img\n");
-		return rc;
-	}
-
-	/*
-	 * SRAM memory not allocated to hash tables. Initializing/Sending
-	 * command to hash tables(filer/routing) operation not supported.
-	 */
-	if (ipa3_ctx->ipa_fltrt_not_hashable) {
-		v4_cmd.hash_rules_addr = 0;
-		v4_cmd.hash_rules_size = 0;
-		v4_cmd.hash_local_addr = 0;
-	} else {
-		v4_cmd.hash_rules_addr = mem.phys_base;
-		v4_cmd.hash_rules_size = mem.size;
-		v4_cmd.hash_local_addr = ipa3_ctx->smem_restricted_bytes +
-			IPA_MEM_PART(v4_flt_hash_ofst);
-	}
-
-	v4_cmd.nhash_rules_addr = mem.phys_base;
-	v4_cmd.nhash_rules_size = mem.size;
-	v4_cmd.nhash_local_addr = ipa3_ctx->smem_restricted_bytes +
-		IPA_MEM_PART(v4_flt_nhash_ofst);
-	IPADBG("putting hashable filtering IPv4 rules to phys 0x%x\n",
-				v4_cmd.hash_local_addr);
-	IPADBG("putting non-hashable filtering IPv4 rules to phys 0x%x\n",
-				v4_cmd.nhash_local_addr);
-	cmd_pyld = ipahal_construct_imm_cmd(
-		IPA_IMM_CMD_IP_V4_FILTER_INIT, &v4_cmd, false);
-	if (!cmd_pyld) {
-		IPAERR("fail construct ip_v4_flt_init imm cmd\n");
-		rc = -EPERM;
-		goto free_mem;
-	}
-
-	ipa3_init_imm_cmd_desc(&desc, cmd_pyld);
-	IPA_DUMP_BUFF(mem.base, mem.phys_base, mem.size);
-
-	if (ipa3_send_cmd(1, &desc)) {
-		IPAERR("fail to send immediate command\n");
-		rc = -EFAULT;
-	}
-
-	ipahal_destroy_imm_cmd(cmd_pyld);
-
-free_mem:
-	ipahal_free_dma_mem(&mem);
-	return rc;
-}
-
-/**
- * _ipa_init_flt6_v3() - Initialize IPA filtering block for IPv6.
- *
- * Return codes: 0 for success, negative value for failure
- */
-int _ipa_init_flt6_v3(void)
-{
-	struct ipa3_desc desc;
-	struct ipa_mem_buffer mem;
-	struct ipahal_imm_cmd_ip_v6_filter_init v6_cmd;
-	struct ipahal_imm_cmd_pyld *cmd_pyld;
-	int rc;
-
-	rc = ipahal_flt_generate_empty_img(ipa3_ctx->ep_flt_num,
-		IPA_MEM_PART(v6_flt_hash_size),
-		IPA_MEM_PART(v6_flt_nhash_size), ipa3_ctx->ep_flt_bitmap,
-		&mem, false);
-	if (rc) {
-		IPAERR("fail generate empty v6 flt img\n");
-		return rc;
-	}
-
-	/*
-	 * SRAM memory not allocated to hash tables. Initializing/Sending
-	 * command to hash tables(filer/routing) operation not supported.
-	 */
-	if (ipa3_ctx->ipa_fltrt_not_hashable) {
-		v6_cmd.hash_rules_addr = 0;
-		v6_cmd.hash_rules_size = 0;
-		v6_cmd.hash_local_addr = 0;
-	} else {
-		v6_cmd.hash_rules_addr = mem.phys_base;
-		v6_cmd.hash_rules_size = mem.size;
-		v6_cmd.hash_local_addr = ipa3_ctx->smem_restricted_bytes +
-			IPA_MEM_PART(v6_flt_hash_ofst);
-	}
-
-	v6_cmd.nhash_rules_addr = mem.phys_base;
-	v6_cmd.nhash_rules_size = mem.size;
-	v6_cmd.nhash_local_addr = ipa3_ctx->smem_restricted_bytes +
-		IPA_MEM_PART(v6_flt_nhash_ofst);
-	IPADBG("putting hashable filtering IPv6 rules to phys 0x%x\n",
-				v6_cmd.hash_local_addr);
-	IPADBG("putting non-hashable filtering IPv6 rules to phys 0x%x\n",
-				v6_cmd.nhash_local_addr);
-
-	cmd_pyld = ipahal_construct_imm_cmd(
-		IPA_IMM_CMD_IP_V6_FILTER_INIT, &v6_cmd, false);
-	if (!cmd_pyld) {
-		IPAERR("fail construct ip_v6_flt_init imm cmd\n");
-		rc = -EPERM;
-		goto free_mem;
-	}
-
-	ipa3_init_imm_cmd_desc(&desc, cmd_pyld);
-	IPA_DUMP_BUFF(mem.base, mem.phys_base, mem.size);
-
-	if (ipa3_send_cmd(1, &desc)) {
-		IPAERR("fail to send immediate command\n");
-		rc = -EFAULT;
-	}
-
-	ipahal_destroy_imm_cmd(cmd_pyld);
-
-free_mem:
-	ipahal_free_dma_mem(&mem);
-	return rc;
-}
-
-static int ipa3_setup_flt_hash_tuple(void)
-{
-	int pipe_idx;
-	struct ipahal_reg_hash_tuple tuple;
-
-	memset(&tuple, 0, sizeof(struct ipahal_reg_hash_tuple));
-
-	for (pipe_idx = 0; pipe_idx < ipa3_ctx->ipa_num_pipes ; pipe_idx++) {
-		if (!ipa_is_ep_support_flt(pipe_idx))
-			continue;
-
-		if (ipa_is_modem_pipe(pipe_idx))
-			continue;
-
-		if (ipa3_set_flt_tuple_mask(pipe_idx, &tuple)) {
-			IPAERR("failed to setup pipe %d flt tuple\n", pipe_idx);
-			return -EFAULT;
-		}
-	}
-
-	return 0;
-}
-
-static int ipa3_setup_rt_hash_tuple(void)
-{
-	int tbl_idx;
-	struct ipahal_reg_hash_tuple tuple;
-
-	memset(&tuple, 0, sizeof(struct ipahal_reg_hash_tuple));
-
-	for (tbl_idx = 0;
-		tbl_idx < max(IPA_MEM_PART(v6_rt_num_index),
-		IPA_MEM_PART(v4_rt_num_index));
-		tbl_idx++) {
-
-		if (tbl_idx >= IPA_MEM_PART(v4_modem_rt_index_lo) &&
-			tbl_idx <= IPA_MEM_PART(v4_modem_rt_index_hi))
-			continue;
-
-		if (tbl_idx >= IPA_MEM_PART(v6_modem_rt_index_lo) &&
-			tbl_idx <= IPA_MEM_PART(v6_modem_rt_index_hi))
-			continue;
-
-		if (ipa3_set_rt_tuple_mask(tbl_idx, &tuple)) {
-			IPAERR("failed to setup tbl %d rt tuple\n", tbl_idx);
-			return -EFAULT;
-		}
-	}
-
-	return 0;
-}
-
-static int ipa3_setup_apps_pipes(void)
-{
-	struct ipa_sys_connect_params sys_in;
-	int result = 0;
-
-	if (ipa3_ctx->gsi_ch20_wa) {
-		IPADBG("Allocating GSI physical channel 20\n");
-		result = ipa_gsi_ch20_wa();
-		if (result) {
-			IPAERR("ipa_gsi_ch20_wa failed %d\n", result);
-			goto fail_ch20_wa;
-		}
-	}
-
-	/* allocate the common PROD event ring */
-	if (ipa3_alloc_common_event_ring()) {
-		IPAERR("ipa3_alloc_common_event_ring failed.\n");
-		result = -EPERM;
-		goto fail_ch20_wa;
-	}
-
-	/* CMD OUT (AP->IPA) */
-	memset(&sys_in, 0, sizeof(struct ipa_sys_connect_params));
-	sys_in.client = IPA_CLIENT_APPS_CMD_PROD;
-	sys_in.desc_fifo_sz = IPA_SYS_DESC_FIFO_SZ;
-	sys_in.ipa_ep_cfg.mode.mode = IPA_DMA;
-	sys_in.ipa_ep_cfg.mode.dst = IPA_CLIENT_APPS_LAN_CONS;
-	if (ipa3_setup_sys_pipe(&sys_in, &ipa3_ctx->clnt_hdl_cmd)) {
-		IPAERR(":setup sys pipe (APPS_CMD_PROD) failed.\n");
-		result = -EPERM;
-		goto fail_ch20_wa;
-	}
-	IPADBG("Apps to IPA cmd pipe is connected\n");
-
-	IPADBG("Will initialize SRAM\n");
-	ipa3_ctx->ctrl->ipa_init_sram();
-	IPADBG("SRAM initialized\n");
-
-	IPADBG("Will initialize HDR\n");
-	ipa3_ctx->ctrl->ipa_init_hdr();
-	IPADBG("HDR initialized\n");
-
-	IPADBG("Will initialize V4 RT\n");
-	ipa3_ctx->ctrl->ipa_init_rt4();
-	IPADBG("V4 RT initialized\n");
-
-	IPADBG("Will initialize V6 RT\n");
-	ipa3_ctx->ctrl->ipa_init_rt6();
-	IPADBG("V6 RT initialized\n");
-
-	IPADBG("Will initialize V4 FLT\n");
-	ipa3_ctx->ctrl->ipa_init_flt4();
-	IPADBG("V4 FLT initialized\n");
-
-	IPADBG("Will initialize V6 FLT\n");
-	ipa3_ctx->ctrl->ipa_init_flt6();
-	IPADBG("V6 FLT initialized\n");
-
-	if (!ipa3_ctx->ipa_fltrt_not_hashable) {
-		if (ipa3_setup_flt_hash_tuple()) {
-			IPAERR(":fail to configure flt hash tuple\n");
-			result = -EPERM;
-			goto fail_flt_hash_tuple;
-		}
-		IPADBG("flt hash tuple is configured\n");
-
-		if (ipa3_setup_rt_hash_tuple()) {
-			IPAERR(":fail to configure rt hash tuple\n");
-			result = -EPERM;
-			goto fail_flt_hash_tuple;
-		}
-		IPADBG("rt hash tuple is configured\n");
-	}
-	if (ipa3_setup_exception_path()) {
-		IPAERR(":fail to setup excp path\n");
-		result = -EPERM;
-		goto fail_flt_hash_tuple;
-	}
-	IPADBG("Exception path was successfully set");
-
-	if (ipa3_setup_dflt_rt_tables()) {
-		IPAERR(":fail to setup dflt routes\n");
-		result = -EPERM;
-		goto fail_flt_hash_tuple;
-	}
-	IPADBG("default routing was set\n");
-
-	/* LAN IN (IPA->AP) */
-	memset(&sys_in, 0, sizeof(struct ipa_sys_connect_params));
-	sys_in.client = IPA_CLIENT_APPS_LAN_CONS;
-	sys_in.desc_fifo_sz = IPA_SYS_DESC_FIFO_SZ;
-	sys_in.notify = ipa3_lan_rx_cb;
-	sys_in.priv = NULL;
-	sys_in.ipa_ep_cfg.hdr.hdr_len = IPA_LAN_RX_HEADER_LENGTH;
-	sys_in.ipa_ep_cfg.hdr_ext.hdr_little_endian = false;
-	sys_in.ipa_ep_cfg.hdr_ext.hdr_total_len_or_pad_valid = true;
-	sys_in.ipa_ep_cfg.hdr_ext.hdr_total_len_or_pad = IPA_HDR_PAD;
-	sys_in.ipa_ep_cfg.hdr_ext.hdr_payload_len_inc_padding = false;
-	sys_in.ipa_ep_cfg.hdr_ext.hdr_total_len_or_pad_offset = 0;
-	sys_in.ipa_ep_cfg.hdr_ext.hdr_pad_to_alignment = 2;
-	sys_in.ipa_ep_cfg.cfg.cs_offload_en = IPA_DISABLE_CS_OFFLOAD;
-
-	/**
-	 * ipa_lan_rx_cb() intended to notify the source EP about packet
-	 * being received on the LAN_CONS via calling the source EP call-back.
-	 * There could be a race condition with calling this call-back. Other
-	 * thread may nullify it - e.g. on EP disconnect.
-	 * This lock intended to protect the access to the source EP call-back
-	 */
-	spin_lock_init(&ipa3_ctx->disconnect_lock);
-	if (ipa3_setup_sys_pipe(&sys_in, &ipa3_ctx->clnt_hdl_data_in)) {
-		IPAERR(":setup sys pipe (LAN_CONS) failed.\n");
-		result = -EPERM;
-		goto fail_flt_hash_tuple;
-	}
-
-	/* LAN OUT (AP->IPA) */
-	if (!ipa3_ctx->ipa_config_is_mhi) {
-		memset(&sys_in, 0, sizeof(struct ipa_sys_connect_params));
-		sys_in.client = IPA_CLIENT_APPS_LAN_PROD;
-		sys_in.desc_fifo_sz = IPA_SYS_TX_DATA_DESC_FIFO_SZ;
-		sys_in.ipa_ep_cfg.mode.mode = IPA_BASIC;
-		if (ipa3_setup_sys_pipe(&sys_in,
-			&ipa3_ctx->clnt_hdl_data_out)) {
-			IPAERR(":setup sys pipe (LAN_PROD) failed.\n");
-			result = -EPERM;
-			goto fail_lan_data_out;
-		}
-	}
-
-	return 0;
-
-fail_lan_data_out:
-	ipa3_teardown_sys_pipe(ipa3_ctx->clnt_hdl_data_in);
-fail_flt_hash_tuple:
-	if (ipa3_ctx->dflt_v6_rt_rule_hdl)
-		__ipa3_del_rt_rule(ipa3_ctx->dflt_v6_rt_rule_hdl);
-	if (ipa3_ctx->dflt_v4_rt_rule_hdl)
-		__ipa3_del_rt_rule(ipa3_ctx->dflt_v4_rt_rule_hdl);
-	if (ipa3_ctx->excp_hdr_hdl)
-		__ipa3_del_hdr(ipa3_ctx->excp_hdr_hdl, false);
-	ipa3_teardown_sys_pipe(ipa3_ctx->clnt_hdl_cmd);
-fail_ch20_wa:
-	return result;
-}
-
-static void ipa3_teardown_apps_pipes(void)
-{
-	if (!ipa3_ctx->ipa_config_is_mhi)
-		ipa3_teardown_sys_pipe(ipa3_ctx->clnt_hdl_data_out);
-	ipa3_teardown_sys_pipe(ipa3_ctx->clnt_hdl_data_in);
-	__ipa3_del_rt_rule(ipa3_ctx->dflt_v6_rt_rule_hdl);
-	__ipa3_del_rt_rule(ipa3_ctx->dflt_v4_rt_rule_hdl);
-	__ipa3_del_hdr(ipa3_ctx->excp_hdr_hdl, false);
-	ipa3_teardown_sys_pipe(ipa3_ctx->clnt_hdl_cmd);
-}
-
-#ifdef CONFIG_COMPAT
-
-static long compat_ipa3_nat_ipv6ct_alloc_table(unsigned long arg,
-	int (alloc_func)(struct ipa_ioc_nat_ipv6ct_table_alloc *))
-{
-	long retval;
-	struct ipa_ioc_nat_ipv6ct_table_alloc32 table_alloc32;
-	struct ipa_ioc_nat_ipv6ct_table_alloc table_alloc;
-
-	retval = copy_from_user(&table_alloc32, (const void __user *)arg,
-		sizeof(struct ipa_ioc_nat_ipv6ct_table_alloc32));
-	if (retval)
-		return retval;
-
-	table_alloc.size = (size_t)table_alloc32.size;
-	table_alloc.offset = (off_t)table_alloc32.offset;
-
-	retval = alloc_func(&table_alloc);
-	if (retval)
-		return retval;
-
-	if (table_alloc.offset) {
-		table_alloc32.offset = (compat_off_t)table_alloc.offset;
-		retval = copy_to_user((void __user *)arg, &table_alloc32,
-			sizeof(struct ipa_ioc_nat_ipv6ct_table_alloc32));
-	}
-
-	return retval;
-}
-
-long compat_ipa3_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
-{
-	long retval = 0;
-	struct ipa3_ioc_nat_alloc_mem32 nat_mem32;
-	struct ipa_ioc_nat_alloc_mem nat_mem;
-
-	switch (cmd) {
-	case IPA_IOC_ADD_HDR32:
-		cmd = IPA_IOC_ADD_HDR;
-		break;
-	case IPA_IOC_DEL_HDR32:
-		cmd = IPA_IOC_DEL_HDR;
-		break;
-	case IPA_IOC_ADD_RT_RULE32:
-		cmd = IPA_IOC_ADD_RT_RULE;
-		break;
-	case IPA_IOC_DEL_RT_RULE32:
-		cmd = IPA_IOC_DEL_RT_RULE;
-		break;
-	case IPA_IOC_ADD_FLT_RULE32:
-		cmd = IPA_IOC_ADD_FLT_RULE;
-		break;
-	case IPA_IOC_DEL_FLT_RULE32:
-		cmd = IPA_IOC_DEL_FLT_RULE;
-		break;
-	case IPA_IOC_GET_RT_TBL32:
-		cmd = IPA_IOC_GET_RT_TBL;
-		break;
-	case IPA_IOC_COPY_HDR32:
-		cmd = IPA_IOC_COPY_HDR;
-		break;
-	case IPA_IOC_QUERY_INTF32:
-		cmd = IPA_IOC_QUERY_INTF;
-		break;
-	case IPA_IOC_QUERY_INTF_TX_PROPS32:
-		cmd = IPA_IOC_QUERY_INTF_TX_PROPS;
-		break;
-	case IPA_IOC_QUERY_INTF_RX_PROPS32:
-		cmd = IPA_IOC_QUERY_INTF_RX_PROPS;
-		break;
-	case IPA_IOC_QUERY_INTF_EXT_PROPS32:
-		cmd = IPA_IOC_QUERY_INTF_EXT_PROPS;
-		break;
-	case IPA_IOC_GET_HDR32:
-		cmd = IPA_IOC_GET_HDR;
-		break;
-	case IPA_IOC_ALLOC_NAT_MEM32:
-		retval = copy_from_user(&nat_mem32, (const void __user *)arg,
-			sizeof(struct ipa3_ioc_nat_alloc_mem32));
-		if (retval)
-			return retval;
-		memcpy(nat_mem.dev_name, nat_mem32.dev_name,
-				IPA_RESOURCE_NAME_MAX);
-		nat_mem.size = (size_t)nat_mem32.size;
-		nat_mem.offset = (off_t)nat_mem32.offset;
-
-		/* null terminate the string */
-		nat_mem.dev_name[IPA_RESOURCE_NAME_MAX - 1] = '\0';
-
-		retval = ipa3_allocate_nat_device(&nat_mem);
-		if (retval)
-			return retval;
-		nat_mem32.offset = (compat_off_t)nat_mem.offset;
-		retval = copy_to_user((void __user *)arg, &nat_mem32,
-			sizeof(struct ipa3_ioc_nat_alloc_mem32));
-		return retval;
-	case IPA_IOC_ALLOC_NAT_TABLE32:
-		return compat_ipa3_nat_ipv6ct_alloc_table(arg,
-			ipa3_allocate_nat_table);
-	case IPA_IOC_ALLOC_IPV6CT_TABLE32:
-		return compat_ipa3_nat_ipv6ct_alloc_table(arg,
-			ipa3_allocate_ipv6ct_table);
-	case IPA_IOC_V4_INIT_NAT32:
-		cmd = IPA_IOC_V4_INIT_NAT;
-		break;
-	case IPA_IOC_INIT_IPV6CT_TABLE32:
-		cmd = IPA_IOC_INIT_IPV6CT_TABLE;
-		break;
-	case IPA_IOC_TABLE_DMA_CMD32:
-		cmd = IPA_IOC_TABLE_DMA_CMD;
-		break;
-	case IPA_IOC_V4_DEL_NAT32:
-		cmd = IPA_IOC_V4_DEL_NAT;
-		break;
-	case IPA_IOC_DEL_NAT_TABLE32:
-		cmd = IPA_IOC_DEL_NAT_TABLE;
-		break;
-	case IPA_IOC_DEL_IPV6CT_TABLE32:
-		cmd = IPA_IOC_DEL_IPV6CT_TABLE;
-		break;
-	case IPA_IOC_NAT_MODIFY_PDN32:
-		cmd = IPA_IOC_NAT_MODIFY_PDN;
-		break;
-	case IPA_IOC_GET_NAT_OFFSET32:
-		cmd = IPA_IOC_GET_NAT_OFFSET;
-		break;
-	case IPA_IOC_PULL_MSG32:
-		cmd = IPA_IOC_PULL_MSG;
-		break;
-	case IPA_IOC_RM_ADD_DEPENDENCY32:
-		cmd = IPA_IOC_RM_ADD_DEPENDENCY;
-		break;
-	case IPA_IOC_RM_DEL_DEPENDENCY32:
-		cmd = IPA_IOC_RM_DEL_DEPENDENCY;
-		break;
-	case IPA_IOC_GENERATE_FLT_EQ32:
-		cmd = IPA_IOC_GENERATE_FLT_EQ;
-		break;
-	case IPA_IOC_QUERY_RT_TBL_INDEX32:
-		cmd = IPA_IOC_QUERY_RT_TBL_INDEX;
-		break;
-	case IPA_IOC_WRITE_QMAPID32:
-		cmd = IPA_IOC_WRITE_QMAPID;
-		break;
-	case IPA_IOC_MDFY_FLT_RULE32:
-		cmd = IPA_IOC_MDFY_FLT_RULE;
-		break;
-	case IPA_IOC_NOTIFY_WAN_UPSTREAM_ROUTE_ADD32:
-		cmd = IPA_IOC_NOTIFY_WAN_UPSTREAM_ROUTE_ADD;
-		break;
-	case IPA_IOC_NOTIFY_WAN_UPSTREAM_ROUTE_DEL32:
-		cmd = IPA_IOC_NOTIFY_WAN_UPSTREAM_ROUTE_DEL;
-		break;
-	case IPA_IOC_NOTIFY_WAN_EMBMS_CONNECTED32:
-		cmd = IPA_IOC_NOTIFY_WAN_EMBMS_CONNECTED;
-		break;
-	case IPA_IOC_MDFY_RT_RULE32:
-		cmd = IPA_IOC_MDFY_RT_RULE;
-		break;
-	case IPA_IOC_COMMIT_HDR:
-	case IPA_IOC_RESET_HDR:
-	case IPA_IOC_COMMIT_RT:
-	case IPA_IOC_RESET_RT:
-	case IPA_IOC_COMMIT_FLT:
-	case IPA_IOC_RESET_FLT:
-	case IPA_IOC_DUMP:
-	case IPA_IOC_PUT_RT_TBL:
-	case IPA_IOC_PUT_HDR:
-	case IPA_IOC_SET_FLT:
-	case IPA_IOC_QUERY_EP_MAPPING:
-		break;
-	default:
-		return -ENOIOCTLCMD;
-	}
-	return ipa3_ioctl(file, cmd, (unsigned long) compat_ptr(arg));
-}
-#endif
-
-static ssize_t ipa3_write(struct file *file, const char __user *buf,
-			  size_t count, loff_t *ppos);
-
-static const struct file_operations ipa3_drv_fops = {
-	.owner = THIS_MODULE,
-	.open = ipa3_open,
-	.read = ipa3_read,
-	.write = ipa3_write,
-	.unlocked_ioctl = ipa3_ioctl,
-#ifdef CONFIG_COMPAT
-	.compat_ioctl = compat_ipa3_ioctl,
-#endif
-};
-
-static int ipa3_get_clks(struct device *dev)
-{
-	if (ipa3_ctx->ipa3_hw_mode != IPA_HW_MODE_NORMAL) {
-		IPADBG("not supported in this HW mode\n");
-		ipa3_clk = NULL;
-		return 0;
-	}
-
-	if (ipa3_res.use_bw_vote) {
-		IPADBG("Vote IPA clock by bw voting via bus scaling driver\n");
-		ipa3_clk = NULL;
-		return 0;
-	}
-
-	ipa3_clk = clk_get(dev, "core_clk");
-	if (IS_ERR(ipa3_clk)) {
-		if (ipa3_clk != ERR_PTR(-EPROBE_DEFER))
-			IPAERR("fail to get ipa clk\n");
-		return PTR_ERR(ipa3_clk);
-	}
-	return 0;
-}
-
-/**
- * _ipa_enable_clks_v3_0() - Enable IPA clocks.
- */
-void _ipa_enable_clks_v3_0(void)
-{
-	IPADBG_LOW("curr_ipa_clk_rate=%d", ipa3_ctx->curr_ipa_clk_rate);
-	if (ipa3_clk) {
-		IPADBG_LOW("enabling gcc_ipa_clk\n");
-		clk_prepare(ipa3_clk);
-		clk_enable(ipa3_clk);
-		clk_set_rate(ipa3_clk, ipa3_ctx->curr_ipa_clk_rate);
-	}
-
-	ipa3_uc_notify_clk_state(true);
-}
-
-static unsigned int ipa3_get_bus_vote(void)
-{
-	unsigned int idx = 1;
-
-	if (ipa3_ctx->curr_ipa_clk_rate == ipa3_ctx->ctrl->ipa_clk_rate_svs2) {
-		idx = 1;
-	} else if (ipa3_ctx->curr_ipa_clk_rate ==
-		ipa3_ctx->ctrl->ipa_clk_rate_svs) {
-		idx = 2;
-	} else if (ipa3_ctx->curr_ipa_clk_rate ==
-		ipa3_ctx->ctrl->ipa_clk_rate_nominal) {
-		idx = 3;
-	} else if (ipa3_ctx->curr_ipa_clk_rate ==
-			ipa3_ctx->ctrl->ipa_clk_rate_turbo) {
-		idx = ipa3_ctx->ctrl->msm_bus_data_ptr->num_usecases - 1;
-	} else {
-		WARN(1, "unexpected clock rate");
-	}
-	IPADBG_LOW("curr %d idx %d\n", ipa3_ctx->curr_ipa_clk_rate, idx);
-
-	return idx;
-}
-
-/**
- * ipa3_enable_clks() - Turn on IPA clocks
- *
- * Return codes:
- * None
- */
-void ipa3_enable_clks(void)
-{
-	if (ipa3_ctx->ipa3_hw_mode != IPA_HW_MODE_NORMAL) {
-		IPAERR("not supported in this mode\n");
-		return;
-	}
-
-	IPADBG("enabling IPA clocks and bus voting\n");
-
-	if (msm_bus_scale_client_update_request(ipa3_ctx->ipa_bus_hdl,
-	    ipa3_get_bus_vote()))
-		WARN(1, "bus scaling failed");
-	ipa3_ctx->ctrl->ipa3_enable_clks();
-	atomic_set(&ipa3_ctx->ipa_clk_vote, 1);
-}
-
-
-/**
- * _ipa_disable_clks_v3_0() - Disable IPA clocks.
- */
-void _ipa_disable_clks_v3_0(void)
-{
-	ipa3_uc_notify_clk_state(false);
-	if (ipa3_clk) {
-		IPADBG_LOW("disabling gcc_ipa_clk\n");
-		clk_disable_unprepare(ipa3_clk);
-	}
-}
-
-/**
- * ipa3_disable_clks() - Turn off IPA clocks
- *
- * Return codes:
- * None
- */
-void ipa3_disable_clks(void)
-{
-	if (ipa3_ctx->ipa3_hw_mode != IPA_HW_MODE_NORMAL) {
-		IPAERR("not supported in this mode\n");
-		return;
-	}
-
-	IPADBG("disabling IPA clocks and bus voting\n");
-
-	ipa3_ctx->ctrl->ipa3_disable_clks();
-
-	ipa_pm_set_clock_index(0);
-
-	if (msm_bus_scale_client_update_request(ipa3_ctx->ipa_bus_hdl, 0))
-		WARN(1, "bus scaling failed");
-	atomic_set(&ipa3_ctx->ipa_clk_vote, 0);
-}
-
-/**
- * ipa3_start_tag_process() - Send TAG packet and wait for it to come back
- *
- * This function is called prior to clock gating when active client counter
- * is 1. TAG process ensures that there are no packets inside IPA HW that
- * were not submitted to the IPA client via the transport. During TAG process
- * all aggregation frames are (force) closed.
- *
- * Return codes:
- * None
- */
-static void ipa3_start_tag_process(struct work_struct *work)
-{
-	int res;
-
-	IPADBG("starting TAG process\n");
-	/* close aggregation frames on all pipes */
-	res = ipa3_tag_aggr_force_close(-1);
-	if (res)
-		IPAERR("ipa3_tag_aggr_force_close failed %d\n", res);
-	IPA_ACTIVE_CLIENTS_DEC_SPECIAL("TAG_PROCESS");
-
-	IPADBG("TAG process done\n");
-}
-
-/**
- * ipa3_active_clients_log_mod() - Log a modification in the active clients
- * reference count
- *
- * This method logs any modification in the active clients reference count:
- * It logs the modification in the circular history buffer
- * It logs the modification in the hash table - looking for an entry,
- * creating one if needed and deleting one if needed.
- *
- * @id: ipa3_active client logging info struct to hold the log information
- * @inc: a boolean variable to indicate whether the modification is an increase
- * or decrease
- * @int_ctx: a boolean variable to indicate whether this call is being made from
- * an interrupt context and therefore should allocate GFP_ATOMIC memory
- *
- * Method process:
- * - Hash the unique identifier string
- * - Find the hash in the table
- *    1)If found, increase or decrease the reference count
- *    2)If not found, allocate a new hash table entry struct and initialize it
- * - Remove and deallocate unneeded data structure
- * - Log the call in the circular history buffer (unless it is a simple call)
- */
-#ifdef CONFIG_IPA_DEBUG
-static void ipa3_active_clients_log_mod(
-		struct ipa_active_client_logging_info *id,
-		bool inc, bool int_ctx)
-{
-	char temp_str[IPA3_ACTIVE_CLIENTS_LOG_LINE_LEN];
-	unsigned long long t;
-	unsigned long nanosec_rem;
-	struct ipa3_active_client_htable_entry *hentry;
-	struct ipa3_active_client_htable_entry *hfound;
-	u32 hkey;
-	char str_to_hash[IPA3_ACTIVE_CLIENTS_LOG_NAME_LEN];
-	unsigned long flags;
-
-	spin_lock_irqsave(&ipa3_ctx->ipa3_active_clients_logging.lock, flags);
-	int_ctx = true;
-	hfound = NULL;
-	memset(str_to_hash, 0, IPA3_ACTIVE_CLIENTS_LOG_NAME_LEN);
-	strlcpy(str_to_hash, id->id_string, IPA3_ACTIVE_CLIENTS_LOG_NAME_LEN);
-	hkey = jhash(str_to_hash, IPA3_ACTIVE_CLIENTS_LOG_NAME_LEN,
-			0);
-	hash_for_each_possible(ipa3_ctx->ipa3_active_clients_logging.htable,
-			hentry, list, hkey) {
-		if (!strcmp(hentry->id_string, id->id_string)) {
-			hentry->count = hentry->count + (inc ? 1 : -1);
-			hfound = hentry;
-		}
-	}
-	if (hfound == NULL) {
-		hentry = NULL;
-		hentry = kzalloc(sizeof(
-				struct ipa3_active_client_htable_entry),
-				int_ctx ? GFP_ATOMIC : GFP_KERNEL);
-		if (hentry == NULL) {
-			spin_unlock_irqrestore(
-				&ipa3_ctx->ipa3_active_clients_logging.lock,
-				flags);
-			return;
-		}
-		hentry->type = id->type;
-		strlcpy(hentry->id_string, id->id_string,
-				IPA3_ACTIVE_CLIENTS_LOG_NAME_LEN);
-		INIT_HLIST_NODE(&hentry->list);
-		hentry->count = inc ? 1 : -1;
-		hash_add(ipa3_ctx->ipa3_active_clients_logging.htable,
-				&hentry->list, hkey);
-	} else if (hfound->count == 0) {
-		hash_del(&hfound->list);
-		kfree(hfound);
-	}
-
-	if (id->type != SIMPLE) {
-		t = local_clock();
-		nanosec_rem = do_div(t, 1000000000) / 1000;
-		snprintf(temp_str, IPA3_ACTIVE_CLIENTS_LOG_LINE_LEN,
-				inc ? "[%5lu.%06lu] ^ %s, %s: %d" :
-						"[%5lu.%06lu] v %s, %s: %d",
-				(unsigned long)t, nanosec_rem,
-				id->id_string, id->file, id->line);
-		ipa3_active_clients_log_insert(temp_str);
-	}
-	spin_unlock_irqrestore(&ipa3_ctx->ipa3_active_clients_logging.lock,
-		flags);
-}
-#else
-static void ipa3_active_clients_log_mod(
-		struct ipa_active_client_logging_info *id,
-		bool inc, bool int_ctx)
-{
-}
-#endif
-
-void ipa3_active_clients_log_dec(struct ipa_active_client_logging_info *id,
-		bool int_ctx)
-{
-	ipa3_active_clients_log_mod(id, false, int_ctx);
-}
-
-void ipa3_active_clients_log_inc(struct ipa_active_client_logging_info *id,
-		bool int_ctx)
-{
-	ipa3_active_clients_log_mod(id, true, int_ctx);
-}
-
-/**
- * ipa3_inc_client_enable_clks() - Increase active clients counter, and
- * enable ipa clocks if necessary
- *
- * Return codes:
- * None
- */
-void ipa3_inc_client_enable_clks(struct ipa_active_client_logging_info *id)
-{
-	int ret;
-
-	ipa3_active_clients_log_inc(id, false);
-	ret = atomic_inc_not_zero(&ipa3_ctx->ipa3_active_clients.cnt);
-	if (ret) {
-		IPADBG_LOW("active clients = %d\n",
-			atomic_read(&ipa3_ctx->ipa3_active_clients.cnt));
-		return;
-	}
-
-	mutex_lock(&ipa3_ctx->ipa3_active_clients.mutex);
-
-	/* somebody might voted to clocks meanwhile */
-	ret = atomic_inc_not_zero(&ipa3_ctx->ipa3_active_clients.cnt);
-	if (ret) {
-		mutex_unlock(&ipa3_ctx->ipa3_active_clients.mutex);
-		IPADBG_LOW("active clients = %d\n",
-			atomic_read(&ipa3_ctx->ipa3_active_clients.cnt));
-		return;
-	}
-
-	ipa3_enable_clks();
-	ipa3_suspend_apps_pipes(false);
-	atomic_inc(&ipa3_ctx->ipa3_active_clients.cnt);
-	IPADBG_LOW("active clients = %d\n",
-		atomic_read(&ipa3_ctx->ipa3_active_clients.cnt));
-	mutex_unlock(&ipa3_ctx->ipa3_active_clients.mutex);
-}
-
-/**
- * ipa3_active_clks_status() - update the current msm bus clock vote
- * status
- */
-int ipa3_active_clks_status(void)
-{
-	return atomic_read(&ipa3_ctx->ipa_clk_vote);
-}
-
-/**
- * ipa3_inc_client_enable_clks_no_block() - Only increment the number of active
- * clients if no asynchronous actions should be done. Asynchronous actions are
- * locking a mutex and waking up IPA HW.
- *
- * Return codes: 0 for success
- *		-EPERM if an asynchronous action should have been done
- */
-int ipa3_inc_client_enable_clks_no_block(struct ipa_active_client_logging_info
-		*id)
-{
-	int ret;
-
-	ret = atomic_inc_not_zero(&ipa3_ctx->ipa3_active_clients.cnt);
-	if (ret) {
-		ipa3_active_clients_log_inc(id, true);
-		IPADBG_LOW("active clients = %d\n",
-			atomic_read(&ipa3_ctx->ipa3_active_clients.cnt));
-		return 0;
-	}
-
-	return -EPERM;
-}
-
-static void __ipa3_dec_client_disable_clks(void)
-{
-	int ret;
-
-	if (!atomic_read(&ipa3_ctx->ipa3_active_clients.cnt)) {
-		IPAERR("trying to disable clocks with refcnt is 0\n");
-		ipa_assert();
-		return;
-	}
-
-	ret = atomic_add_unless(&ipa3_ctx->ipa3_active_clients.cnt, -1, 1);
-	if (ret)
-		goto bail;
-
-	/* seems like this is the only client holding the clocks */
-	mutex_lock(&ipa3_ctx->ipa3_active_clients.mutex);
-	if (atomic_read(&ipa3_ctx->ipa3_active_clients.cnt) == 1 &&
-	    ipa3_ctx->tag_process_before_gating) {
-		ipa3_ctx->tag_process_before_gating = false;
-		/*
-		 * When TAG process ends, active clients will be
-		 * decreased
-		 */
-		queue_work(ipa3_ctx->power_mgmt_wq, &ipa3_tag_work);
-		goto unlock_mutex;
-	}
-
-	/* a different context might increase the clock reference meanwhile */
-	ret = atomic_sub_return(1, &ipa3_ctx->ipa3_active_clients.cnt);
-	if (ret > 0)
-		goto unlock_mutex;
-	ipa3_suspend_apps_pipes(true);
-	ipa3_disable_clks();
-
-unlock_mutex:
-	mutex_unlock(&ipa3_ctx->ipa3_active_clients.mutex);
-bail:
-	IPADBG_LOW("active clients = %d\n",
-		atomic_read(&ipa3_ctx->ipa3_active_clients.cnt));
-}
-
-/**
- * ipa3_dec_client_disable_clks() - Decrease active clients counter
- *
- * In case that there are no active clients this function also starts
- * TAG process. When TAG progress ends ipa clocks will be gated.
- * start_tag_process_again flag is set during this function to signal TAG
- * process to start again as there was another client that may send data to ipa
- *
- * Return codes:
- * None
- */
-void ipa3_dec_client_disable_clks(struct ipa_active_client_logging_info *id)
-{
-	ipa3_active_clients_log_dec(id, false);
-	__ipa3_dec_client_disable_clks();
-}
-
-static void ipa_dec_clients_disable_clks_on_wq(struct work_struct *work)
-{
-	__ipa3_dec_client_disable_clks();
-}
-
-/**
- * ipa3_dec_client_disable_clks_no_block() - Decrease active clients counter
- * if possible without blocking. If this is the last client then the desrease
- * will happen from work queue context.
- *
- * Return codes:
- * None
- */
-void ipa3_dec_client_disable_clks_no_block(
-	struct ipa_active_client_logging_info *id)
-{
-	int ret;
-
-	ipa3_active_clients_log_dec(id, true);
-	ret = atomic_add_unless(&ipa3_ctx->ipa3_active_clients.cnt, -1, 1);
-	if (ret) {
-		IPADBG_LOW("active clients = %d\n",
-			atomic_read(&ipa3_ctx->ipa3_active_clients.cnt));
-		return;
-	}
-
-	/* seems like this is the only client holding the clocks */
-	queue_work(ipa3_ctx->power_mgmt_wq,
-		&ipa_dec_clients_disable_clks_on_wq_work);
-}
-
-/**
- * ipa3_inc_acquire_wakelock() - Increase active clients counter, and
- * acquire wakelock if necessary
- *
- * Return codes:
- * None
- */
-void ipa3_inc_acquire_wakelock(void)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&ipa3_ctx->wakelock_ref_cnt.spinlock, flags);
-	ipa3_ctx->wakelock_ref_cnt.cnt++;
-	if (ipa3_ctx->wakelock_ref_cnt.cnt == 1)
-		__pm_stay_awake(&ipa3_ctx->w_lock);
-	IPADBG_LOW("active wakelock ref cnt = %d\n",
-		ipa3_ctx->wakelock_ref_cnt.cnt);
-	spin_unlock_irqrestore(&ipa3_ctx->wakelock_ref_cnt.spinlock, flags);
-}
-
-/**
- * ipa3_dec_release_wakelock() - Decrease active clients counter
- *
- * In case if the ref count is 0, release the wakelock.
- *
- * Return codes:
- * None
- */
-void ipa3_dec_release_wakelock(void)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&ipa3_ctx->wakelock_ref_cnt.spinlock, flags);
-	ipa3_ctx->wakelock_ref_cnt.cnt--;
-	IPADBG_LOW("active wakelock ref cnt = %d\n",
-		ipa3_ctx->wakelock_ref_cnt.cnt);
-	if (ipa3_ctx->wakelock_ref_cnt.cnt == 0)
-		__pm_relax(&ipa3_ctx->w_lock);
-	spin_unlock_irqrestore(&ipa3_ctx->wakelock_ref_cnt.spinlock, flags);
-}
-
-int ipa3_set_clock_plan_from_pm(int idx)
-{
-	u32 clk_rate;
-
-	IPADBG_LOW("idx = %d\n", idx);
-
-	if (!ipa3_ctx->enable_clock_scaling) {
-		ipa3_ctx->ipa3_active_clients.bus_vote_idx = idx;
-		return 0;
-	}
-
-	if (ipa3_ctx->ipa3_hw_mode != IPA_HW_MODE_NORMAL) {
-		IPAERR("not supported in this mode\n");
-		return 0;
-	}
-
-	if (idx <= 0 || idx >= ipa3_ctx->ctrl->msm_bus_data_ptr->num_usecases) {
-		IPAERR("bad voltage\n");
-		return -EINVAL;
-	}
-
-	if (idx == 1)
-		clk_rate = ipa3_ctx->ctrl->ipa_clk_rate_svs2;
-	else if (idx == 2)
-		clk_rate = ipa3_ctx->ctrl->ipa_clk_rate_svs;
-	else if (idx == 3)
-		clk_rate = ipa3_ctx->ctrl->ipa_clk_rate_nominal;
-	else if (idx == 4)
-		clk_rate = ipa3_ctx->ctrl->ipa_clk_rate_turbo;
-	else {
-		IPAERR("bad voltage\n");
-		WARN_ON(1);
-		return -EFAULT;
-	}
-
-	if (clk_rate == ipa3_ctx->curr_ipa_clk_rate) {
-		IPADBG_LOW("Same voltage\n");
-		return 0;
-	}
-
-	mutex_lock(&ipa3_ctx->ipa3_active_clients.mutex);
-	ipa3_ctx->curr_ipa_clk_rate = clk_rate;
-	ipa3_ctx->ipa3_active_clients.bus_vote_idx = idx;
-	IPADBG_LOW("setting clock rate to %u\n", ipa3_ctx->curr_ipa_clk_rate);
-	if (atomic_read(&ipa3_ctx->ipa3_active_clients.cnt) > 0) {
-		if (ipa3_clk)
-			clk_set_rate(ipa3_clk, ipa3_ctx->curr_ipa_clk_rate);
-		if (msm_bus_scale_client_update_request(ipa3_ctx->ipa_bus_hdl,
-				ipa3_get_bus_vote()))
-			WARN_ON(1);
-	} else {
-		IPADBG_LOW("clocks are gated, not setting rate\n");
-	}
-	mutex_unlock(&ipa3_ctx->ipa3_active_clients.mutex);
-	IPADBG_LOW("Done\n");
-
-	return 0;
-}
-
-int ipa3_set_required_perf_profile(enum ipa_voltage_level floor_voltage,
-				  u32 bandwidth_mbps)
-{
-	enum ipa_voltage_level needed_voltage;
-	u32 clk_rate;
-
-	if (ipa3_ctx->ipa3_hw_mode != IPA_HW_MODE_NORMAL) {
-		IPAERR("not supported in this mode\n");
-		return 0;
-	}
-
-	IPADBG_LOW("floor_voltage=%d, bandwidth_mbps=%u",
-					floor_voltage, bandwidth_mbps);
-
-	if (floor_voltage < IPA_VOLTAGE_UNSPECIFIED ||
-		floor_voltage >= IPA_VOLTAGE_MAX) {
-		IPAERR("bad voltage\n");
-		return -EINVAL;
-	}
-
-	if (ipa3_ctx->enable_clock_scaling) {
-		IPADBG_LOW("Clock scaling is enabled\n");
-		if (bandwidth_mbps >=
-			ipa3_ctx->ctrl->clock_scaling_bw_threshold_turbo)
-			needed_voltage = IPA_VOLTAGE_TURBO;
-		else if (bandwidth_mbps >=
-			ipa3_ctx->ctrl->clock_scaling_bw_threshold_nominal)
-			needed_voltage = IPA_VOLTAGE_NOMINAL;
-		else if (bandwidth_mbps >=
-			ipa3_ctx->ctrl->clock_scaling_bw_threshold_svs)
-			needed_voltage = IPA_VOLTAGE_SVS;
-		else
-			needed_voltage = IPA_VOLTAGE_SVS2;
-	} else {
-		IPADBG_LOW("Clock scaling is disabled\n");
-		needed_voltage = IPA_VOLTAGE_NOMINAL;
-	}
-
-	needed_voltage = max(needed_voltage, floor_voltage);
-	switch (needed_voltage) {
-	case IPA_VOLTAGE_SVS2:
-		clk_rate = ipa3_ctx->ctrl->ipa_clk_rate_svs2;
-		break;
-	case IPA_VOLTAGE_SVS:
-		clk_rate = ipa3_ctx->ctrl->ipa_clk_rate_svs;
-		break;
-	case IPA_VOLTAGE_NOMINAL:
-		clk_rate = ipa3_ctx->ctrl->ipa_clk_rate_nominal;
-		break;
-	case IPA_VOLTAGE_TURBO:
-		clk_rate = ipa3_ctx->ctrl->ipa_clk_rate_turbo;
-		break;
-	default:
-		IPAERR("bad voltage\n");
-		WARN_ON(1);
-		return -EFAULT;
-	}
-
-	if (clk_rate == ipa3_ctx->curr_ipa_clk_rate) {
-		IPADBG_LOW("Same voltage\n");
-		return 0;
-	}
-
-	/* Hold the mutex to avoid race conditions with ipa3_enable_clocks() */
-	mutex_lock(&ipa3_ctx->ipa3_active_clients.mutex);
-	ipa3_ctx->curr_ipa_clk_rate = clk_rate;
-	IPADBG_LOW("setting clock rate to %u\n", ipa3_ctx->curr_ipa_clk_rate);
-	if (atomic_read(&ipa3_ctx->ipa3_active_clients.cnt) > 0) {
-		if (ipa3_clk)
-			clk_set_rate(ipa3_clk, ipa3_ctx->curr_ipa_clk_rate);
-		if (msm_bus_scale_client_update_request(ipa3_ctx->ipa_bus_hdl,
-				ipa3_get_bus_vote()))
-			WARN_ON(1);
-	} else {
-		IPADBG_LOW("clocks are gated, not setting rate\n");
-	}
-	mutex_unlock(&ipa3_ctx->ipa3_active_clients.mutex);
-	IPADBG_LOW("Done\n");
-
-	return 0;
-}
-
-static void ipa3_process_irq_schedule_rel(void)
-{
-	queue_delayed_work(ipa3_ctx->transport_power_mgmt_wq,
-		&ipa3_transport_release_resource_work,
-		msecs_to_jiffies(IPA_TRANSPORT_PROD_TIMEOUT_MSEC));
-}
-
-/**
- * ipa3_suspend_handler() - Handles the suspend interrupt:
- * wakes up the suspended peripheral by requesting its consumer
- * @interrupt:		Interrupt type
- * @private_data:	The client's private data
- * @interrupt_data:	Interrupt specific information data
- */
-void ipa3_suspend_handler(enum ipa_irq_type interrupt,
-				void *private_data,
-				void *interrupt_data)
-{
-	u32 suspend_data =
-		((struct ipa_tx_suspend_irq_data *)interrupt_data)->endpoints;
-	u32 bmsk = 1;
-	u32 i = 0;
-	int res;
-	struct ipa_ep_cfg_holb holb_cfg;
-	u32 pipe_bitmask = 0;
-
-	IPADBG("interrupt=%d, interrupt_data=%u\n",
-		interrupt, suspend_data);
-	memset(&holb_cfg, 0, sizeof(holb_cfg));
-
-	for (i = 0; i < ipa3_ctx->ipa_num_pipes; i++, bmsk = bmsk << 1) {
-		if ((suspend_data & bmsk) && (ipa3_ctx->ep[i].valid)) {
-				pipe_bitmask |= bmsk;
-		}
-	}
-	res = ipa_pm_handle_suspend(pipe_bitmask);
-	if (res) {
-		IPAERR("ipa_pm_handle_suspend failed %d\n", res);
-		return;
-	}
-}
-
-/**
- * ipa3_restore_suspend_handler() - restores the original suspend IRQ handler
- * as it was registered in the IPA init sequence.
- * Return codes:
- * 0: success
- * -EPERM: failed to remove current handler or failed to add original handler
- */
-int ipa3_restore_suspend_handler(void)
-{
-	int result = 0;
-
-	result  = ipa3_remove_interrupt_handler(IPA_TX_SUSPEND_IRQ);
-	if (result) {
-		IPAERR("remove handler for suspend interrupt failed\n");
-		return -EPERM;
-	}
-
-	result = ipa3_add_interrupt_handler(IPA_TX_SUSPEND_IRQ,
-			ipa3_suspend_handler, false, NULL);
-	if (result) {
-		IPAERR("register handler for suspend interrupt failed\n");
-		result = -EPERM;
-	}
-
-	IPADBG("suspend handler successfully restored\n");
-
-	return result;
-}
-
-static int ipa3_apps_cons_release_resource(void)
-{
-	return 0;
-}
-
-static int ipa3_apps_cons_request_resource(void)
-{
-	return 0;
-}
-
-static void ipa3_transport_release_resource(struct work_struct *work)
-{
-	mutex_lock(&ipa3_ctx->transport_pm.transport_pm_mutex);
-	/* check whether still need to decrease client usage */
-	if (atomic_read(&ipa3_ctx->transport_pm.dec_clients)) {
-		if (atomic_read(&ipa3_ctx->transport_pm.eot_activity)) {
-			IPADBG("EOT pending Re-scheduling\n");
-			ipa3_process_irq_schedule_rel();
-		} else {
-			atomic_set(&ipa3_ctx->transport_pm.dec_clients, 0);
-			ipa3_dec_release_wakelock();
-			IPA_ACTIVE_CLIENTS_DEC_SPECIAL("TRANSPORT_RESOURCE");
-		}
-	}
-	atomic_set(&ipa3_ctx->transport_pm.eot_activity, 0);
-	mutex_unlock(&ipa3_ctx->transport_pm.transport_pm_mutex);
-}
-
-int ipa3_create_apps_resource(void)
-{
-	struct ipa_rm_create_params apps_cons_create_params;
-	struct ipa_rm_perf_profile profile;
-	int result = 0;
-
-	memset(&apps_cons_create_params, 0,
-				sizeof(apps_cons_create_params));
-	apps_cons_create_params.name = IPA_RM_RESOURCE_APPS_CONS;
-	apps_cons_create_params.request_resource =
-		ipa3_apps_cons_request_resource;
-	apps_cons_create_params.release_resource =
-		ipa3_apps_cons_release_resource;
-	result = ipa_rm_create_resource(&apps_cons_create_params);
-	if (result) {
-		IPAERR("ipa_rm_create_resource failed\n");
-		return result;
-	}
-
-	profile.max_supported_bandwidth_mbps = IPA_APPS_MAX_BW_IN_MBPS;
-	ipa_rm_set_perf_profile(IPA_RM_RESOURCE_APPS_CONS, &profile);
-
-	return result;
-}
-
-/**
- * ipa3_init_interrupts() - Register to IPA IRQs
- *
- * Return codes: 0 in success, negative in failure
- *
- */
-int ipa3_init_interrupts(void)
-{
-	int result;
-
-	/*register IPA IRQ handler*/
-	result = ipa3_interrupts_init(ipa3_res.ipa_irq, 0,
-			&ipa3_ctx->master_pdev->dev);
-	if (result) {
-		IPAERR("ipa interrupts initialization failed\n");
-		return -ENODEV;
-	}
-
-	/*add handler for suspend interrupt*/
-	result = ipa3_add_interrupt_handler(IPA_TX_SUSPEND_IRQ,
-			ipa3_suspend_handler, false, NULL);
-	if (result) {
-		IPAERR("register handler for suspend interrupt failed\n");
-		result = -ENODEV;
-		goto fail_add_interrupt_handler;
-	}
-
-	return 0;
-
-fail_add_interrupt_handler:
-	ipa3_interrupts_destroy(ipa3_res.ipa_irq, &ipa3_ctx->master_pdev->dev);
-	return result;
-}
-
-/**
- * ipa3_destroy_flt_tbl_idrs() - destroy the idr structure for flt tables
- *  The idr strcuture per filtering table is intended for rule id generation
- *  per filtering rule.
- */
-static void ipa3_destroy_flt_tbl_idrs(void)
-{
-	int i;
-	struct ipa3_flt_tbl *flt_tbl;
-
-	idr_destroy(&ipa3_ctx->flt_rule_ids[IPA_IP_v4]);
-	idr_destroy(&ipa3_ctx->flt_rule_ids[IPA_IP_v6]);
-
-	for (i = 0; i < ipa3_ctx->ipa_num_pipes; i++) {
-		if (!ipa_is_ep_support_flt(i))
-			continue;
-
-		flt_tbl = &ipa3_ctx->flt_tbl[i][IPA_IP_v4];
-		flt_tbl->rule_ids = NULL;
-		flt_tbl = &ipa3_ctx->flt_tbl[i][IPA_IP_v6];
-		flt_tbl->rule_ids = NULL;
-	}
-}
-
-static void ipa3_freeze_clock_vote_and_notify_modem(void)
-{
-	int res;
-	struct ipa_active_client_logging_info log_info;
-
-	if (ipa3_ctx->platform_type == IPA_PLAT_TYPE_APQ) {
-		IPADBG("Ignore smp2p on APQ platform\n");
-		return;
-	}
-
-	if (ipa3_ctx->smp2p_info.res_sent)
-		return;
-
-	if (IS_ERR(ipa3_ctx->smp2p_info.smem_state)) {
-		IPAERR("fail to get smp2p clk resp bit %ld\n",
-			PTR_ERR(ipa3_ctx->smp2p_info.smem_state));
-		return;
-	}
-
-	IPA_ACTIVE_CLIENTS_PREP_SPECIAL(log_info, "FREEZE_VOTE");
-	res = ipa3_inc_client_enable_clks_no_block(&log_info);
-	if (res)
-		ipa3_ctx->smp2p_info.ipa_clk_on = false;
-	else
-		ipa3_ctx->smp2p_info.ipa_clk_on = true;
-
-	qcom_smem_state_update_bits(ipa3_ctx->smp2p_info.smem_state,
-			IPA_SMP2P_SMEM_STATE_MASK,
-			((ipa3_ctx->smp2p_info.ipa_clk_on <<
-			IPA_SMP2P_OUT_CLK_VOTE_IDX) |
-			(1 << IPA_SMP2P_OUT_CLK_RSP_CMPLT_IDX)));
-
-	ipa3_ctx->smp2p_info.res_sent = true;
-	IPADBG("IPA clocks are %s\n",
-		ipa3_ctx->smp2p_info.ipa_clk_on ? "ON" : "OFF");
-}
-
-void ipa3_reset_freeze_vote(void)
-{
-	if (!ipa3_ctx->smp2p_info.res_sent)
-		return;
-
-	if (ipa3_ctx->smp2p_info.ipa_clk_on)
-		IPA_ACTIVE_CLIENTS_DEC_SPECIAL("FREEZE_VOTE");
-
-	qcom_smem_state_update_bits(ipa3_ctx->smp2p_info.smem_state,
-			IPA_SMP2P_SMEM_STATE_MASK,
-			((0 <<
-			IPA_SMP2P_OUT_CLK_VOTE_IDX) |
-			(0 << IPA_SMP2P_OUT_CLK_RSP_CMPLT_IDX)));
-
-	ipa3_ctx->smp2p_info.res_sent = false;
-	ipa3_ctx->smp2p_info.ipa_clk_on = false;
-}
-
-static int ipa3_panic_notifier(struct notifier_block *this,
-	unsigned long event, void *ptr)
-{
-	int res;
-
-	ipa3_freeze_clock_vote_and_notify_modem();
-
-	IPADBG("Calling uC panic handler\n");
-	res = ipa3_uc_panic_notifier(this, event, ptr);
-	if (res)
-		IPAERR("uC panic handler failed %d\n", res);
-
-	if (atomic_read(&ipa3_ctx->ipa_clk_vote)) {
-		ipahal_print_all_regs(false);
-		ipa_save_registers();
-		ipa_wigig_save_regs();
-	}
-
-	return NOTIFY_DONE;
-}
-
-static struct notifier_block ipa3_panic_blk = {
-	.notifier_call = ipa3_panic_notifier,
-	/* IPA panic handler needs to run before modem shuts down */
-	.priority = INT_MAX,
-};
-
-static void ipa3_register_panic_hdlr(void)
-{
-	atomic_notifier_chain_register(&panic_notifier_list,
-		&ipa3_panic_blk);
-}
-
-static void ipa3_trigger_ipa_ready_cbs(void)
-{
-	struct ipa3_ready_cb_info *info;
-
-	mutex_lock(&ipa3_ctx->lock);
-
-	/* Call all the CBs */
-	list_for_each_entry(info, &ipa3_ctx->ipa_ready_cb_list, link)
-		if (info->ready_cb)
-			info->ready_cb(info->user_data);
-
-	mutex_unlock(&ipa3_ctx->lock);
-}
-
-static void ipa3_uc_is_loaded(void)
-{
-	IPADBG("\n");
-	complete_all(&ipa3_ctx->uc_loaded_completion_obj);
-}
-
-static enum gsi_ver ipa3_get_gsi_ver(enum ipa_hw_type ipa_hw_type)
-{
-	enum gsi_ver gsi_ver;
-
-	switch (ipa_hw_type) {
-	case IPA_HW_v3_0:
-	case IPA_HW_v3_1:
-		gsi_ver = GSI_VER_1_0;
-		break;
-	case IPA_HW_v3_5:
-		gsi_ver = GSI_VER_1_2;
-		break;
-	case IPA_HW_v3_5_1:
-		gsi_ver = GSI_VER_1_3;
-		break;
-	case IPA_HW_v4_0:
-	case IPA_HW_v4_1:
-		gsi_ver = GSI_VER_2_0;
-		break;
-	case IPA_HW_v4_2:
-		gsi_ver = GSI_VER_2_2;
-		break;
-	case IPA_HW_v4_5:
-		gsi_ver = GSI_VER_2_5;
-		break;
-	case IPA_HW_v4_7:
-		gsi_ver = GSI_VER_2_7;
-		break;
-	default:
-		IPAERR("No GSI version for ipa type %d\n", ipa_hw_type);
-		WARN_ON(1);
-		gsi_ver = GSI_VER_ERR;
-	}
-
-	IPADBG("GSI version %d\n", gsi_ver);
-
-	return gsi_ver;
-}
-
-static int ipa3_gsi_pre_fw_load_init(void)
-{
-	int result;
-
-	result = gsi_configure_regs(
-		ipa3_res.ipa_mem_base,
-		ipa3_get_gsi_ver(ipa3_res.ipa_hw_type));
-
-	if (result) {
-		IPAERR("Failed to configure GSI registers\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int ipa3_alloc_gsi_channel(void)
-{
-	const struct ipa_gsi_ep_config *gsi_ep_cfg;
-	enum ipa_client_type type;
-	int code = 0;
-	int ret = 0;
-	int i;
-
-	for (i = 0; i < ipa3_ctx->ipa_num_pipes; i++) {
-		type = ipa3_get_client_by_pipe(i);
-		gsi_ep_cfg = ipa3_get_gsi_ep_info(type);
-		IPADBG("for ep %d client is %d\n", i, type);
-		if (!gsi_ep_cfg)
-			continue;
-
-		ret = gsi_alloc_channel_ee(gsi_ep_cfg->ipa_gsi_chan_num,
-					gsi_ep_cfg->ee, &code);
-		if (ret == GSI_STATUS_SUCCESS) {
-			IPADBG("alloc gsi ch %d ee %d with code %d\n",
-					gsi_ep_cfg->ipa_gsi_chan_num,
-					gsi_ep_cfg->ee,
-					code);
-		} else {
-			IPAERR("failed to alloc ch %d ee %d code %d\n",
-					gsi_ep_cfg->ipa_gsi_chan_num,
-					gsi_ep_cfg->ee,
-					code);
-			return ret;
-		}
-	}
-	return ret;
-}
-/**
- * ipa3_post_init() - Initialize the IPA Driver (Part II).
- * This part contains all initialization which requires interaction with
- * IPA HW (via GSI).
- *
- * @resource_p:	contain platform specific values from DST file
- * @pdev:	The platform device structure representing the IPA driver
- *
- * Function initialization process:
- * - Initialize endpoints bitmaps
- * - Initialize resource groups min and max values
- * - Initialize filtering lists heads and idr
- * - Initialize interrupts
- * - Register GSI
- * - Setup APPS pipes
- * - Initialize tethering bridge
- * - Initialize IPA debugfs
- * - Initialize IPA uC interface
- * - Initialize WDI interface
- * - Initialize USB interface
- * - Register for panic handler
- * - Trigger IPA ready callbacks (to all subscribers)
- * - Trigger IPA completion object (to all who wait on it)
- */
-static int ipa3_post_init(const struct ipa3_plat_drv_res *resource_p,
-			  struct device *ipa_dev)
-{
-	int result;
-	struct gsi_per_props gsi_props;
-	struct ipa3_uc_hdlrs uc_hdlrs = { 0 };
-	struct ipa3_flt_tbl *flt_tbl;
-	int i;
-	struct idr *idr;
-
-	if (ipa3_ctx == NULL) {
-		IPADBG("IPA driver haven't initialized\n");
-		return -ENXIO;
-	}
-
-	/* Prevent consequent calls from trying to load the FW again. */
-	if (ipa3_ctx->ipa_initialization_complete)
-		return 0;
-
-	IPADBG("active clients = %d\n",
-			atomic_read(&ipa3_ctx->ipa3_active_clients.cnt));
-	/* move proxy vote for modem on ipa3_post_init */
-	if (ipa3_ctx->ipa_hw_type != IPA_HW_v4_0)
-		ipa3_proxy_clk_vote();
-
-	/* The following will retrieve and save the gsi fw version */
-	ipa_save_gsi_ver();
-
-	if (ipahal_init(ipa3_ctx->ipa_hw_type, ipa3_ctx->mmio,
-		ipa3_ctx->pdev)) {
-		IPAERR("fail to init ipahal\n");
-		result = -EFAULT;
-		goto fail_ipahal;
-	}
-
-	result = ipa3_init_hw();
-	if (result) {
-		IPAERR(":error initializing HW\n");
-		result = -ENODEV;
-		goto fail_init_hw;
-	}
-	IPADBG("IPA HW initialization sequence completed");
-
-	ipa3_ctx->ipa_num_pipes = ipa3_get_num_pipes();
-	IPADBG("IPA Pipes num %u\n", ipa3_ctx->ipa_num_pipes);
-	if (ipa3_ctx->ipa_num_pipes > IPA3_MAX_NUM_PIPES) {
-		IPAERR("IPA has more pipes then supported has %d, max %d\n",
-			ipa3_ctx->ipa_num_pipes, IPA3_MAX_NUM_PIPES);
-		result = -ENODEV;
-		goto fail_init_hw;
-	}
-
-	ipa3_ctx->ctrl->ipa_sram_read_settings();
-	IPADBG("SRAM, size: 0x%x, restricted bytes: 0x%x\n",
-		ipa3_ctx->smem_sz, ipa3_ctx->smem_restricted_bytes);
-
-	IPADBG("hdr_lcl=%u ip4_rt_hash=%u ip4_rt_nonhash=%u\n",
-		ipa3_ctx->hdr_tbl_lcl, ipa3_ctx->ip4_rt_tbl_hash_lcl,
-		ipa3_ctx->ip4_rt_tbl_nhash_lcl);
-
-	IPADBG("ip6_rt_hash=%u ip6_rt_nonhash=%u\n",
-		ipa3_ctx->ip6_rt_tbl_hash_lcl, ipa3_ctx->ip6_rt_tbl_nhash_lcl);
-
-	IPADBG("ip4_flt_hash=%u ip4_flt_nonhash=%u\n",
-		ipa3_ctx->ip4_flt_tbl_hash_lcl,
-		ipa3_ctx->ip4_flt_tbl_nhash_lcl);
-
-	IPADBG("ip6_flt_hash=%u ip6_flt_nonhash=%u\n",
-		ipa3_ctx->ip6_flt_tbl_hash_lcl,
-		ipa3_ctx->ip6_flt_tbl_nhash_lcl);
-
-	if (ipa3_ctx->smem_reqd_sz > ipa3_ctx->smem_sz) {
-		IPAERR("SW expect more core memory, needed %d, avail %d\n",
-			ipa3_ctx->smem_reqd_sz, ipa3_ctx->smem_sz);
-		result = -ENOMEM;
-		goto fail_init_hw;
-	}
-
-	result = ipa3_allocate_dma_task_for_gsi();
-	if (result) {
-		IPAERR("failed to allocate dma task\n");
-		goto fail_dma_task;
-	}
-
-	if (ipa3_nat_ipv6ct_init_devices()) {
-		IPAERR("unable to init NAT and IPv6CT devices\n");
-		result = -ENODEV;
-		goto fail_nat_ipv6ct_init_dev;
-	}
-
-	result = ipa3_alloc_pkt_init();
-	if (result) {
-		IPAERR("Failed to alloc pkt_init payload\n");
-		result = -ENODEV;
-		goto fail_allok_pkt_init;
-	}
-
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v3_5)
-		ipa3_enable_dcd();
-
-	/*
-	 * indication whether working in MHI config or non MHI config is given
-	 * in ipa3_write which is launched before ipa3_post_init. i.e. from
-	 * this point it is safe to use ipa3_ep_mapping array and the correct
-	 * entry will be returned from ipa3_get_hw_type_index()
-	 */
-	ipa_init_ep_flt_bitmap();
-	IPADBG("EP with flt support bitmap 0x%x (%u pipes)\n",
-		ipa3_ctx->ep_flt_bitmap, ipa3_ctx->ep_flt_num);
-
-	/* Assign resource limitation to each group */
-	ipa3_set_resorce_groups_min_max_limits();
-
-	idr = &(ipa3_ctx->flt_rule_ids[IPA_IP_v4]);
-	idr_init(idr);
-	idr = &(ipa3_ctx->flt_rule_ids[IPA_IP_v6]);
-	idr_init(idr);
-
-	for (i = 0; i < ipa3_ctx->ipa_num_pipes; i++) {
-		if (!ipa_is_ep_support_flt(i))
-			continue;
-
-		flt_tbl = &ipa3_ctx->flt_tbl[i][IPA_IP_v4];
-		INIT_LIST_HEAD(&flt_tbl->head_flt_rule_list);
-		flt_tbl->in_sys[IPA_RULE_HASHABLE] =
-			!ipa3_ctx->ip4_flt_tbl_hash_lcl;
-		flt_tbl->in_sys[IPA_RULE_NON_HASHABLE] =
-			!ipa3_ctx->ip4_flt_tbl_nhash_lcl;
-		flt_tbl->rule_ids = &ipa3_ctx->flt_rule_ids[IPA_IP_v4];
-
-		flt_tbl = &ipa3_ctx->flt_tbl[i][IPA_IP_v6];
-		INIT_LIST_HEAD(&flt_tbl->head_flt_rule_list);
-		flt_tbl->in_sys[IPA_RULE_HASHABLE] =
-			!ipa3_ctx->ip6_flt_tbl_hash_lcl;
-		flt_tbl->in_sys[IPA_RULE_NON_HASHABLE] =
-			!ipa3_ctx->ip6_flt_tbl_nhash_lcl;
-		flt_tbl->rule_ids = &ipa3_ctx->flt_rule_ids[IPA_IP_v6];
-	}
-
-	result = ipa3_init_interrupts();
-	if (result) {
-		IPAERR("ipa initialization of interrupts failed\n");
-		result = -ENODEV;
-		goto fail_init_interrupts;
-	}
-
-	/*
-	 * Disable prefetch for USB or MHI at IPAv3.5/IPA.3.5.1
-	 * This is to allow MBIM to work.
-	 */
-	if ((ipa3_ctx->ipa_hw_type >= IPA_HW_v3_5
-		&& ipa3_ctx->ipa_hw_type < IPA_HW_v4_0) &&
-		(!ipa3_ctx->ipa_config_is_mhi))
-		ipa3_disable_prefetch(IPA_CLIENT_USB_CONS);
-
-	if ((ipa3_ctx->ipa_hw_type >= IPA_HW_v3_5
-		&& ipa3_ctx->ipa_hw_type < IPA_HW_v4_0) &&
-		(ipa3_ctx->ipa_config_is_mhi))
-		ipa3_disable_prefetch(IPA_CLIENT_MHI_CONS);
-
-	memset(&gsi_props, 0, sizeof(gsi_props));
-	gsi_props.ver = ipa3_get_gsi_ver(resource_p->ipa_hw_type);
-	gsi_props.ee = resource_p->ee;
-	gsi_props.intr = GSI_INTR_IRQ;
-	gsi_props.phys_addr = resource_p->transport_mem_base;
-	gsi_props.size = resource_p->transport_mem_size;
-	if (ipa3_ctx->ipa3_hw_mode == IPA_HW_MODE_EMULATION) {
-		gsi_props.irq = resource_p->emulator_irq;
-		gsi_props.emulator_intcntrlr_client_isr = ipa3_get_isr();
-		gsi_props.emulator_intcntrlr_addr =
-		    resource_p->emulator_intcntrlr_mem_base;
-		gsi_props.emulator_intcntrlr_size =
-		    resource_p->emulator_intcntrlr_mem_size;
-	} else {
-		gsi_props.irq = resource_p->transport_irq;
-	}
-	gsi_props.notify_cb = ipa_gsi_notify_cb;
-	gsi_props.req_clk_cb = NULL;
-	gsi_props.rel_clk_cb = NULL;
-	gsi_props.clk_status_cb = ipa3_active_clks_status;
-
-	if (ipa3_ctx->ipa_config_is_mhi) {
-		gsi_props.mhi_er_id_limits_valid = true;
-		gsi_props.mhi_er_id_limits[0] = resource_p->mhi_evid_limits[0];
-		gsi_props.mhi_er_id_limits[1] = resource_p->mhi_evid_limits[1];
-	}
-
-	result = gsi_register_device(&gsi_props,
-		&ipa3_ctx->gsi_dev_hdl);
-	if (result != GSI_STATUS_SUCCESS) {
-		IPAERR(":gsi register error - %d\n", result);
-		result = -ENODEV;
-		goto fail_register_device;
-	}
-	IPADBG("IPA gsi is registered\n");
-	/* GSI 2.2 requires to allocate all EE GSI channel
-	 * during device bootup.
-	 */
-	if (ipa3_get_gsi_ver(resource_p->ipa_hw_type) == GSI_VER_2_2) {
-		result = ipa3_alloc_gsi_channel();
-		if (result) {
-			IPAERR("Failed to alloc the GSI channels\n");
-			result = -ENODEV;
-			goto fail_alloc_gsi_channel;
-		}
-	}
-
-	/* setup the AP-IPA pipes */
-	if (ipa3_setup_apps_pipes()) {
-		IPAERR(":failed to setup IPA-Apps pipes\n");
-		result = -ENODEV;
-		goto fail_setup_apps_pipes;
-	}
-	IPADBG("IPA GPI pipes were connected\n");
-
-	if (ipa3_ctx->use_ipa_teth_bridge) {
-		/* Initialize the tethering bridge driver */
-		result = ipa3_teth_bridge_driver_init();
-		if (result) {
-			IPAERR(":teth_bridge init failed (%d)\n", -result);
-			result = -ENODEV;
-			goto fail_teth_bridge_driver_init;
-		}
-		IPADBG("teth_bridge initialized");
-	}
-
-	result = ipa3_uc_interface_init();
-	if (result)
-		IPAERR(":ipa Uc interface init failed (%d)\n", -result);
-	else
-		IPADBG(":ipa Uc interface init ok\n");
-
-	uc_hdlrs.ipa_uc_loaded_hdlr = ipa3_uc_is_loaded;
-	ipa3_uc_register_handlers(IPA_HW_FEATURE_COMMON, &uc_hdlrs);
-
-	result = ipa3_wdi_init();
-	if (result)
-		IPAERR(":wdi init failed (%d)\n", -result);
-	else
-		IPADBG(":wdi init ok\n");
-
-	result = ipa3_wigig_init_i();
-	if (result)
-		IPAERR(":wigig init failed (%d)\n", -result);
-	else
-		IPADBG(":wigig init ok\n");
-
-	result = ipa3_ntn_init();
-	if (result)
-		IPAERR(":ntn init failed (%d)\n", -result);
-	else
-		IPADBG(":ntn init ok\n");
-
-	result = ipa_hw_stats_init();
-	if (result)
-		IPAERR("fail to init stats %d\n", result);
-	else
-		IPADBG(":stats init ok\n");
-
-	ipa3_register_panic_hdlr();
-
-	ipa3_debugfs_init();
-
-	mutex_lock(&ipa3_ctx->lock);
-	ipa3_ctx->ipa_initialization_complete = true;
-	mutex_unlock(&ipa3_ctx->lock);
-
-	ipa3_trigger_ipa_ready_cbs();
-	complete_all(&ipa3_ctx->init_completion_obj);
-	pr_info("IPA driver initialization was successful.\n");
-
-	return 0;
-
-fail_teth_bridge_driver_init:
-	ipa3_teardown_apps_pipes();
-fail_alloc_gsi_channel:
-fail_setup_apps_pipes:
-	gsi_deregister_device(ipa3_ctx->gsi_dev_hdl, false);
-fail_register_device:
-	ipa3_destroy_flt_tbl_idrs();
-fail_init_interrupts:
-	 ipa3_remove_interrupt_handler(IPA_TX_SUSPEND_IRQ);
-	 ipa3_interrupts_destroy(ipa3_res.ipa_irq, &ipa3_ctx->master_pdev->dev);
-fail_allok_pkt_init:
-	ipa3_nat_ipv6ct_destroy_devices();
-fail_nat_ipv6ct_init_dev:
-	ipa3_free_dma_task_for_gsi();
-fail_dma_task:
-fail_init_hw:
-	ipahal_destroy();
-fail_ipahal:
-	ipa3_proxy_clk_unvote();
-
-	return result;
-}
-
-static int ipa3_manual_load_ipa_fws(void)
-{
-	int result;
-	const struct firmware *fw;
-	const char *path = IPA_FWS_PATH;
-
-	if (ipa3_ctx->ipa3_hw_mode == IPA_HW_MODE_EMULATION) {
-		switch (ipa3_get_emulation_type()) {
-		case IPA_HW_v3_5_1:
-			path = IPA_FWS_PATH_3_5_1;
-			break;
-		case IPA_HW_v4_0:
-			path = IPA_FWS_PATH_4_0;
-			break;
-		case IPA_HW_v4_5:
-			path = IPA_FWS_PATH_4_5;
-			break;
-		default:
-			break;
-		}
-	}
-
-	IPADBG("Manual FW loading (%s) process initiated\n", path);
-
-	result = request_firmware(&fw, path, ipa3_ctx->cdev.dev);
-	if (result < 0) {
-		IPAERR("request_firmware failed, error %d\n", result);
-		return result;
-	}
-
-	IPADBG("FWs are available for loading\n");
-
-	if (ipa3_ctx->ipa3_hw_mode == IPA_HW_MODE_EMULATION) {
-		result = emulator_load_fws(fw,
-			ipa3_res.transport_mem_base,
-			ipa3_res.transport_mem_size,
-			ipa3_get_gsi_ver(ipa3_res.ipa_hw_type));
-	} else {
-		result = ipa3_load_fws(fw, ipa3_res.transport_mem_base,
-			ipa3_get_gsi_ver(ipa3_res.ipa_hw_type));
-	}
-
-	if (result) {
-		IPAERR("Manual IPA FWs loading has failed\n");
-		release_firmware(fw);
-		return result;
-	}
-
-	result = gsi_enable_fw(ipa3_res.transport_mem_base,
-				ipa3_res.transport_mem_size,
-				ipa3_get_gsi_ver(ipa3_res.ipa_hw_type));
-	if (result) {
-		IPAERR("Failed to enable GSI FW\n");
-		release_firmware(fw);
-		return result;
-	}
-
-	release_firmware(fw);
-
-	IPADBG("Manual FW loading process is complete\n");
-
-	return 0;
-}
-
-static int ipa3_pil_load_ipa_fws(const char *sub_sys)
-{
-	void *subsystem_get_retval = NULL;
-
-	IPADBG("PIL FW loading process initiated sub_sys=%s\n",
-		sub_sys);
-
-	subsystem_get_retval = subsystem_get(sub_sys);
-	if (IS_ERR_OR_NULL(subsystem_get_retval)) {
-		IPAERR("Unable to PIL load FW for sub_sys=%s\n", sub_sys);
-		return -EINVAL;
-	}
-
-	IPADBG("PIL FW loading process is complete sub_sys=%s\n", sub_sys);
-	return 0;
-}
-
-static void ipa3_load_ipa_fw(struct work_struct *work)
-{
-	int result;
-
-	IPADBG("Entry\n");
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	result = ipa3_attach_to_smmu();
-	if (result) {
-		IPAERR("IPA attach to smmu failed %d\n", result);
-		IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-		return;
-	}
-
-	if (ipa3_ctx->ipa3_hw_mode != IPA_HW_MODE_EMULATION &&
-	    ((ipa3_ctx->platform_type != IPA_PLAT_TYPE_MDM) ||
-	    (ipa3_ctx->ipa_hw_type >= IPA_HW_v3_5)))
-		result = ipa3_pil_load_ipa_fws(IPA_SUBSYSTEM_NAME);
-	else
-		result = ipa3_manual_load_ipa_fws();
-
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-
-	if (result) {
-		IPAERR("IPA FW loading process has failed result=%d\n",
-			result);
-		return;
-	}
-	pr_info("IPA FW loaded successfully\n");
-
-	result = ipa3_post_init(&ipa3_res, ipa3_ctx->cdev.dev);
-	if (result) {
-		IPAERR("IPA post init failed %d\n", result);
-		return;
-	}
-
-	if (ipa3_ctx->platform_type == IPA_PLAT_TYPE_APQ &&
-		ipa3_ctx->ipa3_hw_mode != IPA_HW_MODE_VIRTUAL &&
-		ipa3_ctx->ipa3_hw_mode != IPA_HW_MODE_EMULATION) {
-
-		IPADBG("Loading IPA uC via PIL\n");
-
-		/* Unvoting will happen when uC loaded event received. */
-		ipa3_proxy_clk_vote();
-
-		result = ipa3_pil_load_ipa_fws(IPA_UC_SUBSYSTEM_NAME);
-		if (result) {
-			IPAERR("IPA uC loading process has failed result=%d\n",
-				result);
-			return;
-		}
-		IPADBG("IPA uC PIL loading succeeded\n");
-	}
-}
-
-static ssize_t ipa3_write(struct file *file, const char __user *buf,
-			  size_t count, loff_t *ppos)
-{
-	unsigned long missing;
-
-	char dbg_buff[32] = { 0 };
-
-	if (count >= sizeof(dbg_buff))
-		return -EFAULT;
-
-	missing = copy_from_user(dbg_buff, buf, count);
-
-	if (missing) {
-		IPAERR("Unable to copy data from user\n");
-		return -EFAULT;
-	}
-
-	if (count > 0)
-		dbg_buff[count] = '\0';
-
-	IPADBG("user input string %s\n", dbg_buff);
-
-	/* Prevent consequent calls from trying to load the FW again. */
-	if (ipa3_is_ready())
-		return count;
-
-	/* Check MHI configuration on MDM devices */
-	if (ipa3_ctx->platform_type == IPA_PLAT_TYPE_MDM) {
-
-		if (strnstr(dbg_buff, "vlan", strlen(dbg_buff))) {
-			if (strnstr(dbg_buff, "eth", strlen(dbg_buff)))
-				ipa3_ctx->vlan_mode_iface[IPA_VLAN_IF_EMAC] =
-				true;
-			if (strnstr(dbg_buff, "rndis", strlen(dbg_buff)))
-				ipa3_ctx->vlan_mode_iface[IPA_VLAN_IF_RNDIS] =
-				true;
-			if (strnstr(dbg_buff, "ecm", strlen(dbg_buff)))
-				ipa3_ctx->vlan_mode_iface[IPA_VLAN_IF_ECM] =
-				true;
-
-			/*
-			 * when vlan mode is passed to our dev we expect
-			 * another write
-			 */
-			return count;
-		}
-
-		/* trim ending newline character if any */
-		if (count && (dbg_buff[count - 1] == '\n'))
-			dbg_buff[count - 1] = '\0';
-
-		/*
-		 * This logic enforeces MHI mode based on userspace input.
-		 * Note that MHI mode could be already determined due
-		 *  to previous logic.
-		 */
-		if (!strcasecmp(dbg_buff, "MHI")) {
-			ipa3_ctx->ipa_config_is_mhi = true;
-		} else if (strcmp(dbg_buff, "1")) {
-			IPAERR("got invalid string %s not loading FW\n",
-				dbg_buff);
-			return count;
-		}
-		pr_info("IPA is loading with %sMHI configuration\n",
-			ipa3_ctx->ipa_config_is_mhi ? "" : "non ");
-	}
-
-	/* Prevent multiple calls from trying to load the FW again. */
-	if (ipa3_ctx->fw_loaded) {
-		IPAERR("not load FW again\n");
-		return count;
-	}
-
-	/* Schedule WQ to load ipa-fws */
-	ipa3_ctx->fw_loaded = true;
-
-	queue_work(ipa3_ctx->transport_power_mgmt_wq,
-		&ipa3_fw_loading_work);
-
-	IPADBG("Scheduled a work to load IPA FW\n");
-	return count;
-}
-
-/**
- * ipa3_tz_unlock_reg - Unlocks memory regions so that they become accessible
- *	from AP.
- * @reg_info - Pointer to array of memory regions to unlock
- * @num_regs - Number of elements in the array
- *
- * Converts the input array of regions to a struct that TZ understands and
- * issues an SCM call.
- * Also flushes the memory cache to DDR in order to make sure that TZ sees the
- * correct data structure.
- *
- * Returns: 0 on success, negative on failure
- */
-int ipa3_tz_unlock_reg(struct ipa_tz_unlock_reg_info *reg_info, u16 num_regs)
-{
-	int i, size, ret;
-	struct tz_smmu_ipa_protect_region_iovec_s *ipa_tz_unlock_vec;
-	struct tz_smmu_ipa_protect_region_s cmd_buf;
-	struct scm_desc desc = {0};
-
-	if (reg_info ==  NULL || num_regs == 0) {
-		IPAERR("Bad parameters\n");
-		return -EFAULT;
-	}
-
-	size = num_regs * sizeof(struct tz_smmu_ipa_protect_region_iovec_s);
-	ipa_tz_unlock_vec = kzalloc(PAGE_ALIGN(size), GFP_KERNEL);
-	if (ipa_tz_unlock_vec == NULL)
-		return -ENOMEM;
-
-	for (i = 0; i < num_regs; i++) {
-		ipa_tz_unlock_vec[i].input_addr = reg_info[i].reg_addr ^
-			(reg_info[i].reg_addr & 0xFFF);
-		ipa_tz_unlock_vec[i].output_addr = reg_info[i].reg_addr ^
-			(reg_info[i].reg_addr & 0xFFF);
-		ipa_tz_unlock_vec[i].size = reg_info[i].size;
-		ipa_tz_unlock_vec[i].attr = IPA_TZ_UNLOCK_ATTRIBUTE;
-	}
-
-	/* pass physical address of command buffer */
-	cmd_buf.iovec_buf = virt_to_phys((void *)ipa_tz_unlock_vec);
-	cmd_buf.size_bytes = size;
-
-		desc.args[0] = virt_to_phys((void *)ipa_tz_unlock_vec);
-		desc.args[1] = size;
-		desc.arginfo = SCM_ARGS(2);
-		ret = scm_call2(SCM_SIP_FNID(SCM_SVC_MP,
-				TZ_MEM_PROTECT_REGION_ID), &desc);
-
-		if (ret) {
-			IPAERR("scm call SCM_SVC_MP failed: %d\n", ret);
-			kfree(ipa_tz_unlock_vec);
-			return -EFAULT;
-		}
-	kfree(ipa_tz_unlock_vec);
-	return 0;
-}
-
-static int ipa3_alloc_pkt_init(void)
-{
-	struct ipa_mem_buffer mem;
-	struct ipahal_imm_cmd_pyld *cmd_pyld;
-	struct ipahal_imm_cmd_ip_packet_init cmd = {0};
-	int i;
-
-	cmd_pyld = ipahal_construct_imm_cmd(IPA_IMM_CMD_IP_PACKET_INIT,
-		&cmd, false);
-	if (!cmd_pyld) {
-		IPAERR("failed to construct IMM cmd\n");
-		return -ENOMEM;
-	}
-	ipa3_ctx->pkt_init_imm_opcode = cmd_pyld->opcode;
-
-	mem.size = cmd_pyld->len * ipa3_ctx->ipa_num_pipes;
-	mem.base = dma_alloc_coherent(ipa3_ctx->pdev, mem.size,
-		&mem.phys_base, GFP_KERNEL);
-	if (!mem.base) {
-		IPAERR("failed to alloc DMA buff of size %d\n", mem.size);
-		ipahal_destroy_imm_cmd(cmd_pyld);
-		return -ENOMEM;
-	}
-	ipahal_destroy_imm_cmd(cmd_pyld);
-
-	memset(mem.base, 0, mem.size);
-	for (i = 0; i < ipa3_ctx->ipa_num_pipes; i++) {
-		cmd.destination_pipe_index = i;
-		cmd_pyld = ipahal_construct_imm_cmd(IPA_IMM_CMD_IP_PACKET_INIT,
-			&cmd, false);
-		if (!cmd_pyld) {
-			IPAERR("failed to construct IMM cmd\n");
-			dma_free_coherent(ipa3_ctx->pdev,
-				mem.size,
-				mem.base,
-				mem.phys_base);
-			return -ENOMEM;
-		}
-		memcpy(mem.base + i * cmd_pyld->len, cmd_pyld->data,
-			cmd_pyld->len);
-		ipa3_ctx->pkt_init_imm[i] = mem.phys_base + i * cmd_pyld->len;
-		ipahal_destroy_imm_cmd(cmd_pyld);
-	}
-
-	return 0;
-}
-
-/*
- * SCM call to check if secure dump is allowed.
- *
- * Returns true in secure dump allowed.
- * Return false when secure dump not allowed.
- */
-#define TZ_UTIL_GET_SEC_DUMP_STATE  0x10
-static bool ipa_is_mem_dump_allowed(void)
-{
-	struct scm_desc desc = {0};
-	int ret = 0;
-
-	desc.args[0] = 0;
-	desc.arginfo = 0;
-
-	ret = scm_call2(
-		SCM_SIP_FNID(SCM_SVC_UTIL, TZ_UTIL_GET_SEC_DUMP_STATE),
-		&desc);
-	if (ret) {
-		IPAERR("SCM DUMP_STATE call failed\n");
-		return false;
-	}
-
-	return (desc.ret[0] == 1);
-}
-
-/**
- * ipa3_pre_init() - Initialize the IPA Driver.
- * This part contains all initialization which doesn't require IPA HW, such
- * as structure allocations and initializations, register writes, etc.
- *
- * @resource_p:	contain platform specific values from DST file
- * @pdev:	The platform device structure representing the IPA driver
- *
- * Function initialization process:
- * Allocate memory for the driver context data struct
- * Initializing the ipa3_ctx with :
- *    1)parsed values from the dts file
- *    2)parameters passed to the module initialization
- *    3)read HW values(such as core memory size)
- * Map IPA core registers to CPU memory
- * Restart IPA core(HW reset)
- * Initialize the look-aside caches(kmem_cache/slab) for filter,
- *   routing and IPA-tree
- * Create memory pool with 4 objects for DMA operations(each object
- *   is 512Bytes long), this object will be use for tx(A5->IPA)
- * Initialize lists head(routing, hdr, system pipes)
- * Initialize mutexes (for ipa_ctx and NAT memory mutexes)
- * Initialize spinlocks (for list related to A5<->IPA pipes)
- * Initialize 2 single-threaded work-queue named "ipa rx wq" and "ipa tx wq"
- * Initialize Red-Black-Tree(s) for handles of header,routing rule,
- *  routing table ,filtering rule
- * Initialize the filter block by committing IPV4 and IPV6 default rules
- * Create empty routing table in system memory(no committing)
- * Create a char-device for IPA
- * Initialize IPA PM (power manager)
- * Configure GSI registers (in GSI case)
- */
-static int ipa3_pre_init(const struct ipa3_plat_drv_res *resource_p,
-		struct platform_device *ipa_pdev)
-{
-	int result = 0;
-	int i, j;
-	struct ipa3_rt_tbl_set *rset;
-	struct ipa_active_client_logging_info log_info;
-	struct cdev *cdev;
-
-	IPADBG("IPA Driver initialization started\n");
-
-	ipa3_ctx = kzalloc(sizeof(*ipa3_ctx), GFP_KERNEL);
-	if (!ipa3_ctx) {
-		result = -ENOMEM;
-		goto fail_mem_ctx;
-	}
-
-	ipa3_ctx->logbuf = ipc_log_context_create(IPA_IPC_LOG_PAGES, "ipa", 0);
-	if (ipa3_ctx->logbuf == NULL)
-		IPADBG("failed to create IPC log, continue...\n");
-
-	/* ipa3_ctx->pdev and ipa3_ctx->uc_pdev will be set in the smmu probes*/
-	ipa3_ctx->master_pdev = ipa_pdev;
-	for (i = 0; i < IPA_SMMU_CB_MAX; i++)
-		ipa3_ctx->s1_bypass_arr[i] = true;
-
-	/* initialize the gsi protocol info for uC debug stats */
-	for (i = 0; i < IPA_HW_PROTOCOL_MAX; i++) {
-		ipa3_ctx->gsi_info[i].protocol = i;
-		/* initialize all to be not started */
-		for (j = 0; j < MAX_CH_STATS_SUPPORTED; j++)
-			ipa3_ctx->gsi_info[i].ch_id_info[j].ch_id =
-				0xFF;
-	}
-
-	ipa3_ctx->ipa_wrapper_base = resource_p->ipa_mem_base;
-	ipa3_ctx->ipa_wrapper_size = resource_p->ipa_mem_size;
-	ipa3_ctx->ipa_hw_type = resource_p->ipa_hw_type;
-	ipa3_ctx->ipa3_hw_mode = resource_p->ipa3_hw_mode;
-	ipa3_ctx->platform_type = resource_p->platform_type;
-	ipa3_ctx->use_ipa_teth_bridge = resource_p->use_ipa_teth_bridge;
-	ipa3_ctx->modem_cfg_emb_pipe_flt = resource_p->modem_cfg_emb_pipe_flt;
-	ipa3_ctx->ipa_wdi2 = resource_p->ipa_wdi2;
-	ipa3_ctx->ipa_wdi2_over_gsi = resource_p->ipa_wdi2_over_gsi;
-	ipa3_ctx->ipa_wdi3_over_gsi = resource_p->ipa_wdi3_over_gsi;
-	ipa3_ctx->ipa_fltrt_not_hashable = resource_p->ipa_fltrt_not_hashable;
-	ipa3_ctx->use_64_bit_dma_mask = resource_p->use_64_bit_dma_mask;
-	ipa3_ctx->wan_rx_ring_size = resource_p->wan_rx_ring_size;
-	ipa3_ctx->lan_rx_ring_size = resource_p->lan_rx_ring_size;
-	ipa3_ctx->skip_uc_pipe_reset = resource_p->skip_uc_pipe_reset;
-	ipa3_ctx->tethered_flow_control = resource_p->tethered_flow_control;
-	ipa3_ctx->ee = resource_p->ee;
-	ipa3_ctx->gsi_ch20_wa = resource_p->gsi_ch20_wa;
-	ipa3_ctx->wdi_over_pcie = resource_p->wdi_over_pcie;
-	ipa3_ctx->ipa3_active_clients_logging.log_rdy = false;
-	ipa3_ctx->ipa_config_is_mhi = resource_p->ipa_mhi_dynamic_config;
-	ipa3_ctx->mhi_evid_limits[0] = resource_p->mhi_evid_limits[0];
-	ipa3_ctx->mhi_evid_limits[1] = resource_p->mhi_evid_limits[1];
-	ipa3_ctx->entire_ipa_block_size = resource_p->entire_ipa_block_size;
-	ipa3_ctx->do_register_collection_on_crash =
-	    resource_p->do_register_collection_on_crash;
-	ipa3_ctx->do_testbus_collection_on_crash =
-	    resource_p->do_testbus_collection_on_crash;
-	ipa3_ctx->do_non_tn_collection_on_crash =
-	    resource_p->do_non_tn_collection_on_crash;
-	ipa3_ctx->secure_debug_check_action =
-		resource_p->secure_debug_check_action;
-	ipa3_ctx->do_ram_collection_on_crash =
-		resource_p->do_ram_collection_on_crash;
-
-	if (ipa3_ctx->secure_debug_check_action == USE_SCM) {
-		if (ipa_is_mem_dump_allowed())
-			ipa3_ctx->sd_state = SD_ENABLED;
-		else
-			ipa3_ctx->sd_state = SD_DISABLED;
-	} else {
-		if (ipa3_ctx->secure_debug_check_action == OVERRIDE_SCM_TRUE)
-			ipa3_ctx->sd_state = SD_ENABLED;
-		else
-			/* secure_debug_check_action == OVERRIDE_SCM_FALSE */
-			ipa3_ctx->sd_state = SD_DISABLED;
-	}
-
-	if (ipa3_ctx->sd_state == SD_ENABLED) {
-		/* secure debug is enabled. */
-		IPADBG("secure debug enabled\n");
-	} else {
-		/* secure debug is disabled. */
-		IPADBG("secure debug disabled\n");
-		ipa3_ctx->do_testbus_collection_on_crash = false;
-	}
-	ipa3_ctx->ipa_endp_delay_wa = resource_p->ipa_endp_delay_wa;
-
-	WARN(ipa3_ctx->ipa3_hw_mode != IPA_HW_MODE_NORMAL,
-		"Non NORMAL IPA HW mode, is this emulation platform ?");
-
-	if (resource_p->ipa_tz_unlock_reg) {
-		ipa3_ctx->ipa_tz_unlock_reg_num =
-			resource_p->ipa_tz_unlock_reg_num;
-		ipa3_ctx->ipa_tz_unlock_reg = kcalloc(
-			ipa3_ctx->ipa_tz_unlock_reg_num,
-			sizeof(*ipa3_ctx->ipa_tz_unlock_reg),
-			GFP_KERNEL);
-		if (ipa3_ctx->ipa_tz_unlock_reg == NULL) {
-			result = -ENOMEM;
-			goto fail_tz_unlock_reg;
-		}
-		for (i = 0; i < ipa3_ctx->ipa_tz_unlock_reg_num; i++) {
-			ipa3_ctx->ipa_tz_unlock_reg[i].reg_addr =
-				resource_p->ipa_tz_unlock_reg[i].reg_addr;
-			ipa3_ctx->ipa_tz_unlock_reg[i].size =
-				resource_p->ipa_tz_unlock_reg[i].size;
-		}
-
-		/* unlock registers for uc */
-		result = ipa3_tz_unlock_reg(ipa3_ctx->ipa_tz_unlock_reg,
-					    ipa3_ctx->ipa_tz_unlock_reg_num);
-		if (result)
-			IPAERR("Failed to unlock memory region using TZ\n");
-	}
-
-	/* default aggregation parameters */
-	ipa3_ctx->aggregation_type = IPA_MBIM_16;
-	ipa3_ctx->aggregation_byte_limit = 1;
-	ipa3_ctx->aggregation_time_limit = 0;
-
-	ipa3_ctx->ctrl = kzalloc(sizeof(*ipa3_ctx->ctrl), GFP_KERNEL);
-	if (!ipa3_ctx->ctrl) {
-		result = -ENOMEM;
-		goto fail_mem_ctrl;
-	}
-	result = ipa3_controller_static_bind(ipa3_ctx->ctrl,
-			ipa3_ctx->ipa_hw_type);
-	if (result) {
-		IPAERR("fail to static bind IPA ctrl\n");
-		result = -EFAULT;
-		goto fail_bind;
-	}
-
-	result = ipa3_init_mem_partition(ipa3_ctx->ipa_hw_type);
-	if (result) {
-		IPAERR(":ipa3_init_mem_partition failed\n");
-		result = -ENODEV;
-		goto fail_init_mem_partition;
-	}
-
-	if (ipa3_ctx->ipa3_hw_mode != IPA_HW_MODE_VIRTUAL &&
-	    ipa3_ctx->ipa3_hw_mode != IPA_HW_MODE_EMULATION) {
-		ipa3_ctx->ctrl->msm_bus_data_ptr =
-			msm_bus_cl_get_pdata(ipa3_ctx->master_pdev);
-		if (ipa3_ctx->ctrl->msm_bus_data_ptr == NULL) {
-			IPAERR("failed to get bus scaling\n");
-			goto fail_bus_reg;
-		}
-		IPADBG("Use bus scaling info from device tree #usecases=%d\n",
-			ipa3_ctx->ctrl->msm_bus_data_ptr->num_usecases);
-
-		/* get BUS handle */
-		ipa3_ctx->ipa_bus_hdl =
-			msm_bus_scale_register_client(
-				ipa3_ctx->ctrl->msm_bus_data_ptr);
-		if (!ipa3_ctx->ipa_bus_hdl) {
-			IPAERR("fail to register with bus mgr!\n");
-			ipa3_ctx->ctrl->msm_bus_data_ptr = NULL;
-			result = -EPROBE_DEFER;
-			goto fail_bus_reg;
-		}
-	}
-
-	/* get IPA clocks */
-	result = ipa3_get_clks(&ipa3_ctx->master_pdev->dev);
-	if (result)
-		goto fail_clk;
-
-	/* init active_clients_log after getting ipa-clk */
-	result = ipa3_active_clients_log_init();
-	if (result)
-		goto fail_init_active_client;
-
-	/* Enable ipa3_ctx->enable_clock_scaling */
-	ipa3_ctx->enable_clock_scaling = 1;
-	/* vote for svs2 on bootup */
-	ipa3_ctx->curr_ipa_clk_rate = ipa3_ctx->ctrl->ipa_clk_rate_svs2;
-
-	/* Enable ipa3_ctx->enable_napi_chain */
-	ipa3_ctx->enable_napi_chain = 1;
-
-	/* enable IPA clocks explicitly to allow the initialization */
-	ipa3_enable_clks();
-
-	/* setup IPA register access */
-	IPADBG("Mapping 0x%x\n", resource_p->ipa_mem_base +
-		ipa3_ctx->ctrl->ipa_reg_base_ofst);
-	ipa3_ctx->mmio = ioremap(resource_p->ipa_mem_base +
-			ipa3_ctx->ctrl->ipa_reg_base_ofst,
-			resource_p->ipa_mem_size);
-	if (!ipa3_ctx->mmio) {
-		IPAERR(":ipa-base ioremap err\n");
-		result = -EFAULT;
-		goto fail_remap;
-	}
-
-	IPADBG(
-	    "base(0x%x)+offset(0x%x)=(0x%x) mapped to (%pK) with len (0x%x)\n",
-	    resource_p->ipa_mem_base,
-	    ipa3_ctx->ctrl->ipa_reg_base_ofst,
-	    resource_p->ipa_mem_base + ipa3_ctx->ctrl->ipa_reg_base_ofst,
-	    ipa3_ctx->mmio,
-	    resource_p->ipa_mem_size);
-
-	/*
-	 * Setup access for register collection/dump on crash
-	 */
-	if (ipa_reg_save_init(IPA_MEM_INIT_VAL) != 0) {
-		result = -EFAULT;
-		goto fail_gsi_map;
-	}
-
-	/*
-	 * Since we now know where the transport's registers live,
-	 * let's set up access to them.  This is done since subseqent
-	 * functions, that deal with the transport, require the
-	 * access.
-	 */
-	if (gsi_map_base(
-		ipa3_res.transport_mem_base,
-		ipa3_res.transport_mem_size) != 0) {
-		IPAERR("Allocation of gsi base failed\n");
-		result = -EFAULT;
-		goto fail_gsi_map;
-	}
-
-	mutex_init(&ipa3_ctx->ipa3_active_clients.mutex);
-
-	IPA_ACTIVE_CLIENTS_PREP_SPECIAL(log_info, "PROXY_CLK_VOTE");
-	ipa3_active_clients_log_inc(&log_info, false);
-	ipa3_ctx->q6_proxy_clk_vote_valid = true;
-	ipa3_ctx->q6_proxy_clk_vote_cnt = 1;
-
-	/*Updating the proxy vote cnt 1 */
-	atomic_set(&ipa3_ctx->ipa3_active_clients.cnt, 1);
-
-	/* Create workqueues for power management */
-	ipa3_ctx->power_mgmt_wq =
-		create_singlethread_workqueue("ipa_power_mgmt");
-	if (!ipa3_ctx->power_mgmt_wq) {
-		IPAERR("failed to create power mgmt wq\n");
-		result = -ENOMEM;
-		goto fail_init_hw;
-	}
-
-	ipa3_ctx->transport_power_mgmt_wq =
-		create_singlethread_workqueue("transport_power_mgmt");
-	if (!ipa3_ctx->transport_power_mgmt_wq) {
-		IPAERR("failed to create transport power mgmt wq\n");
-		result = -ENOMEM;
-		goto fail_create_transport_wq;
-	}
-
-	mutex_init(&ipa3_ctx->transport_pm.transport_pm_mutex);
-
-	/* init the lookaside cache */
-	ipa3_ctx->flt_rule_cache = kmem_cache_create("IPA_FLT",
-			sizeof(struct ipa3_flt_entry), 0, 0, NULL);
-	if (!ipa3_ctx->flt_rule_cache) {
-		IPAERR(":ipa flt cache create failed\n");
-		result = -ENOMEM;
-		goto fail_flt_rule_cache;
-	}
-	ipa3_ctx->rt_rule_cache = kmem_cache_create("IPA_RT",
-			sizeof(struct ipa3_rt_entry), 0, 0, NULL);
-	if (!ipa3_ctx->rt_rule_cache) {
-		IPAERR(":ipa rt cache create failed\n");
-		result = -ENOMEM;
-		goto fail_rt_rule_cache;
-	}
-	ipa3_ctx->hdr_cache = kmem_cache_create("IPA_HDR",
-			sizeof(struct ipa3_hdr_entry), 0, 0, NULL);
-	if (!ipa3_ctx->hdr_cache) {
-		IPAERR(":ipa hdr cache create failed\n");
-		result = -ENOMEM;
-		goto fail_hdr_cache;
-	}
-	ipa3_ctx->hdr_offset_cache =
-	   kmem_cache_create("IPA_HDR_OFFSET",
-			   sizeof(struct ipa_hdr_offset_entry), 0, 0, NULL);
-	if (!ipa3_ctx->hdr_offset_cache) {
-		IPAERR(":ipa hdr off cache create failed\n");
-		result = -ENOMEM;
-		goto fail_hdr_offset_cache;
-	}
-	ipa3_ctx->hdr_proc_ctx_cache = kmem_cache_create("IPA_HDR_PROC_CTX",
-		sizeof(struct ipa3_hdr_proc_ctx_entry), 0, 0, NULL);
-	if (!ipa3_ctx->hdr_proc_ctx_cache) {
-		IPAERR(":ipa hdr proc ctx cache create failed\n");
-		result = -ENOMEM;
-		goto fail_hdr_proc_ctx_cache;
-	}
-	ipa3_ctx->hdr_proc_ctx_offset_cache =
-		kmem_cache_create("IPA_HDR_PROC_CTX_OFFSET",
-		sizeof(struct ipa3_hdr_proc_ctx_offset_entry), 0, 0, NULL);
-	if (!ipa3_ctx->hdr_proc_ctx_offset_cache) {
-		IPAERR(":ipa hdr proc ctx off cache create failed\n");
-		result = -ENOMEM;
-		goto fail_hdr_proc_ctx_offset_cache;
-	}
-	ipa3_ctx->rt_tbl_cache = kmem_cache_create("IPA_RT_TBL",
-			sizeof(struct ipa3_rt_tbl), 0, 0, NULL);
-	if (!ipa3_ctx->rt_tbl_cache) {
-		IPAERR(":ipa rt tbl cache create failed\n");
-		result = -ENOMEM;
-		goto fail_rt_tbl_cache;
-	}
-	ipa3_ctx->tx_pkt_wrapper_cache =
-	   kmem_cache_create("IPA_TX_PKT_WRAPPER",
-			   sizeof(struct ipa3_tx_pkt_wrapper), 0, 0, NULL);
-	if (!ipa3_ctx->tx_pkt_wrapper_cache) {
-		IPAERR(":ipa tx pkt wrapper cache create failed\n");
-		result = -ENOMEM;
-		goto fail_tx_pkt_wrapper_cache;
-	}
-	ipa3_ctx->rx_pkt_wrapper_cache =
-	   kmem_cache_create("IPA_RX_PKT_WRAPPER",
-			   sizeof(struct ipa3_rx_pkt_wrapper), 0, 0, NULL);
-	if (!ipa3_ctx->rx_pkt_wrapper_cache) {
-		IPAERR(":ipa rx pkt wrapper cache create failed\n");
-		result = -ENOMEM;
-		goto fail_rx_pkt_wrapper_cache;
-	}
-
-	/* init the various list heads */
-	INIT_LIST_HEAD(&ipa3_ctx->hdr_tbl.head_hdr_entry_list);
-	for (i = 0; i < IPA_HDR_BIN_MAX; i++) {
-		INIT_LIST_HEAD(&ipa3_ctx->hdr_tbl.head_offset_list[i]);
-		INIT_LIST_HEAD(&ipa3_ctx->hdr_tbl.head_free_offset_list[i]);
-	}
-	INIT_LIST_HEAD(&ipa3_ctx->hdr_proc_ctx_tbl.head_proc_ctx_entry_list);
-	for (i = 0; i < IPA_HDR_PROC_CTX_BIN_MAX; i++) {
-		INIT_LIST_HEAD(
-			&ipa3_ctx->hdr_proc_ctx_tbl.head_offset_list[i]);
-		INIT_LIST_HEAD(
-			&ipa3_ctx->hdr_proc_ctx_tbl.head_free_offset_list[i]);
-	}
-	INIT_LIST_HEAD(&ipa3_ctx->rt_tbl_set[IPA_IP_v4].head_rt_tbl_list);
-	idr_init(&ipa3_ctx->rt_tbl_set[IPA_IP_v4].rule_ids);
-	INIT_LIST_HEAD(&ipa3_ctx->rt_tbl_set[IPA_IP_v6].head_rt_tbl_list);
-	idr_init(&ipa3_ctx->rt_tbl_set[IPA_IP_v6].rule_ids);
-
-	rset = &ipa3_ctx->reap_rt_tbl_set[IPA_IP_v4];
-	INIT_LIST_HEAD(&rset->head_rt_tbl_list);
-	idr_init(&rset->rule_ids);
-	rset = &ipa3_ctx->reap_rt_tbl_set[IPA_IP_v6];
-	INIT_LIST_HEAD(&rset->head_rt_tbl_list);
-	idr_init(&rset->rule_ids);
-	idr_init(&ipa3_ctx->flt_rt_counters.hdl);
-	spin_lock_init(&ipa3_ctx->flt_rt_counters.hdl_lock);
-	memset(&ipa3_ctx->flt_rt_counters.used_hw, 0,
-		   sizeof(ipa3_ctx->flt_rt_counters.used_hw));
-	memset(&ipa3_ctx->flt_rt_counters.used_sw, 0,
-		   sizeof(ipa3_ctx->flt_rt_counters.used_sw));
-
-	INIT_LIST_HEAD(&ipa3_ctx->intf_list);
-	INIT_LIST_HEAD(&ipa3_ctx->msg_list);
-	INIT_LIST_HEAD(&ipa3_ctx->pull_msg_list);
-	init_waitqueue_head(&ipa3_ctx->msg_waitq);
-	mutex_init(&ipa3_ctx->msg_lock);
-
-	/* store wlan client-connect-msg-list */
-	INIT_LIST_HEAD(&ipa3_ctx->msg_wlan_client_list);
-	mutex_init(&ipa3_ctx->msg_wlan_client_lock);
-
-	mutex_init(&ipa3_ctx->lock);
-	mutex_init(&ipa3_ctx->q6_proxy_clk_vote_mutex);
-	mutex_init(&ipa3_ctx->ipa_cne_evt_lock);
-
-	idr_init(&ipa3_ctx->ipa_idr);
-	spin_lock_init(&ipa3_ctx->idr_lock);
-
-	/* wlan related member */
-	memset(&ipa3_ctx->wc_memb, 0, sizeof(ipa3_ctx->wc_memb));
-	spin_lock_init(&ipa3_ctx->wc_memb.wlan_spinlock);
-	spin_lock_init(&ipa3_ctx->wc_memb.ipa_tx_mul_spinlock);
-	INIT_LIST_HEAD(&ipa3_ctx->wc_memb.wlan_comm_desc_list);
-
-	ipa3_ctx->cdev.class = class_create(THIS_MODULE, DRV_NAME);
-
-	result = alloc_chrdev_region(&ipa3_ctx->cdev.dev_num, 0, 1, DRV_NAME);
-	if (result) {
-		IPAERR("alloc_chrdev_region err\n");
-		result = -ENODEV;
-		goto fail_alloc_chrdev_region;
-	}
-
-	ipa3_ctx->cdev.dev = device_create(ipa3_ctx->cdev.class, NULL,
-		 ipa3_ctx->cdev.dev_num, ipa3_ctx, DRV_NAME);
-	if (IS_ERR(ipa3_ctx->cdev.dev)) {
-		IPAERR(":device_create err.\n");
-		result = -ENODEV;
-		goto fail_device_create;
-	}
-
-	/* Create a wakeup source. */
-	wakeup_source_init(&ipa3_ctx->w_lock, "IPA_WS");
-	spin_lock_init(&ipa3_ctx->wakelock_ref_cnt.spinlock);
-
-	/* Initialize Power Management framework */
-	result = ipa_pm_init(&ipa3_res.pm_init);
-	if (result) {
-		IPAERR("IPA PM initialization failed (%d)\n", -result);
-		result = -ENODEV;
-		goto fail_ipa_pm_init;
-	}
-	IPADBG("IPA power manager initialized\n");
-
-	INIT_LIST_HEAD(&ipa3_ctx->ipa_ready_cb_list);
-
-	init_completion(&ipa3_ctx->init_completion_obj);
-	init_completion(&ipa3_ctx->uc_loaded_completion_obj);
-
-	result = ipa3_dma_setup();
-	if (result) {
-		IPAERR("Failed to setup IPA DMA\n");
-		result = -ENODEV;
-		goto fail_ipa_dma_setup;
-	}
-
-	/*
-	 * We can't register the GSI driver yet, as it expects
-	 * the GSI FW to be up and running before the registration.
-	 *
-	 * For IPA3.0 and the emulation system, the GSI configuration
-	 * is done by the GSI driver.
-	 *
-	 * For IPA3.1 (and on), the GSI configuration is done by TZ.
-	 */
-	if (ipa3_ctx->ipa_hw_type == IPA_HW_v3_0 ||
-	    ipa3_ctx->ipa3_hw_mode == IPA_HW_MODE_EMULATION) {
-		result = ipa3_gsi_pre_fw_load_init();
-		if (result) {
-			IPAERR("gsi pre FW loading config failed\n");
-			result = -ENODEV;
-			goto fail_gsi_pre_fw_load_init;
-		}
-	}
-
-	cdev = &ipa3_ctx->cdev.cdev;
-	cdev_init(cdev, &ipa3_drv_fops);
-	cdev->owner = THIS_MODULE;
-	cdev->ops = &ipa3_drv_fops;  /* from LDD3 */
-
-	result = cdev_add(cdev, ipa3_ctx->cdev.dev_num, 1);
-	if (result) {
-		IPAERR(":cdev_add err=%d\n", -result);
-		result = -ENODEV;
-		goto fail_cdev_add;
-	}
-	IPADBG("ipa cdev added successful. major:%d minor:%d\n",
-			MAJOR(ipa3_ctx->cdev.dev_num),
-			MINOR(ipa3_ctx->cdev.dev_num));
-
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_1) {
-		result = ipa_odl_init();
-		if (result) {
-			IPADBG("Error: ODL init fialed\n");
-			result = -ENODEV;
-			goto fail_cdev_add;
-		}
-	}
-
-	/*
-	 * for IPA 4.0 offline charge is not needed and we need to prevent
-	 * power collapse until IPA uC is loaded.
-	 */
-
-	/* proxy vote for modem is added in ipa3_post_init() phase */
-	if (ipa3_ctx->ipa_hw_type != IPA_HW_v4_0)
-		ipa3_proxy_clk_unvote();
-	return 0;
-fail_cdev_add:
-fail_gsi_pre_fw_load_init:
-	ipa3_dma_shutdown();
-fail_ipa_dma_setup:
-	ipa_pm_destroy();
-fail_ipa_pm_init:
-	device_destroy(ipa3_ctx->cdev.class, ipa3_ctx->cdev.dev_num);
-fail_device_create:
-	unregister_chrdev_region(ipa3_ctx->cdev.dev_num, 1);
-fail_alloc_chrdev_region:
-	idr_destroy(&ipa3_ctx->ipa_idr);
-	rset = &ipa3_ctx->reap_rt_tbl_set[IPA_IP_v6];
-	idr_destroy(&rset->rule_ids);
-	rset = &ipa3_ctx->reap_rt_tbl_set[IPA_IP_v4];
-	idr_destroy(&rset->rule_ids);
-	idr_destroy(&ipa3_ctx->rt_tbl_set[IPA_IP_v6].rule_ids);
-	idr_destroy(&ipa3_ctx->rt_tbl_set[IPA_IP_v4].rule_ids);
-	kmem_cache_destroy(ipa3_ctx->rx_pkt_wrapper_cache);
-fail_rx_pkt_wrapper_cache:
-	kmem_cache_destroy(ipa3_ctx->tx_pkt_wrapper_cache);
-fail_tx_pkt_wrapper_cache:
-	kmem_cache_destroy(ipa3_ctx->rt_tbl_cache);
-fail_rt_tbl_cache:
-	kmem_cache_destroy(ipa3_ctx->hdr_proc_ctx_offset_cache);
-fail_hdr_proc_ctx_offset_cache:
-	kmem_cache_destroy(ipa3_ctx->hdr_proc_ctx_cache);
-fail_hdr_proc_ctx_cache:
-	kmem_cache_destroy(ipa3_ctx->hdr_offset_cache);
-fail_hdr_offset_cache:
-	kmem_cache_destroy(ipa3_ctx->hdr_cache);
-fail_hdr_cache:
-	kmem_cache_destroy(ipa3_ctx->rt_rule_cache);
-fail_rt_rule_cache:
-	kmem_cache_destroy(ipa3_ctx->flt_rule_cache);
-fail_flt_rule_cache:
-	destroy_workqueue(ipa3_ctx->transport_power_mgmt_wq);
-fail_create_transport_wq:
-	destroy_workqueue(ipa3_ctx->power_mgmt_wq);
-fail_init_hw:
-	gsi_unmap_base();
-fail_gsi_map:
-	if (ipa3_ctx->reg_collection_base)
-		iounmap(ipa3_ctx->reg_collection_base);
-	iounmap(ipa3_ctx->mmio);
-fail_remap:
-	ipa3_disable_clks();
-	ipa3_active_clients_log_destroy();
-fail_init_active_client:
-	if (ipa3_clk)
-		clk_put(ipa3_clk);
-	ipa3_clk = NULL;
-fail_clk:
-	if (ipa3_ctx->ipa_bus_hdl)
-		msm_bus_scale_unregister_client(ipa3_ctx->ipa_bus_hdl);
-fail_bus_reg:
-	if (ipa3_ctx->ctrl->msm_bus_data_ptr)
-		msm_bus_cl_clear_pdata(ipa3_ctx->ctrl->msm_bus_data_ptr);
-fail_init_mem_partition:
-fail_bind:
-	kfree(ipa3_ctx->ctrl);
-fail_mem_ctrl:
-	kfree(ipa3_ctx->ipa_tz_unlock_reg);
-fail_tz_unlock_reg:
-	if (ipa3_ctx->logbuf)
-		ipc_log_context_destroy(ipa3_ctx->logbuf);
-	kfree(ipa3_ctx);
-	ipa3_ctx = NULL;
-fail_mem_ctx:
-	return result;
-}
-
-static int get_ipa_dts_pm_info(struct platform_device *pdev,
-	struct ipa3_plat_drv_res *ipa_drv_res)
-{
-	int result;
-	int i, j;
-
-	result = of_property_read_u32(pdev->dev.of_node,
-		"qcom,msm-bus,num-cases",
-		&ipa_drv_res->pm_init.threshold_size);
-	/* No vote is ignored */
-	ipa_drv_res->pm_init.threshold_size -= 2;
-	if (result || ipa_drv_res->pm_init.threshold_size >
-		IPA_PM_THRESHOLD_MAX) {
-		IPAERR("invalid property qcom,msm-bus,num-cases %d\n",
-			ipa_drv_res->pm_init.threshold_size);
-		return -EFAULT;
-	}
-
-	result = of_property_read_u32_array(pdev->dev.of_node,
-		"qcom,throughput-threshold",
-		ipa_drv_res->pm_init.default_threshold,
-		ipa_drv_res->pm_init.threshold_size);
-	if (result) {
-		IPAERR("failed to read qcom,throughput-thresholds\n");
-		return -EFAULT;
-	}
-
-	result = of_property_count_strings(pdev->dev.of_node,
-		"qcom,scaling-exceptions");
-	if (result < 0) {
-		IPADBG("no exception list for ipa pm\n");
-		result = 0;
-	}
-
-	if (result % (ipa_drv_res->pm_init.threshold_size + 1)) {
-		IPAERR("failed to read qcom,scaling-exceptions\n");
-		return -EFAULT;
-	}
-
-	ipa_drv_res->pm_init.exception_size = result /
-		(ipa_drv_res->pm_init.threshold_size + 1);
-	if (ipa_drv_res->pm_init.exception_size >=
-		IPA_PM_EXCEPTION_MAX) {
-		IPAERR("exception list larger then max %d\n",
-			ipa_drv_res->pm_init.exception_size);
-		return -EFAULT;
-	}
-
-	for (i = 0; i < ipa_drv_res->pm_init.exception_size; i++) {
-		struct ipa_pm_exception *ex = ipa_drv_res->pm_init.exceptions;
-
-		result = of_property_read_string_index(pdev->dev.of_node,
-			"qcom,scaling-exceptions",
-			i * (ipa_drv_res->pm_init.threshold_size + 1),
-			&ex[i].usecase);
-		if (result) {
-			IPAERR("failed to read qcom,scaling-exceptions");
-			return -EFAULT;
-		}
-
-		for (j = 0; j < ipa_drv_res->pm_init.threshold_size; j++) {
-			const char *str;
-
-			result = of_property_read_string_index(
-				pdev->dev.of_node,
-				"qcom,scaling-exceptions",
-				i * (ipa_drv_res->pm_init.threshold_size + 1)
-				+ j + 1,
-				&str);
-			if (result) {
-				IPAERR("failed to read qcom,scaling-exceptions"
-					);
-				return -EFAULT;
-			}
-
-			if (kstrtou32(str, 0, &ex[i].threshold[j])) {
-				IPAERR("error str=%s\n", str);
-				return -EFAULT;
-			}
-		}
-	}
-
-	return 0;
-}
-
-static int get_ipa_dts_configuration(struct platform_device *pdev,
-		struct ipa3_plat_drv_res *ipa_drv_res)
-{
-	int i, result, pos;
-	struct resource *resource;
-	u32 *ipa_tz_unlock_reg;
-	int elem_num;
-	u32 mhi_evid_limits[2];
-
-	/* initialize ipa3_res */
-	ipa_drv_res->ipa_pipe_mem_start_ofst = IPA_PIPE_MEM_START_OFST;
-	ipa_drv_res->ipa_pipe_mem_size = IPA_PIPE_MEM_SIZE;
-	ipa_drv_res->ipa_hw_type = 0;
-	ipa_drv_res->ipa3_hw_mode = 0;
-	ipa_drv_res->platform_type = 0;
-	ipa_drv_res->modem_cfg_emb_pipe_flt = false;
-	ipa_drv_res->ipa_wdi2 = false;
-	ipa_drv_res->ipa_wdi2_over_gsi = false;
-	ipa_drv_res->ipa_wdi3_over_gsi = false;
-	ipa_drv_res->ipa_mhi_dynamic_config = false;
-	ipa_drv_res->use_64_bit_dma_mask = false;
-	ipa_drv_res->use_bw_vote = false;
-	ipa_drv_res->wan_rx_ring_size = IPA_GENERIC_RX_POOL_SZ;
-	ipa_drv_res->lan_rx_ring_size = IPA_GENERIC_RX_POOL_SZ;
-	ipa_drv_res->apply_rg10_wa = false;
-	ipa_drv_res->gsi_ch20_wa = false;
-	ipa_drv_res->ipa_tz_unlock_reg_num = 0;
-	ipa_drv_res->ipa_tz_unlock_reg = NULL;
-	ipa_drv_res->mhi_evid_limits[0] = IPA_MHI_GSI_EVENT_RING_ID_START;
-	ipa_drv_res->mhi_evid_limits[1] = IPA_MHI_GSI_EVENT_RING_ID_END;
-	ipa_drv_res->ipa_fltrt_not_hashable = false;
-	ipa_drv_res->ipa_endp_delay_wa = false;
-
-	/* Get IPA HW Version */
-	result = of_property_read_u32(pdev->dev.of_node, "qcom,ipa-hw-ver",
-					&ipa_drv_res->ipa_hw_type);
-	if ((result) || (ipa_drv_res->ipa_hw_type == 0)) {
-		IPAERR(":get resource failed for ipa-hw-ver\n");
-		return -ENODEV;
-	}
-	IPADBG(": ipa_hw_type = %d", ipa_drv_res->ipa_hw_type);
-
-	if (ipa_drv_res->ipa_hw_type < IPA_HW_v3_0) {
-		IPAERR(":IPA version below 3.0 not supported\n");
-		return -ENODEV;
-	}
-
-	if (ipa_drv_res->ipa_hw_type >= IPA_HW_MAX) {
-		IPAERR(":IPA version is greater than the MAX\n");
-		return -ENODEV;
-	}
-
-	/* Get IPA HW mode */
-	result = of_property_read_u32(pdev->dev.of_node, "qcom,ipa-hw-mode",
-			&ipa_drv_res->ipa3_hw_mode);
-	if (result)
-		IPADBG("using default (IPA_MODE_NORMAL) for ipa-hw-mode\n");
-	else
-		IPADBG(": found ipa_drv_res->ipa3_hw_mode = %d",
-				ipa_drv_res->ipa3_hw_mode);
-
-	/* Get Platform Type */
-	result = of_property_read_u32(pdev->dev.of_node, "qcom,platform-type",
-			&ipa_drv_res->platform_type);
-	if (result)
-		IPADBG("using default (IPA_PLAT_TYPE_MDM) for platform-type\n");
-	else
-		IPADBG(": found ipa_drv_res->platform_type = %d",
-				ipa_drv_res->platform_type);
-
-	/* Get IPA WAN / LAN RX pool size */
-	result = of_property_read_u32(pdev->dev.of_node,
-			"qcom,wan-rx-ring-size",
-			&ipa_drv_res->wan_rx_ring_size);
-	if (result)
-		IPADBG("using default for wan-rx-ring-size = %u\n",
-				ipa_drv_res->wan_rx_ring_size);
-	else
-		IPADBG(": found ipa_drv_res->wan-rx-ring-size = %u",
-				ipa_drv_res->wan_rx_ring_size);
-
-	result = of_property_read_u32(pdev->dev.of_node,
-			"qcom,lan-rx-ring-size",
-			&ipa_drv_res->lan_rx_ring_size);
-	if (result)
-		IPADBG("using default for lan-rx-ring-size = %u\n",
-			ipa_drv_res->lan_rx_ring_size);
-	else
-		IPADBG(": found ipa_drv_res->lan-rx-ring-size = %u",
-			ipa_drv_res->lan_rx_ring_size);
-
-	ipa_drv_res->use_ipa_teth_bridge =
-			of_property_read_bool(pdev->dev.of_node,
-			"qcom,use-ipa-tethering-bridge");
-	IPADBG(": using ipa teth bridge = %s",
-		ipa_drv_res->use_ipa_teth_bridge
-		? "True" : "False");
-
-	ipa_drv_res->ipa_mhi_dynamic_config =
-			of_property_read_bool(pdev->dev.of_node,
-			"qcom,use-ipa-in-mhi-mode");
-	IPADBG(": ipa_mhi_dynamic_config (%s)\n",
-		ipa_drv_res->ipa_mhi_dynamic_config
-		? "True" : "False");
-
-	ipa_drv_res->modem_cfg_emb_pipe_flt =
-			of_property_read_bool(pdev->dev.of_node,
-			"qcom,modem-cfg-emb-pipe-flt");
-	IPADBG(": modem configure embedded pipe filtering = %s\n",
-			ipa_drv_res->modem_cfg_emb_pipe_flt
-			? "True" : "False");
-	ipa_drv_res->ipa_wdi2_over_gsi =
-			of_property_read_bool(pdev->dev.of_node,
-			"qcom,ipa-wdi2_over_gsi");
-	IPADBG(": WDI-2.0 over gsi= %s\n",
-			ipa_drv_res->ipa_wdi2_over_gsi
-			? "True" : "False");
-	ipa_drv_res->ipa_endp_delay_wa =
-			of_property_read_bool(pdev->dev.of_node,
-			"qcom,ipa-endp-delay-wa");
-	IPADBG(": endppoint delay wa = %s\n",
-			ipa_drv_res->ipa_endp_delay_wa
-			? "True" : "False");
-
-	ipa_drv_res->ipa_wdi3_over_gsi =
-			of_property_read_bool(pdev->dev.of_node,
-			"qcom,ipa-wdi3-over-gsi");
-	IPADBG(": WDI-3.0 over gsi= %s\n",
-			ipa_drv_res->ipa_wdi3_over_gsi
-			? "True" : "False");
-
-	ipa_drv_res->ipa_wdi2 =
-			of_property_read_bool(pdev->dev.of_node,
-			"qcom,ipa-wdi2");
-	IPADBG(": WDI-2.0 = %s\n",
-			ipa_drv_res->ipa_wdi2
-			? "True" : "False");
-
-	ipa_drv_res->ipa_fltrt_not_hashable =
-			of_property_read_bool(pdev->dev.of_node,
-			"qcom,ipa-fltrt-not-hashable");
-	IPADBG(": IPA filter/route rule hashable = %s\n",
-			ipa_drv_res->ipa_fltrt_not_hashable
-			? "True" : "False");
-
-	ipa_drv_res->use_64_bit_dma_mask =
-			of_property_read_bool(pdev->dev.of_node,
-			"qcom,use-64-bit-dma-mask");
-	IPADBG(": use_64_bit_dma_mask = %s\n",
-			ipa_drv_res->use_64_bit_dma_mask
-			? "True" : "False");
-
-	ipa_drv_res->use_bw_vote =
-			of_property_read_bool(pdev->dev.of_node,
-			"qcom,bandwidth-vote-for-ipa");
-	IPADBG(": use_bw_vote = %s\n",
-			ipa_drv_res->use_bw_vote
-			? "True" : "False");
-
-	ipa_drv_res->skip_uc_pipe_reset =
-		of_property_read_bool(pdev->dev.of_node,
-		"qcom,skip-uc-pipe-reset");
-	IPADBG(": skip uC pipe reset = %s\n",
-		ipa_drv_res->skip_uc_pipe_reset
-		? "True" : "False");
-
-	ipa_drv_res->tethered_flow_control =
-		of_property_read_bool(pdev->dev.of_node,
-		"qcom,tethered-flow-control");
-	IPADBG(": Use apps based flow control = %s\n",
-		ipa_drv_res->tethered_flow_control
-		? "True" : "False");
-
-	/* Get IPA wrapper address */
-	resource = platform_get_resource_byname(pdev, IORESOURCE_MEM,
-			"ipa-base");
-	if (!resource) {
-		IPAERR(":get resource failed for ipa-base!\n");
-		return -ENODEV;
-	}
-	ipa_drv_res->ipa_mem_base = resource->start;
-	ipa_drv_res->ipa_mem_size = resource_size(resource);
-	IPADBG(": ipa-base = 0x%x, size = 0x%x\n",
-			ipa_drv_res->ipa_mem_base,
-			ipa_drv_res->ipa_mem_size);
-
-	smmu_info.ipa_base = ipa_drv_res->ipa_mem_base;
-	smmu_info.ipa_size = ipa_drv_res->ipa_mem_size;
-
-	/* Get IPA GSI address */
-	resource = platform_get_resource_byname(pdev, IORESOURCE_MEM,
-			"gsi-base");
-	if (!resource) {
-		IPAERR(":get resource failed for gsi-base\n");
-		return -ENODEV;
-	}
-	ipa_drv_res->transport_mem_base = resource->start;
-	ipa_drv_res->transport_mem_size = resource_size(resource);
-	IPADBG(": gsi-base = 0x%x, size = 0x%x\n",
-			ipa_drv_res->transport_mem_base,
-			ipa_drv_res->transport_mem_size);
-
-	/* Get IPA GSI IRQ number */
-	resource = platform_get_resource_byname(pdev, IORESOURCE_IRQ,
-			"gsi-irq");
-	if (!resource) {
-		IPAERR(":get resource failed for gsi-irq\n");
-		return -ENODEV;
-	}
-	ipa_drv_res->transport_irq = resource->start;
-	IPADBG(": gsi-irq = %d\n", ipa_drv_res->transport_irq);
-
-	/* Get IPA pipe mem start ofst */
-	resource = platform_get_resource_byname(pdev, IORESOURCE_MEM,
-			"ipa-pipe-mem");
-	if (!resource) {
-		IPADBG(":not using pipe memory - resource nonexisting\n");
-	} else {
-		ipa_drv_res->ipa_pipe_mem_start_ofst = resource->start;
-		ipa_drv_res->ipa_pipe_mem_size = resource_size(resource);
-		IPADBG(":using pipe memory - at 0x%x of size 0x%x\n",
-				ipa_drv_res->ipa_pipe_mem_start_ofst,
-				ipa_drv_res->ipa_pipe_mem_size);
-	}
-
-	/* Get IPA IRQ number */
-	resource = platform_get_resource_byname(pdev, IORESOURCE_IRQ,
-			"ipa-irq");
-	if (!resource) {
-		IPAERR(":get resource failed for ipa-irq\n");
-		return -ENODEV;
-	}
-	ipa_drv_res->ipa_irq = resource->start;
-	IPADBG(":ipa-irq = %d\n", ipa_drv_res->ipa_irq);
-
-	result = of_property_read_u32(pdev->dev.of_node, "qcom,ee",
-			&ipa_drv_res->ee);
-	if (result)
-		ipa_drv_res->ee = 0;
-	IPADBG(":ee = %u\n", ipa_drv_res->ee);
-
-	ipa_drv_res->apply_rg10_wa =
-		of_property_read_bool(pdev->dev.of_node,
-		"qcom,use-rg10-limitation-mitigation");
-	IPADBG(": Use Register Group 10 limitation mitigation = %s\n",
-		ipa_drv_res->apply_rg10_wa
-		? "True" : "False");
-
-	ipa_drv_res->gsi_ch20_wa =
-		of_property_read_bool(pdev->dev.of_node,
-		"qcom,do-not-use-ch-gsi-20");
-	IPADBG(": GSI CH 20 WA is = %s\n",
-		ipa_drv_res->gsi_ch20_wa
-		? "Needed" : "Not needed");
-
-	elem_num = of_property_count_elems_of_size(pdev->dev.of_node,
-		"qcom,mhi-event-ring-id-limits", sizeof(u32));
-
-	if (elem_num == 2) {
-		if (of_property_read_u32_array(pdev->dev.of_node,
-			"qcom,mhi-event-ring-id-limits", mhi_evid_limits, 2)) {
-			IPAERR("failed to read mhi event ring id limits\n");
-			return -EFAULT;
-		}
-		if (mhi_evid_limits[0] > mhi_evid_limits[1]) {
-			IPAERR("mhi event ring id low limit > high limit\n");
-			return -EFAULT;
-		}
-		ipa_drv_res->mhi_evid_limits[0] = mhi_evid_limits[0];
-		ipa_drv_res->mhi_evid_limits[1] = mhi_evid_limits[1];
-		IPADBG(": mhi-event-ring-id-limits start=%u end=%u\n",
-			mhi_evid_limits[0], mhi_evid_limits[1]);
-	} else {
-		if (elem_num > 0) {
-			IPAERR("Invalid mhi event ring id limits number %d\n",
-				elem_num);
-			return -EINVAL;
-		}
-		IPADBG("use default mhi evt ring id limits start=%u end=%u\n",
-			ipa_drv_res->mhi_evid_limits[0],
-			ipa_drv_res->mhi_evid_limits[1]);
-	}
-
-	elem_num = of_property_count_elems_of_size(pdev->dev.of_node,
-		"qcom,ipa-tz-unlock-reg", sizeof(u32));
-
-	if (elem_num > 0 && elem_num % 2 == 0) {
-		ipa_drv_res->ipa_tz_unlock_reg_num = elem_num / 2;
-
-		ipa_tz_unlock_reg = kcalloc(elem_num, sizeof(u32), GFP_KERNEL);
-		if (ipa_tz_unlock_reg == NULL)
-			return -ENOMEM;
-
-		ipa_drv_res->ipa_tz_unlock_reg = kcalloc(
-			ipa_drv_res->ipa_tz_unlock_reg_num,
-			sizeof(*ipa_drv_res->ipa_tz_unlock_reg),
-			GFP_KERNEL);
-		if (ipa_drv_res->ipa_tz_unlock_reg == NULL) {
-			kfree(ipa_tz_unlock_reg);
-			return -ENOMEM;
-		}
-
-		if (of_property_read_u32_array(pdev->dev.of_node,
-			"qcom,ipa-tz-unlock-reg", ipa_tz_unlock_reg,
-			elem_num)) {
-			IPAERR("failed to read register addresses\n");
-			kfree(ipa_tz_unlock_reg);
-			kfree(ipa_drv_res->ipa_tz_unlock_reg);
-			return -EFAULT;
-		}
-
-		pos = 0;
-		for (i = 0; i < ipa_drv_res->ipa_tz_unlock_reg_num; i++) {
-			ipa_drv_res->ipa_tz_unlock_reg[i].reg_addr =
-				ipa_tz_unlock_reg[pos++];
-			ipa_drv_res->ipa_tz_unlock_reg[i].size =
-				ipa_tz_unlock_reg[pos++];
-			IPADBG("tz unlock reg %d: addr 0x%pa size %llu\n", i,
-				&ipa_drv_res->ipa_tz_unlock_reg[i].reg_addr,
-				ipa_drv_res->ipa_tz_unlock_reg[i].size);
-		}
-		kfree(ipa_tz_unlock_reg);
-	}
-
-	/* get IPA PM related information */
-	result = get_ipa_dts_pm_info(pdev, ipa_drv_res);
-	if (result) {
-		IPAERR("failed to get pm info from dts %d\n", result);
-		return result;
-	}
-
-	ipa_drv_res->wdi_over_pcie =
-		of_property_read_bool(pdev->dev.of_node,
-		"qcom,wlan-ce-db-over-pcie");
-	IPADBG("Is wdi_over_pcie ? (%s)\n",
-		ipa_drv_res->wdi_over_pcie ? "Yes":"No");
-
-	/*
-	 * If we're on emulator, get its interrupt controller's mem
-	 * start and size
-	 */
-	if (ipa_drv_res->ipa3_hw_mode == IPA_HW_MODE_EMULATION) {
-		resource = platform_get_resource_byname(
-		    pdev, IORESOURCE_MEM, "intctrl-base");
-		if (!resource) {
-			IPAERR(":Can't find intctrl-base resource\n");
-			return -ENODEV;
-		}
-		ipa_drv_res->emulator_intcntrlr_mem_base =
-		    resource->start;
-		ipa_drv_res->emulator_intcntrlr_mem_size =
-		    resource_size(resource);
-		IPADBG(":using intctrl-base at 0x%x of size 0x%x\n",
-		       ipa_drv_res->emulator_intcntrlr_mem_base,
-		       ipa_drv_res->emulator_intcntrlr_mem_size);
-	}
-
-	ipa_drv_res->entire_ipa_block_size = 0x100000;
-	result = of_property_read_u32(pdev->dev.of_node,
-				      "qcom,entire-ipa-block-size",
-				      &ipa_drv_res->entire_ipa_block_size);
-	IPADBG(": entire_ipa_block_size = %d\n",
-	       ipa_drv_res->entire_ipa_block_size);
-
-	/*
-	 * We'll read register-collection-on-crash here, but log it
-	 * later below because its value may change based on other
-	 * subsequent dtsi reads......
-	 */
-	ipa_drv_res->do_register_collection_on_crash =
-	    of_property_read_bool(pdev->dev.of_node,
-				  "qcom,register-collection-on-crash");
-	/*
-	 * We'll read testbus-collection-on-crash here...
-	 */
-	ipa_drv_res->do_testbus_collection_on_crash =
-	    of_property_read_bool(pdev->dev.of_node,
-				  "qcom,testbus-collection-on-crash");
-	IPADBG(": doing testbus collection on crash = %u\n",
-	       ipa_drv_res->do_testbus_collection_on_crash);
-
-	/*
-	 * We'll read non-tn-collection-on-crash here...
-	 */
-	ipa_drv_res->do_non_tn_collection_on_crash =
-	    of_property_read_bool(pdev->dev.of_node,
-				  "qcom,non-tn-collection-on-crash");
-	IPADBG(": doing non-tn collection on crash = %u\n",
-	       ipa_drv_res->do_non_tn_collection_on_crash);
-
-	/*
-	 * We'll read ram-collection-on-crash here...
-	 */
-	ipa_drv_res->do_ram_collection_on_crash =
-		of_property_read_bool(
-			pdev->dev.of_node,
-			"qcom,ram-collection-on-crash");
-	IPADBG(": doing ram collection on crash = %u\n",
-		   ipa_drv_res->do_ram_collection_on_crash);
-
-	if (ipa_drv_res->do_testbus_collection_on_crash ||
-		ipa_drv_res->do_non_tn_collection_on_crash ||
-		ipa_drv_res->do_ram_collection_on_crash)
-		ipa_drv_res->do_register_collection_on_crash = true;
-
-	IPADBG(": doing register collection on crash = %u\n",
-	       ipa_drv_res->do_register_collection_on_crash);
-
-	result = of_property_read_u32(
-		pdev->dev.of_node,
-		"qcom,secure-debug-check-action",
-		&ipa_drv_res->secure_debug_check_action);
-	if (result ||
-		(ipa_drv_res->secure_debug_check_action != 0 &&
-		 ipa_drv_res->secure_debug_check_action != 1 &&
-		 ipa_drv_res->secure_debug_check_action != 2))
-		ipa_drv_res->secure_debug_check_action = USE_SCM;
-
-	IPADBG(": secure-debug-check-action = %d\n",
-		   ipa_drv_res->secure_debug_check_action);
-
-	return 0;
-}
-
-static int ipa_smmu_wlan_cb_probe(struct device *dev)
-{
-	struct ipa_smmu_cb_ctx *cb = ipa3_get_smmu_ctx(IPA_SMMU_CB_WLAN);
-	int fast = 0;
-	int bypass = 0;
-	u32 add_map_size;
-	const u32 *add_map;
-	int i;
-	u32 iova_ap_mapping[2];
-
-	IPADBG("WLAN CB PROBE dev=%pK\n", dev);
-
-	if (!smmu_info.present[IPA_SMMU_CB_WLAN]) {
-		IPAERR("WLAN SMMU is disabled\n");
-		return 0;
-	}
-
-	IPADBG("WLAN CB PROBE dev=%pK retrieving IOMMU mapping\n", dev);
-
-	cb->iommu_domain = iommu_get_domain_for_dev(dev);
-	if (IS_ERR_OR_NULL(cb->iommu_domain)) {
-		IPAERR("could not get iommu domain\n");
-		return -EINVAL;
-	}
-
-	IPADBG("WLAN CB PROBE mapping retrieved\n");
-	cb->is_cache_coherent = of_property_read_bool(dev->of_node,
-							"dma-coherent");
-	cb->dev   = dev;
-	cb->valid = true;
-
-	cb->va_start = cb->va_end  = cb->va_size = 0;
-	if (of_property_read_u32_array(
-			dev->of_node, "qcom,iommu-dma-addr-pool",
-			iova_ap_mapping, 2) == 0) {
-		cb->va_start = iova_ap_mapping[0];
-		cb->va_size  = iova_ap_mapping[1];
-		cb->va_end   = cb->va_start + cb->va_size;
-	}
-
-	IPADBG("WLAN CB PROBE dev=%pK va_start=0x%x va_size=0x%x\n",
-		   dev, cb->va_start, cb->va_size);
-
-	/*
-	 * Prior to these calls to iommu_domain_get_attr(), these
-	 * attributes were set in this function relative to dtsi values
-	 * defined for this driver.  In other words, if corresponding ipa
-	 * driver owned values were found in the dtsi, they were read and
-	 * set here.
-	 *
-	 * In this new world, the developer will use iommu owned dtsi
-	 * settings to set them there.  This new logic below, simply
-	 * checks to see if they've been set in dtsi.  If so, the logic
-	 * further below acts accordingly...
-	 */
-	iommu_domain_get_attr(cb->iommu_domain, DOMAIN_ATTR_S1_BYPASS, &bypass);
-	iommu_domain_get_attr(cb->iommu_domain, DOMAIN_ATTR_FAST, &fast);
-
-	IPADBG(
-	  "WLAN CB PROBE dev=%pK DOMAIN ATTRS bypass=%d fast=%d\n",
-	  dev, bypass, fast);
-
-	ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_WLAN] = (bypass != 0);
-
-	/* MAP ipa-uc ram */
-	add_map = of_get_property(dev->of_node,
-		"qcom,additional-mapping", &add_map_size);
-	if (add_map) {
-		/* mapping size is an array of 3-tuple of u32 */
-		if (add_map_size % (3 * sizeof(u32))) {
-			IPAERR("wrong additional mapping format\n");
-			cb->valid = false;
-			return -EFAULT;
-		}
-
-		/* iterate of each entry of the additional mapping array */
-		for (i = 0; i < add_map_size / sizeof(u32); i += 3) {
-			u32 iova = be32_to_cpu(add_map[i]);
-			u32 pa = be32_to_cpu(add_map[i + 1]);
-			u32 size = be32_to_cpu(add_map[i + 2]);
-			unsigned long iova_p;
-			phys_addr_t pa_p;
-			u32 size_p;
-
-			IPA_SMMU_ROUND_TO_PAGE(iova, pa, size,
-				iova_p, pa_p, size_p);
-			IPADBG_LOW("mapping 0x%lx to 0x%pa size %d\n",
-				iova_p, &pa_p, size_p);
-			ipa3_iommu_map(cb->iommu_domain,
-				iova_p, pa_p, size_p,
-				IOMMU_READ | IOMMU_WRITE | IOMMU_MMIO);
-		}
-	}
-
-	return 0;
-}
-
-static int ipa_smmu_uc_cb_probe(struct device *dev)
-{
-	struct ipa_smmu_cb_ctx *cb = ipa3_get_smmu_ctx(IPA_SMMU_CB_UC);
-	int bypass = 0;
-	int fast = 0;
-	u32 iova_ap_mapping[2];
-
-	IPADBG("UC CB PROBE dev=%pK\n", dev);
-
-	if (!smmu_info.present[IPA_SMMU_CB_UC]) {
-		IPAERR("UC SMMU is disabled\n");
-		return 0;
-	}
-
-	if (smmu_info.use_64_bit_dma_mask) {
-		if (dma_set_mask(dev, DMA_BIT_MASK(64)) ||
-			dma_set_coherent_mask(dev, DMA_BIT_MASK(64))) {
-			IPAERR("DMA set 64bit mask failed\n");
-			return -EOPNOTSUPP;
-		}
-	} else {
-		if (dma_set_mask(dev, DMA_BIT_MASK(32)) ||
-			dma_set_coherent_mask(dev, DMA_BIT_MASK(32))) {
-			IPAERR("DMA set 32bit mask failed\n");
-			return -EOPNOTSUPP;
-		}
-	}
-
-	IPADBG("UC CB PROBE dev=%pK retrieving IOMMU mapping\n", dev);
-
-	cb->iommu_domain = iommu_get_domain_for_dev(dev);
-	if (IS_ERR_OR_NULL(cb->iommu_domain)) {
-		IPAERR("could not get iommu domain\n");
-		return -EINVAL;
-	}
-
-	IPADBG("UC CB PROBE mapping retrieved\n");
-
-	cb->is_cache_coherent = of_property_read_bool(dev->of_node,
-						"dma-coherent");
-	cb->dev   = dev;
-	cb->valid = true;
-
-	cb->va_start = cb->va_end  = cb->va_size = 0;
-	if (of_property_read_u32_array(
-			dev->of_node, "qcom,iommu-dma-addr-pool",
-			iova_ap_mapping, 2) == 0) {
-		cb->va_start = iova_ap_mapping[0];
-		cb->va_size  = iova_ap_mapping[1];
-		cb->va_end   = cb->va_start + cb->va_size;
-	}
-
-	IPADBG("UC CB PROBE dev=%pK va_start=0x%x va_size=0x%x\n",
-		   dev, cb->va_start, cb->va_size);
-
-	/*
-	 * Prior to these calls to iommu_domain_get_attr(), these
-	 * attributes were set in this function relative to dtsi values
-	 * defined for this driver.  In other words, if corresponding ipa
-	 * driver owned values were found in the dtsi, they were read and
-	 * set here.
-	 *
-	 * In this new world, the developer will use iommu owned dtsi
-	 * settings to set them there.  This new logic below, simply
-	 * checks to see if they've been set in dtsi.  If so, the logic
-	 * further below acts accordingly...
-	 */
-	iommu_domain_get_attr(cb->iommu_domain, DOMAIN_ATTR_S1_BYPASS, &bypass);
-	iommu_domain_get_attr(cb->iommu_domain, DOMAIN_ATTR_FAST, &fast);
-
-	IPADBG("UC CB PROBE dev=%pK DOMAIN ATTRS bypass=%d fast=%d\n",
-		   dev, bypass, fast);
-
-	ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_UC] = (bypass != 0);
-
-	ipa3_ctx->uc_pdev = dev;
-
-	return 0;
-}
-
-static int ipa_smmu_ap_cb_probe(struct device *dev)
-{
-	struct ipa_smmu_cb_ctx *cb = ipa3_get_smmu_ctx(IPA_SMMU_CB_AP);
-	int fast = 0;
-	int bypass = 0;
-	u32 add_map_size;
-	const u32 *add_map;
-	void *smem_addr;
-	size_t smem_size;
-	u32 ipa_smem_size = 0;
-	int ret;
-	int i;
-	unsigned long iova_p;
-	phys_addr_t pa_p;
-	u32 size_p;
-	phys_addr_t iova;
-	phys_addr_t pa;
-	u32 iova_ap_mapping[2];
-
-	IPADBG("AP CB PROBE dev=%pK\n", dev);
-
-	if (!smmu_info.present[IPA_SMMU_CB_AP]) {
-		IPAERR("AP SMMU is disabled");
-		return 0;
-	}
-
-	if (smmu_info.use_64_bit_dma_mask) {
-		if (dma_set_mask(dev, DMA_BIT_MASK(64)) ||
-			dma_set_coherent_mask(dev, DMA_BIT_MASK(64))) {
-			IPAERR("DMA set 64bit mask failed\n");
-			return -EOPNOTSUPP;
-		}
-	} else {
-		if (dma_set_mask(dev, DMA_BIT_MASK(32)) ||
-			dma_set_coherent_mask(dev, DMA_BIT_MASK(32))) {
-			IPAERR("DMA set 32bit mask failed\n");
-			return -EOPNOTSUPP;
-		}
-	}
-
-	IPADBG("AP CB PROBE dev=%pK retrieving IOMMU mapping\n", dev);
-
-	cb->iommu_domain = iommu_get_domain_for_dev(dev);
-	if (IS_ERR_OR_NULL(cb->iommu_domain)) {
-		IPAERR("could not get iommu domain\n");
-		return -EINVAL;
-	}
-
-	IPADBG("AP CB PROBE mapping retrieved\n");
-
-	cb->is_cache_coherent = of_property_read_bool(dev->of_node,
-						"dma-coherent");
-	cb->dev   = dev;
-	cb->valid = true;
-
-	cb->va_start = cb->va_end  = cb->va_size = 0;
-	if (of_property_read_u32_array(
-			dev->of_node, "qcom,iommu-dma-addr-pool",
-			iova_ap_mapping, 2) == 0) {
-		cb->va_start = iova_ap_mapping[0];
-		cb->va_size  = iova_ap_mapping[1];
-		cb->va_end   = cb->va_start + cb->va_size;
-	}
-
-	IPADBG("AP CB PROBE dev=%pK va_start=0x%x va_size=0x%x\n",
-		   dev, cb->va_start, cb->va_size);
-
-	/*
-	 * Prior to these calls to iommu_domain_get_attr(), these
-	 * attributes were set in this function relative to dtsi values
-	 * defined for this driver.  In other words, if corresponding ipa
-	 * driver owned values were found in the dtsi, they were read and
-	 * set here.
-	 *
-	 * In this new world, the developer will use iommu owned dtsi
-	 * settings to set them there.  This new logic below, simply
-	 * checks to see if they've been set in dtsi.  If so, the logic
-	 * further below acts accordingly...
-	 */
-	iommu_domain_get_attr(cb->iommu_domain, DOMAIN_ATTR_S1_BYPASS, &bypass);
-	iommu_domain_get_attr(cb->iommu_domain, DOMAIN_ATTR_FAST, &fast);
-
-	IPADBG("AP CB PROBE dev=%pK DOMAIN ATTRS bypass=%d fast=%d\n",
-		   dev, bypass, fast);
-
-	ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_AP] = (bypass != 0);
-
-	add_map = of_get_property(dev->of_node,
-		"qcom,additional-mapping", &add_map_size);
-	if (add_map) {
-		/* mapping size is an array of 3-tuple of u32 */
-		if (add_map_size % (3 * sizeof(u32))) {
-			IPAERR("wrong additional mapping format\n");
-			cb->valid = false;
-			return -EFAULT;
-		}
-
-		/* iterate of each entry of the additional mapping array */
-		for (i = 0; i < add_map_size / sizeof(u32); i += 3) {
-			u32 iova = be32_to_cpu(add_map[i]);
-			u32 pa = be32_to_cpu(add_map[i + 1]);
-			u32 size = be32_to_cpu(add_map[i + 2]);
-			unsigned long iova_p;
-			phys_addr_t pa_p;
-			u32 size_p;
-
-			IPA_SMMU_ROUND_TO_PAGE(iova, pa, size,
-				iova_p, pa_p, size_p);
-			IPADBG_LOW("mapping 0x%lx to 0x%pa size %d\n",
-				iova_p, &pa_p, size_p);
-			ipa3_iommu_map(cb->iommu_domain,
-				iova_p, pa_p, size_p,
-				IOMMU_READ | IOMMU_WRITE | IOMMU_MMIO);
-		}
-	}
-
-	ret = of_property_read_u32(dev->of_node, "qcom,ipa-q6-smem-size",
-					&ipa_smem_size);
-	if (ret) {
-		IPADBG("ipa q6 smem size (default) = %u\n", IPA_SMEM_SIZE);
-		ipa_smem_size = IPA_SMEM_SIZE;
-	} else {
-		IPADBG("ipa q6 smem size = %u\n", ipa_smem_size);
-	}
-
-	if (ipa3_ctx->platform_type != IPA_PLAT_TYPE_APQ) {
-		/* map SMEM memory for IPA table accesses */
-		ret = qcom_smem_alloc(SMEM_MODEM,
-			SMEM_IPA_FILTER_TABLE,
-			ipa_smem_size);
-
-		if (ret < 0 && ret != -EEXIST) {
-			IPAERR("unable to allocate smem MODEM entry\n");
-			cb->valid = false;
-			return -EFAULT;
-		}
-		smem_addr = qcom_smem_get(SMEM_MODEM,
-			SMEM_IPA_FILTER_TABLE,
-			&smem_size);
-		if (IS_ERR(smem_addr)) {
-			IPAERR("unable to acquire smem MODEM entry\n");
-			cb->valid = false;
-			return -EFAULT;
-		}
-		if (smem_size != ipa_smem_size)
-			IPAERR("unexpected read q6 smem size %zu %u\n",
-				smem_size, ipa_smem_size);
-
-		iova = qcom_smem_virt_to_phys(smem_addr);
-		pa = iova;
-
-		IPA_SMMU_ROUND_TO_PAGE(iova, pa, ipa_smem_size,
-				iova_p, pa_p, size_p);
-		IPADBG("mapping 0x%lx to 0x%pa size %d\n",
-				iova_p, &pa_p, size_p);
-		ipa3_iommu_map(cb->iommu_domain,
-				iova_p, pa_p, size_p,
-				IOMMU_READ | IOMMU_WRITE);
-	}
-
-	smmu_info.present[IPA_SMMU_CB_AP] = true;
-
-	ipa3_ctx->pdev = dev;
-	cb->next_addr = cb->va_end;
-
-	return 0;
-}
-
-static int ipa_smmu_11ad_cb_probe(struct device *dev)
-{
-	int bypass = 0;
-	struct ipa_smmu_cb_ctx *cb = ipa3_get_smmu_ctx(IPA_SMMU_CB_11AD);
-	u32 iova_ap_mapping[2];
-
-	IPADBG("11AD CB probe: dev=%pK\n", dev);
-
-	if (!smmu_info.present[IPA_SMMU_CB_11AD]) {
-		IPAERR("11AD SMMU is disabled");
-		return 0;
-	}
-
-	cb->iommu_domain = iommu_get_domain_for_dev(dev);
-	if (IS_ERR_OR_NULL(cb->iommu_domain)) {
-		IPAERR("could not get iommu domain\n");
-		return -EINVAL;
-	}
-	cb->is_cache_coherent = of_property_read_bool(dev->of_node,
-							"dma-coherent");
-	cb->dev   = dev;
-	cb->valid = true;
-
-	cb->va_start = cb->va_end  = cb->va_size = 0;
-	if (of_property_read_u32_array(
-			dev->of_node, "qcom,iommu-dma-addr-pool",
-			iova_ap_mapping, 2) == 0) {
-		cb->va_start = iova_ap_mapping[0];
-		cb->va_size  = iova_ap_mapping[1];
-		cb->va_end   = cb->va_start + cb->va_size;
-	}
-
-	IPADBG("11AD CB PROBE dev=%pK va_start=0x%x va_size=0x%x\n",
-		   dev, cb->va_start, cb->va_size);
-
-	iommu_domain_get_attr(cb->iommu_domain, DOMAIN_ATTR_S1_BYPASS, &bypass);
-
-	IPADBG("11AD CB PROBE dev=%pK DOMAIN ATTRS bypass=%d\n",
-		   dev, bypass);
-
-	ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_11AD] = (bypass != 0);
-
-	if (of_property_read_bool(dev->of_node, "qcom,shared-cb")) {
-		IPADBG("11AD using shared CB\n");
-		cb->shared = true;
-	}
-
-	return 0;
-}
-
-static int ipa_smmu_cb_probe(struct device *dev, enum ipa_smmu_cb_type cb_type)
-{
-	switch (cb_type) {
-	case IPA_SMMU_CB_AP:
-		return ipa_smmu_ap_cb_probe(dev);
-	case IPA_SMMU_CB_WLAN:
-		return ipa_smmu_wlan_cb_probe(dev);
-	case IPA_SMMU_CB_UC:
-		return ipa_smmu_uc_cb_probe(dev);
-	case IPA_SMMU_CB_11AD:
-		return ipa_smmu_11ad_cb_probe(dev);
-	case IPA_SMMU_CB_MAX:
-		IPAERR("Invalid cb_type\n");
-	}
-	return 0;
-}
-
-static int ipa3_attach_to_smmu(void)
-{
-	struct ipa_smmu_cb_ctx *cb;
-	int i, result;
-
-	ipa3_ctx->pdev = &ipa3_ctx->master_pdev->dev;
-	ipa3_ctx->uc_pdev = &ipa3_ctx->master_pdev->dev;
-
-	if (smmu_info.arm_smmu) {
-		IPADBG("smmu is enabled\n");
-		for (i = 0; i < IPA_SMMU_CB_MAX; i++) {
-			cb = ipa3_get_smmu_ctx(i);
-			result = ipa_smmu_cb_probe(cb->dev, i);
-			if (result)
-				IPAERR("probe failed for cb %d\n", i);
-		}
-	} else {
-		IPADBG("smmu is disabled\n");
-	}
-	return 0;
-}
-
-static irqreturn_t ipa3_smp2p_modem_clk_query_isr(int irq, void *ctxt)
-{
-	ipa3_freeze_clock_vote_and_notify_modem();
-
-	return IRQ_HANDLED;
-}
-
-static int ipa3_smp2p_probe(struct device *dev)
-{
-	struct device_node *node = dev->of_node;
-	int res;
-	int irq = 0;
-
-	if (ipa3_ctx == NULL) {
-		IPAERR("ipa3_ctx was not initialized\n");
-		return -EPROBE_DEFER;
-	}
-	IPADBG("node->name=%s\n", node->name);
-	if (ipa3_ctx->platform_type == IPA_PLAT_TYPE_APQ) {
-		IPADBG("Ignore smp2p on APQ platform\n");
-		return 0;
-	}
-
-	if (strcmp("qcom,smp2p_map_ipa_1_out", node->name) == 0) {
-		if (of_find_property(node, "qcom,smem-states", NULL)) {
-			ipa3_ctx->smp2p_info.smem_state =
-			qcom_smem_state_get(dev, "ipa-smp2p-out",
-			&ipa3_ctx->smp2p_info.smem_bit);
-			if (IS_ERR(ipa3_ctx->smp2p_info.smem_state)) {
-				IPAERR("fail to get smp2p clk resp bit %ld\n",
-				PTR_ERR(ipa3_ctx->smp2p_info.smem_state));
-				return PTR_ERR(ipa3_ctx->smp2p_info.smem_state);
-			}
-			IPADBG("smem_bit=%d\n", ipa3_ctx->smp2p_info.smem_bit);
-		}
-	} else if (strcmp("qcom,smp2p_map_ipa_1_in", node->name) == 0) {
-		res = irq = of_irq_get_byname(node, "ipa-smp2p-in");
-		if (res < 0) {
-			IPADBG("of_irq_get_byname returned %d\n", irq);
-			return res;
-		}
-
-		ipa3_ctx->smp2p_info.in_base_id = irq;
-		IPADBG("smp2p irq#=%d\n", irq);
-		res = devm_request_threaded_irq(dev, irq, NULL,
-			(irq_handler_t)ipa3_smp2p_modem_clk_query_isr,
-			IRQF_TRIGGER_RISING | IRQF_ONESHOT,
-			"ipa_smp2p_clk_vote", dev);
-		if (res) {
-			IPAERR("fail to register smp2p irq=%d\n", irq);
-			return -ENODEV;
-		}
-	}
-	return 0;
-}
-
-int ipa3_plat_drv_probe(struct platform_device *pdev_p,
-	struct ipa_api_controller *api_ctrl,
-	const struct of_device_id *pdrv_match)
-{
-	int result;
-	struct device *dev = &pdev_p->dev;
-	struct ipa_smmu_cb_ctx *cb;
-
-	IPADBG("IPA driver probing started\n");
-	IPADBG("dev->of_node->name = %s\n", dev->of_node->name);
-
-	if (of_device_is_compatible(dev->of_node, "qcom,ipa-smmu-ap-cb")) {
-		if (ipa3_ctx == NULL) {
-			IPAERR("ipa3_ctx was not initialized\n");
-			return -EPROBE_DEFER;
-		}
-		cb = ipa3_get_smmu_ctx(IPA_SMMU_CB_AP);
-		cb->dev = dev;
-		smmu_info.present[IPA_SMMU_CB_AP] = true;
-
-		return 0;
-	}
-
-	if (of_device_is_compatible(dev->of_node, "qcom,ipa-smmu-wlan-cb")) {
-		if (ipa3_ctx == NULL) {
-			IPAERR("ipa3_ctx was not initialized\n");
-			return -EPROBE_DEFER;
-		}
-		cb = ipa3_get_smmu_ctx(IPA_SMMU_CB_WLAN);
-		cb->dev = dev;
-		smmu_info.present[IPA_SMMU_CB_WLAN] = true;
-
-		return 0;
-	}
-
-	if (of_device_is_compatible(dev->of_node, "qcom,ipa-smmu-uc-cb")) {
-		if (ipa3_ctx == NULL) {
-			IPAERR("ipa3_ctx was not initialized\n");
-			return -EPROBE_DEFER;
-		}
-		cb =  ipa3_get_smmu_ctx(IPA_SMMU_CB_UC);
-		cb->dev = dev;
-		smmu_info.present[IPA_SMMU_CB_UC] = true;
-
-		return 0;
-	}
-
-	if (of_device_is_compatible(dev->of_node, "qcom,ipa-smmu-11ad-cb")) {
-		if (ipa3_ctx == NULL) {
-			IPAERR("ipa3_ctx was not initialized\n");
-			return -EPROBE_DEFER;
-		}
-		cb = ipa3_get_smmu_ctx(IPA_SMMU_CB_11AD);
-		cb->dev = dev;
-		smmu_info.present[IPA_SMMU_CB_11AD] = true;
-
-		return 0;
-	}
-
-	if (of_device_is_compatible(dev->of_node,
-	    "qcom,smp2p-map-ipa-1-out"))
-		return ipa3_smp2p_probe(dev);
-	if (of_device_is_compatible(dev->of_node,
-	    "qcom,smp2p-map-ipa-1-in"))
-		return ipa3_smp2p_probe(dev);
-
-	result = get_ipa_dts_configuration(pdev_p, &ipa3_res);
-	if (result) {
-		IPAERR("IPA dts parsing failed\n");
-		return result;
-	}
-
-	result = ipa3_bind_api_controller(ipa3_res.ipa_hw_type, api_ctrl);
-	if (result) {
-		IPAERR("IPA API binding failed\n");
-		return result;
-	}
-
-	if (of_property_read_bool(pdev_p->dev.of_node, "qcom,arm-smmu")) {
-		if (of_property_read_bool(pdev_p->dev.of_node,
-			"qcom,use-64-bit-dma-mask"))
-			smmu_info.use_64_bit_dma_mask = true;
-		smmu_info.arm_smmu = true;
-	} else {
-		if (of_property_read_bool(pdev_p->dev.of_node,
-			"qcom,use-64-bit-dma-mask")) {
-			if (dma_set_mask(&pdev_p->dev, DMA_BIT_MASK(64)) ||
-			    dma_set_coherent_mask(&pdev_p->dev,
-			    DMA_BIT_MASK(64))) {
-				IPAERR("DMA set 64bit mask failed\n");
-				return -EOPNOTSUPP;
-			}
-		} else {
-			if (dma_set_mask(&pdev_p->dev, DMA_BIT_MASK(32)) ||
-			    dma_set_coherent_mask(&pdev_p->dev,
-			    DMA_BIT_MASK(32))) {
-				IPAERR("DMA set 32bit mask failed\n");
-				return -EOPNOTSUPP;
-			}
-		}
-	}
-
-	/* Proceed to real initialization */
-	result = ipa3_pre_init(&ipa3_res, pdev_p);
-	if (result) {
-		IPAERR("ipa3_init failed\n");
-		return result;
-	}
-
-	result = of_platform_populate(pdev_p->dev.of_node,
-		pdrv_match, NULL, &pdev_p->dev);
-	if (result) {
-		IPAERR("failed to populate platform\n");
-		return result;
-	}
-
-	return result;
-}
-
-/**
- * ipa3_ap_suspend() - suspend callback for runtime_pm
- * @dev: pointer to device
- *
- * This callback will be invoked by the runtime_pm framework when an AP suspend
- * operation is invoked, usually by pressing a suspend button.
- *
- * Returns -EAGAIN to runtime_pm framework in case IPA is in use by AP.
- * This will postpone the suspend operation until IPA is no longer used by AP.
- */
-int ipa3_ap_suspend(struct device *dev)
-{
-	int i;
-
-	IPADBG("Enter...\n");
-
-	/* In case there is a tx/rx handler in polling mode fail to suspend */
-	for (i = 0; i < ipa3_ctx->ipa_num_pipes; i++) {
-		if (ipa3_ctx->ep[i].sys &&
-			atomic_read(&ipa3_ctx->ep[i].sys->curr_polling_state)) {
-			IPAERR("EP %d is in polling state, do not suspend\n",
-				i);
-			return -EAGAIN;
-		}
-	}
-
-	ipa_pm_deactivate_all_deferred();
-
-	IPADBG("Exit\n");
-
-	return 0;
-}
-
-/**
- * ipa3_ap_resume() - resume callback for runtime_pm
- * @dev: pointer to device
- *
- * This callback will be invoked by the runtime_pm framework when an AP resume
- * operation is invoked.
- *
- * Always returns 0 since resume should always succeed.
- */
-int ipa3_ap_resume(struct device *dev)
-{
-	return 0;
-}
-
-struct ipa3_context *ipa3_get_ctx(void)
-{
-	return ipa3_ctx;
-}
-
-static void ipa_gsi_notify_cb(struct gsi_per_notify *notify)
-{
-	/*
-	 * These values are reported by hardware. Any error indicates
-	 * hardware unexpected state.
-	 */
-	switch (notify->evt_id) {
-	case GSI_PER_EVT_GLOB_ERROR:
-		IPAERR("Got GSI_PER_EVT_GLOB_ERROR\n");
-		IPAERR("Err_desc = 0x%04x\n", notify->data.err_desc);
-		break;
-	case GSI_PER_EVT_GLOB_GP1:
-		IPAERR("Got GSI_PER_EVT_GLOB_GP1\n");
-		ipa_assert();
-		break;
-	case GSI_PER_EVT_GLOB_GP2:
-		IPAERR("Got GSI_PER_EVT_GLOB_GP2\n");
-		ipa_assert();
-		break;
-	case GSI_PER_EVT_GLOB_GP3:
-		IPAERR("Got GSI_PER_EVT_GLOB_GP3\n");
-		ipa_assert();
-		break;
-	case GSI_PER_EVT_GENERAL_BREAK_POINT:
-		IPAERR("Got GSI_PER_EVT_GENERAL_BREAK_POINT\n");
-		break;
-	case GSI_PER_EVT_GENERAL_BUS_ERROR:
-		IPAERR("Got GSI_PER_EVT_GENERAL_BUS_ERROR\n");
-		ipa_assert();
-		break;
-	case GSI_PER_EVT_GENERAL_CMD_FIFO_OVERFLOW:
-		IPAERR("Got GSI_PER_EVT_GENERAL_CMD_FIFO_OVERFLOW\n");
-		ipa_assert();
-		break;
-	case GSI_PER_EVT_GENERAL_MCS_STACK_OVERFLOW:
-		IPAERR("Got GSI_PER_EVT_GENERAL_MCS_STACK_OVERFLOW\n");
-		ipa_assert();
-		break;
-	default:
-		IPAERR("Received unexpected evt: %d\n",
-			notify->evt_id);
-		ipa_assert();
-	}
-}
-
-int ipa3_register_ipa_ready_cb(void (*ipa_ready_cb)(void *), void *user_data)
-{
-	struct ipa3_ready_cb_info *cb_info = NULL;
-
-	/* check ipa3_ctx existed or not */
-	if (!ipa3_ctx) {
-		IPADBG("IPA driver haven't initialized\n");
-		return -ENXIO;
-	}
-	mutex_lock(&ipa3_ctx->lock);
-	if (ipa3_ctx->ipa_initialization_complete) {
-		mutex_unlock(&ipa3_ctx->lock);
-		IPADBG("IPA driver finished initialization already\n");
-		return -EEXIST;
-	}
-
-	cb_info = kmalloc(sizeof(struct ipa3_ready_cb_info), GFP_KERNEL);
-	if (!cb_info) {
-		mutex_unlock(&ipa3_ctx->lock);
-		return -ENOMEM;
-	}
-
-	cb_info->ready_cb = ipa_ready_cb;
-	cb_info->user_data = user_data;
-
-	list_add_tail(&cb_info->link, &ipa3_ctx->ipa_ready_cb_list);
-	mutex_unlock(&ipa3_ctx->lock);
-
-	return 0;
-}
-
-int ipa3_iommu_map(struct iommu_domain *domain,
-	unsigned long iova, phys_addr_t paddr, size_t size, int prot)
-{
-	struct ipa_smmu_cb_ctx *cb = NULL;
-
-	IPADBG_LOW("domain =0x%pK iova 0x%lx\n", domain, iova);
-	IPADBG_LOW("paddr =0x%pa size 0x%x\n", &paddr, (u32)size);
-
-	/* make sure no overlapping */
-	if (domain == ipa3_get_smmu_domain()) {
-		cb = ipa3_get_smmu_ctx(IPA_SMMU_CB_AP);
-		if (iova >= cb->va_start && iova < cb->va_end) {
-			IPAERR("iommu AP overlap addr 0x%lx\n", iova);
-			ipa_assert();
-			return -EFAULT;
-		}
-	} else if (domain == ipa3_get_wlan_smmu_domain()) {
-		/* wlan is one time map */
-		cb = ipa3_get_smmu_ctx(IPA_SMMU_CB_WLAN);
-	} else if (domain == ipa3_get_11ad_smmu_domain()) {
-		/* 11ad is one time map */
-		cb = ipa3_get_smmu_ctx(IPA_SMMU_CB_11AD);
-	} else if (domain == ipa3_get_uc_smmu_domain()) {
-		cb = ipa3_get_smmu_ctx(IPA_SMMU_CB_UC);
-		if (iova >= cb->va_start && iova < cb->va_end) {
-			IPAERR("iommu uC overlap addr 0x%lx\n", iova);
-			ipa_assert();
-			return -EFAULT;
-		}
-	} else {
-		IPAERR("Unexpected domain 0x%pK\n", domain);
-		ipa_assert();
-		return -EFAULT;
-	}
-
-	if (cb == NULL) {
-		IPAERR("Unexpected cb turning NULL for domain 0x%pK\n", domain);
-		ipa_assert();
-	}
-
-	/*
-	 * IOMMU_CACHE is needed to make the entries cachable
-	 * if cache coherency is enabled in dtsi.
-	 */
-	if (cb->is_cache_coherent)
-		prot |= IOMMU_CACHE;
-
-	return iommu_map(domain, iova, paddr, size, prot);
-}
-
-/**
- * ipa3_get_smmu_params()- Return the ipa3 smmu related params.
- */
-int ipa3_get_smmu_params(struct ipa_smmu_in_params *in,
-	struct ipa_smmu_out_params *out)
-{
-	bool is_smmu_enable = false;
-
-	if (out == NULL || in == NULL) {
-		IPAERR("bad parms for Client SMMU out params\n");
-		return -EINVAL;
-	}
-
-	if (!ipa3_ctx) {
-		IPAERR("IPA not yet initialized\n");
-		return -EINVAL;
-	}
-
-	out->shared_cb = false;
-
-	switch (in->smmu_client) {
-	case IPA_SMMU_WLAN_CLIENT:
-		if (ipa3_ctx->ipa_wdi3_over_gsi ||
-			ipa3_ctx->ipa_wdi2_over_gsi)
-			is_smmu_enable =
-				!(ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_AP] ||
-				ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_WLAN]);
-		else
-			is_smmu_enable =
-			!(ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_UC] ||
-			ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_WLAN]);
-		break;
-	case IPA_SMMU_WIGIG_CLIENT:
-		is_smmu_enable = !(ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_UC] ||
-			ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_11AD] ||
-			ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_AP]);
-		if (is_smmu_enable) {
-			if (ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_UC] ||
-				ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_11AD] ||
-				ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_AP]) {
-				IPAERR("11AD SMMU Discrepancy (%d %d %d)\n",
-				ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_UC],
-				ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_AP],
-				ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_11AD]);
-				WARN_ON(1);
-				return -EINVAL;
-			}
-		} else {
-			if (!ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_UC] ||
-				!ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_11AD] ||
-				!ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_AP]) {
-				IPAERR("11AD SMMU Discrepancy (%d %d %d)\n",
-				ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_UC],
-				ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_AP],
-				ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_11AD]);
-				WARN_ON(1);
-				return -EINVAL;
-			}
-		}
-		out->shared_cb = (ipa3_get_smmu_ctx(IPA_SMMU_CB_11AD))->shared;
-		break;
-	case IPA_SMMU_AP_CLIENT:
-		is_smmu_enable =
-			!(ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_AP]);
-		break;
-	default:
-		is_smmu_enable = false;
-		IPAERR("Trying to get illegal clients SMMU status");
-		return -EINVAL;
-	}
-
-	out->smmu_enable = is_smmu_enable;
-
-	return 0;
-}
-
-#define MAX_LEN 96
-
-void ipa_pc_qmp_enable(void)
-{
-	char buf[MAX_LEN] = "{class: bcm, res: ipa_pc, val: 1}";
-	struct qmp_pkt pkt;
-	int ret = 0;
-	struct ipa3_pc_mbox_data *mbox_data = &ipa3_ctx->pc_mbox;
-
-	IPADBG("Enter\n");
-
-	/* prepare the mailbox struct */
-	mbox_data->mbox_client.dev = &ipa3_ctx->master_pdev->dev;
-	mbox_data->mbox_client.tx_block = true;
-	mbox_data->mbox_client.tx_tout = MBOX_TOUT_MS;
-	mbox_data->mbox_client.knows_txdone = false;
-
-	mbox_data->mbox = mbox_request_channel(&mbox_data->mbox_client, 0);
-	if (IS_ERR(mbox_data->mbox)) {
-		ret = PTR_ERR(mbox_data->mbox);
-		if (ret != -EPROBE_DEFER)
-			IPAERR("mailbox channel request failed, ret=%d\n", ret);
-
-		return;
-	}
-
-	/* prepare the QMP packet to send */
-	pkt.size = MAX_LEN;
-	pkt.data = buf;
-
-	/* send the QMP packet to AOP */
-	ret = mbox_send_message(mbox_data->mbox, &pkt);
-	if (ret < 0)
-		IPAERR("qmp message send failed, ret=%d\n", ret);
-
-	if (mbox_data->mbox) {
-		mbox_free_channel(mbox_data->mbox);
-		mbox_data->mbox = NULL;
-	}
-}
-
-/**************************************************************
- *            PCIe Version
- *************************************************************/
-
-int ipa3_pci_drv_probe(
-	struct pci_dev            *pci_dev,
-	struct ipa_api_controller *api_ctrl,
-	const struct of_device_id *pdrv_match)
-{
-	int result;
-	struct ipa3_plat_drv_res *ipa_drv_res;
-	u32 bar0_offset;
-	u32 mem_start;
-	u32 mem_end;
-	uint32_t bits;
-	uint32_t ipa_start, gsi_start, intctrl_start;
-	struct device *dev;
-	static struct platform_device platform_dev;
-
-	if (!pci_dev || !api_ctrl || !pdrv_match) {
-		IPAERR(
-		    "Bad arg: pci_dev (%pK) and/or api_ctrl (%pK) and/or pdrv_match (%pK)\n",
-		    pci_dev, api_ctrl, pdrv_match);
-		return -EOPNOTSUPP;
-	}
-
-	dev = &(pci_dev->dev);
-
-	IPADBG("IPA PCI driver probing started\n");
-
-	/*
-	 * Follow PCI driver flow here.
-	 * pci_enable_device:  Enables device and assigns resources
-	 * pci_request_region:  Makes BAR0 address region usable
-	 */
-	result = pci_enable_device(pci_dev);
-	if (result < 0) {
-		IPAERR("pci_enable_device() failed\n");
-		return -EOPNOTSUPP;
-	}
-
-	result = pci_request_region(pci_dev, 0, "IPA Memory");
-	if (result < 0) {
-		IPAERR("pci_request_region() failed\n");
-		pci_disable_device(pci_dev);
-		return -EOPNOTSUPP;
-	}
-
-	/*
-	 * When in the PCI/emulation environment, &platform_dev is
-	 * passed to get_ipa_dts_configuration(), but is unused, since
-	 * all usages of it in the function are replaced by CPP
-	 * relative to definitions in ipa_emulation_stubs.h.  Passing
-	 * &platform_dev makes code validity tools happy.
-	 */
-	if (get_ipa_dts_configuration(&platform_dev, &ipa3_res) != 0) {
-		IPAERR("get_ipa_dts_configuration() failed\n");
-		pci_release_region(pci_dev, 0);
-		pci_disable_device(pci_dev);
-		return -EOPNOTSUPP;
-	}
-
-	ipa_drv_res = &ipa3_res;
-
-	result =
-		of_property_read_u32(NULL, "emulator-bar0-offset",
-				     &bar0_offset);
-	if (result) {
-		IPAERR(":get resource failed for emulator-bar0-offset!\n");
-		pci_release_region(pci_dev, 0);
-		pci_disable_device(pci_dev);
-		return -ENODEV;
-	}
-	IPADBG(":using emulator-bar0-offset 0x%08X\n", bar0_offset);
-
-	ipa_start     = ipa_drv_res->ipa_mem_base;
-	gsi_start     = ipa_drv_res->transport_mem_base;
-	intctrl_start = ipa_drv_res->emulator_intcntrlr_mem_base;
-
-	/*
-	 * Where will we be inerrupted at?
-	 */
-	ipa_drv_res->emulator_irq = pci_dev->irq;
-	IPADBG(
-	    "EMULATION PCI_INTERRUPT_PIN(%u)\n",
-	    ipa_drv_res->emulator_irq);
-
-	/*
-	 * Set the ipa_mem_base to the PCI base address of BAR0
-	 */
-	mem_start = pci_resource_start(pci_dev, 0);
-	mem_end   = pci_resource_end(pci_dev, 0);
-
-	IPADBG("PCI START                = 0x%x\n", mem_start);
-	IPADBG("PCI END                  = 0x%x\n", mem_end);
-
-	ipa_drv_res->ipa_mem_base = mem_start + bar0_offset;
-
-	smmu_info.ipa_base = ipa_drv_res->ipa_mem_base;
-	smmu_info.ipa_size = ipa_drv_res->ipa_mem_size;
-
-	ipa_drv_res->transport_mem_base =
-	    ipa_drv_res->ipa_mem_base + (gsi_start - ipa_start);
-
-	ipa_drv_res->emulator_intcntrlr_mem_base =
-	    ipa_drv_res->ipa_mem_base + (intctrl_start - ipa_start);
-
-	IPADBG("ipa_mem_base                = 0x%x\n",
-	       ipa_drv_res->ipa_mem_base);
-	IPADBG("ipa_mem_size                = 0x%x\n",
-	       ipa_drv_res->ipa_mem_size);
-
-	IPADBG("transport_mem_base          = 0x%x\n",
-	       ipa_drv_res->transport_mem_base);
-	IPADBG("transport_mem_size          = 0x%x\n",
-	       ipa_drv_res->transport_mem_size);
-
-	IPADBG("emulator_intcntrlr_mem_base = 0x%x\n",
-	       ipa_drv_res->emulator_intcntrlr_mem_base);
-	IPADBG("emulator_intcntrlr_mem_size = 0x%x\n",
-	       ipa_drv_res->emulator_intcntrlr_mem_size);
-
-	result = ipa3_bind_api_controller(ipa_drv_res->ipa_hw_type, api_ctrl);
-	if (result != 0) {
-		IPAERR("ipa3_bind_api_controller() failed\n");
-		pci_release_region(pci_dev, 0);
-		pci_disable_device(pci_dev);
-		return result;
-	}
-
-	bits = (ipa_drv_res->use_64_bit_dma_mask) ? 64 : 32;
-
-	if (dma_set_mask(dev, DMA_BIT_MASK(bits)) != 0) {
-		IPAERR("dma_set_mask(%pK, %u) failed\n", dev, bits);
-		pci_release_region(pci_dev, 0);
-		pci_disable_device(pci_dev);
-		return -EOPNOTSUPP;
-	}
-
-	if (dma_set_coherent_mask(dev, DMA_BIT_MASK(bits)) != 0) {
-		IPAERR("dma_set_coherent_mask(%pK, %u) failed\n", dev, bits);
-		pci_release_region(pci_dev, 0);
-		pci_disable_device(pci_dev);
-		return -EOPNOTSUPP;
-	}
-
-	pci_set_master(pci_dev);
-
-	memset(&platform_dev, 0, sizeof(platform_dev));
-	platform_dev.dev = *dev;
-
-	/* Proceed to real initialization */
-	result = ipa3_pre_init(&ipa3_res, &platform_dev);
-	if (result) {
-		IPAERR("ipa3_init failed\n");
-		pci_clear_master(pci_dev);
-		pci_release_region(pci_dev, 0);
-		pci_disable_device(pci_dev);
-		return result;
-	}
-
-	return result;
-}
-
-/*
- * The following returns transport register memory location and
- * size...
- */
-int ipa3_get_transport_info(
-	phys_addr_t *phys_addr_ptr,
-	unsigned long *size_ptr)
-{
-	if (!phys_addr_ptr || !size_ptr) {
-		IPAERR("Bad arg: phys_addr_ptr(%pK) and/or size_ptr(%pK)\n",
-		       phys_addr_ptr, size_ptr);
-		return -EINVAL;
-	}
-
-	*phys_addr_ptr = ipa3_res.transport_mem_base;
-	*size_ptr      = ipa3_res.transport_mem_size;
-
-	return 0;
-}
-EXPORT_SYMBOL(ipa3_get_transport_info);
-
-static uint emulation_type = IPA_HW_v4_0;
-
-/*
- * The following returns emulation type...
- */
-uint ipa3_get_emulation_type(void)
-{
-	return emulation_type;
-}
-
-MODULE_LICENSE("GPL v2");
-MODULE_DESCRIPTION("IPA HW device driver");
-
-/*
- * Module parameter. Invoke as follows:
- *     insmod ipat.ko emulation_type=[13|14|17|...|N]
- * Examples:
- *   insmod ipat.ko emulation_type=13 # for IPA 3.5.1
- *   insmod ipat.ko emulation_type=14 # for IPA 4.0
- *   insmod ipat.ko emulation_type=17 # for IPA 4.5
- *
- * NOTE: The emulation_type values need to come from: enum ipa_hw_type
- *
- */
-
-module_param(emulation_type, uint, 0000);
-MODULE_PARM_DESC(
-	emulation_type,
-	"emulation_type=N N can be 13 for IPA 3.5.1, 14 for IPA 4.0, 17 for IPA 4.5");

ipa/ipa_v3/ipa_client.c

@@ -1,1744 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2012-2019, The Linux Foundation. All rights reserved.
- */
-
-#include <asm/barrier.h>
-#include <linux/delay.h>
-#include <linux/device.h>
-#include "ipa_i.h"
-#include "linux/msm_gsi.h"
-
-/*
- * These values were determined empirically and shows good E2E bi-
- * directional throughputs
- */
-#define IPA_HOLB_TMR_EN 0x1
-#define IPA_HOLB_TMR_DIS 0x0
-#define IPA_POLL_AGGR_STATE_RETRIES_NUM 3
-#define IPA_POLL_AGGR_STATE_SLEEP_MSEC 1
-
-#define IPA_PKT_FLUSH_TO_US 100
-
-#define IPA_POLL_FOR_EMPTINESS_NUM 50
-#define IPA_POLL_FOR_EMPTINESS_SLEEP_USEC 20
-#define IPA_CHANNEL_STOP_IN_PROC_TO_MSEC 5
-#define IPA_CHANNEL_STOP_IN_PROC_SLEEP_USEC 200
-
-/* xfer_rsc_idx should be 7 bits */
-#define IPA_XFER_RSC_IDX_MAX 127
-
-static int ipa3_is_xdci_channel_empty(struct ipa3_ep_context *ep,
-	bool *is_empty);
-static void ipa3_start_gsi_debug_monitor(u32 clnt_hdl);
-
-int ipa3_enable_data_path(u32 clnt_hdl)
-{
-	struct ipa3_ep_context *ep = &ipa3_ctx->ep[clnt_hdl];
-	struct ipa_ep_cfg_holb holb_cfg;
-	struct ipa_ep_cfg_ctrl ep_cfg_ctrl;
-	int res = 0;
-	struct ipahal_reg_endp_init_rsrc_grp rsrc_grp;
-
-	/* Assign the resource group for pipe */
-	memset(&rsrc_grp, 0, sizeof(rsrc_grp));
-	rsrc_grp.rsrc_grp = ipa_get_ep_group(ep->client);
-	if (rsrc_grp.rsrc_grp == -1) {
-		IPAERR("invalid group for client %d\n", ep->client);
-		WARN_ON(1);
-		return -EFAULT;
-	}
-
-	IPADBG("Setting group %d for pipe %d\n",
-		rsrc_grp.rsrc_grp, clnt_hdl);
-	ipahal_write_reg_n_fields(IPA_ENDP_INIT_RSRC_GRP_n, clnt_hdl,
-		&rsrc_grp);
-
-	IPADBG("Enabling data path\n");
-	if (IPA_CLIENT_IS_CONS(ep->client)) {
-		memset(&holb_cfg, 0, sizeof(holb_cfg));
-		/*
-		 * Set HOLB on USB DPL CONS to avoid IPA stall
-		 * if DPL client is not pulling the data
-		 * on other end from IPA hw.
-		 */
-		if ((ep->client == IPA_CLIENT_USB_DPL_CONS) ||
-				(ep->client == IPA_CLIENT_MHI_DPL_CONS))
-			holb_cfg.en = IPA_HOLB_TMR_EN;
-		else
-			holb_cfg.en = IPA_HOLB_TMR_DIS;
-		holb_cfg.tmr_val = 0;
-		res = ipa3_cfg_ep_holb(clnt_hdl, &holb_cfg);
-	}
-
-	/* Enable the pipe */
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_0) {
-		if (IPA_CLIENT_IS_CONS(ep->client) &&
-		    (ep->keep_ipa_awake ||
-		    ipa3_ctx->resume_on_connect[ep->client] ||
-		    !ipa3_should_pipe_be_suspended(ep->client))) {
-			memset(&ep_cfg_ctrl, 0, sizeof(ep_cfg_ctrl));
-			ep_cfg_ctrl.ipa_ep_suspend = false;
-			res = ipa3_cfg_ep_ctrl(clnt_hdl, &ep_cfg_ctrl);
-		}
-	}
-
-	return res;
-}
-
-int ipa3_disable_data_path(u32 clnt_hdl)
-{
-	struct ipa3_ep_context *ep = &ipa3_ctx->ep[clnt_hdl];
-	struct ipa_ep_cfg_holb holb_cfg;
-	struct ipa_ep_cfg_ctrl ep_cfg_ctrl;
-	struct ipa_ep_cfg_aggr ep_aggr;
-	int res = 0;
-
-	IPADBG("Disabling data path\n");
-	if (IPA_CLIENT_IS_CONS(ep->client)) {
-		memset(&holb_cfg, 0, sizeof(holb_cfg));
-		holb_cfg.en = IPA_HOLB_TMR_EN;
-		holb_cfg.tmr_val = 0;
-		res = ipa3_cfg_ep_holb(clnt_hdl, &holb_cfg);
-	}
-
-	/*
-	 * for IPA 4.0 and above aggregation frame is closed together with
-	 * channel STOP
-	 */
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_0) {
-		/* Suspend the pipe */
-		if (IPA_CLIENT_IS_CONS(ep->client)) {
-			memset(&ep_cfg_ctrl, 0, sizeof(struct ipa_ep_cfg_ctrl));
-			ep_cfg_ctrl.ipa_ep_suspend = true;
-			res = ipa3_cfg_ep_ctrl(clnt_hdl, &ep_cfg_ctrl);
-		}
-
-		udelay(IPA_PKT_FLUSH_TO_US);
-		ipahal_read_reg_n_fields(IPA_ENDP_INIT_AGGR_n, clnt_hdl,
-			&ep_aggr);
-		if (ep_aggr.aggr_en) {
-			res = ipa3_tag_aggr_force_close(clnt_hdl);
-			if (res) {
-				IPAERR("tag process timeout client:%d err:%d\n",
-					clnt_hdl, res);
-				ipa_assert();
-			}
-		}
-	}
-
-	return res;
-}
-
-int ipa3_reset_gsi_channel(u32 clnt_hdl)
-{
-	struct ipa3_ep_context *ep;
-	int result = -EFAULT;
-	enum gsi_status gsi_res;
-	bool undo_aggr_value = false;
-	struct ipahal_reg_clkon_cfg fields;
-
-	IPADBG("entry\n");
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-		ipa3_ctx->ep[clnt_hdl].valid == 0) {
-		IPAERR("Bad parameter\n");
-		return -EINVAL;
-	}
-
-	ep = &ipa3_ctx->ep[clnt_hdl];
-
-	if (!ep->keep_ipa_awake)
-		IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	/*
-	 * IPAv4.0 HW has a limitation where WSEQ in MBIM NTH header is not
-	 * reset to 0 when MBIM pipe is reset. Workaround is to disable
-	 * HW clock gating for AGGR block using IPA_CLKON_CFG reg. undo flag to
-	 * disable the bit after reset is finished
-	 */
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0) {
-		if (ep->cfg.aggr.aggr == IPA_MBIM_16 &&
-			ep->cfg.aggr.aggr_en != IPA_BYPASS_AGGR) {
-			ipahal_read_reg_fields(IPA_CLKON_CFG, &fields);
-			if (fields.open_aggr_wrapper) {
-				undo_aggr_value = true;
-				fields.open_aggr_wrapper = false;
-				ipahal_write_reg_fields(IPA_CLKON_CFG, &fields);
-			}
-		}
-	}
-
-	/*
-	 * Reset channel
-	 * If the reset called after stop, need to wait 1ms
-	 */
-	msleep(IPA_POLL_AGGR_STATE_SLEEP_MSEC);
-	gsi_res = gsi_reset_channel(ep->gsi_chan_hdl);
-	if (gsi_res != GSI_STATUS_SUCCESS) {
-		IPAERR("Error resetting channel: %d\n", gsi_res);
-		result = -EFAULT;
-		goto reset_chan_fail;
-	}
-
-	/* undo the aggr value if flag was set above*/
-	if (undo_aggr_value) {
-		fields.open_aggr_wrapper = false;
-		ipahal_write_reg_fields(IPA_CLKON_CFG, &fields);
-	}
-
-	if (!ep->keep_ipa_awake)
-		IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	IPADBG("exit\n");
-	return 0;
-
-reset_chan_fail:
-	/* undo the aggr value if flag was set above*/
-	if (undo_aggr_value) {
-		fields.open_aggr_wrapper = false;
-		ipahal_write_reg_fields(IPA_CLKON_CFG, &fields);
-	}
-
-	if (!ep->keep_ipa_awake)
-		IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-	return result;
-}
-
-int ipa3_reset_gsi_event_ring(u32 clnt_hdl)
-{
-	struct ipa3_ep_context *ep;
-	int result = -EFAULT;
-	enum gsi_status gsi_res;
-
-	IPADBG("entry\n");
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-		ipa3_ctx->ep[clnt_hdl].valid == 0) {
-		IPAERR("Bad parameter.\n");
-		return -EINVAL;
-	}
-
-	ep = &ipa3_ctx->ep[clnt_hdl];
-
-	if (!ep->keep_ipa_awake)
-		IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-	/* Reset event ring */
-	gsi_res = gsi_reset_evt_ring(ep->gsi_evt_ring_hdl);
-	if (gsi_res != GSI_STATUS_SUCCESS) {
-		IPAERR("Error resetting event: %d\n", gsi_res);
-		result = -EFAULT;
-		goto reset_evt_fail;
-	}
-
-	if (!ep->keep_ipa_awake)
-		IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	IPADBG("exit\n");
-	return 0;
-
-reset_evt_fail:
-	if (!ep->keep_ipa_awake)
-		IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-	return result;
-}
-
-static bool ipa3_is_legal_params(struct ipa_request_gsi_channel_params *params)
-{
-	if (params->client >= IPA_CLIENT_MAX)
-		return false;
-	else
-		return true;
-}
-
-static void ipa3_start_gsi_debug_monitor(u32 clnt_hdl)
-{
-	struct IpaHwOffloadStatsAllocCmdData_t *gsi_info;
-	struct ipa3_ep_context *ep;
-	enum ipa_client_type client_type;
-
-	IPADBG("entry\n");
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-		ipa3_ctx->ep[clnt_hdl].valid == 0) {
-		IPAERR("Bad parameters.\n");
-		return;
-	}
-
-	ep = &ipa3_ctx->ep[clnt_hdl];
-	client_type = ipa3_get_client_mapping(clnt_hdl);
-
-	/* start uC gsi dbg stats monitor */
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_5 &&
-		ipa3_ctx->ipa_hw_type != IPA_HW_v4_7) {
-		switch (client_type) {
-		case IPA_CLIENT_MHI_PRIME_TETH_PROD:
-			gsi_info = &ipa3_ctx->gsi_info[IPA_HW_PROTOCOL_MHIP];
-			gsi_info->ch_id_info[0].ch_id = ep->gsi_chan_hdl;
-			gsi_info->ch_id_info[0].dir = DIR_PRODUCER;
-			ipa3_uc_debug_stats_alloc(*gsi_info);
-			break;
-		case IPA_CLIENT_MHI_PRIME_TETH_CONS:
-			gsi_info = &ipa3_ctx->gsi_info[IPA_HW_PROTOCOL_MHIP];
-			gsi_info->ch_id_info[1].ch_id = ep->gsi_chan_hdl;
-			gsi_info->ch_id_info[1].dir = DIR_CONSUMER;
-			ipa3_uc_debug_stats_alloc(*gsi_info);
-			break;
-		case IPA_CLIENT_MHI_PRIME_RMNET_PROD:
-			gsi_info = &ipa3_ctx->gsi_info[IPA_HW_PROTOCOL_MHIP];
-			gsi_info->ch_id_info[2].ch_id = ep->gsi_chan_hdl;
-			gsi_info->ch_id_info[2].dir = DIR_PRODUCER;
-			ipa3_uc_debug_stats_alloc(*gsi_info);
-			break;
-		case IPA_CLIENT_MHI_PRIME_RMNET_CONS:
-			gsi_info = &ipa3_ctx->gsi_info[IPA_HW_PROTOCOL_MHIP];
-			gsi_info->ch_id_info[3].ch_id = ep->gsi_chan_hdl;
-			gsi_info->ch_id_info[3].dir = DIR_CONSUMER;
-			ipa3_uc_debug_stats_alloc(*gsi_info);
-			break;
-		case IPA_CLIENT_USB_PROD:
-			gsi_info = &ipa3_ctx->gsi_info[IPA_HW_PROTOCOL_USB];
-			gsi_info->ch_id_info[0].ch_id = ep->gsi_chan_hdl;
-			gsi_info->ch_id_info[0].dir = DIR_PRODUCER;
-			ipa3_uc_debug_stats_alloc(*gsi_info);
-			break;
-		case IPA_CLIENT_USB_CONS:
-			gsi_info = &ipa3_ctx->gsi_info[IPA_HW_PROTOCOL_USB];
-			gsi_info->ch_id_info[1].ch_id = ep->gsi_chan_hdl;
-			gsi_info->ch_id_info[1].dir = DIR_CONSUMER;
-			ipa3_uc_debug_stats_alloc(*gsi_info);
-			break;
-		default:
-			IPADBG("client_type %d not supported\n",
-				client_type);
-		}
-	}
-}
-
-int ipa3_smmu_map_peer_reg(phys_addr_t phys_addr, bool map,
-	enum ipa_smmu_cb_type cb_type)
-{
-	struct iommu_domain *smmu_domain;
-	int res;
-
-	if (!VALID_IPA_SMMU_CB_TYPE(cb_type)) {
-		IPAERR("invalid cb_type\n");
-		return -EINVAL;
-	}
-
-	if (ipa3_ctx->s1_bypass_arr[cb_type]) {
-		IPADBG("CB# %d is set to s1 bypass\n", cb_type);
-		return 0;
-	}
-
-	smmu_domain = ipa3_get_smmu_domain_by_type(cb_type);
-	if (!smmu_domain) {
-		IPAERR("invalid smmu domain\n");
-		return -EINVAL;
-	}
-
-	if (map) {
-		res = ipa3_iommu_map(smmu_domain, phys_addr, phys_addr,
-			PAGE_SIZE, IOMMU_READ | IOMMU_WRITE | IOMMU_MMIO);
-	} else {
-		res = iommu_unmap(smmu_domain, phys_addr, PAGE_SIZE);
-		res = (res != PAGE_SIZE);
-	}
-	if (res) {
-		IPAERR("Fail to %s reg 0x%pa\n", map ? "map" : "unmap",
-			&phys_addr);
-		return -EINVAL;
-	}
-
-	IPADBG("Peer reg 0x%pa %s\n", &phys_addr, map ? "map" : "unmap");
-
-	return 0;
-}
-
-int ipa3_smmu_map_peer_buff(u64 iova, u32 size, bool map, struct sg_table *sgt,
-	enum ipa_smmu_cb_type cb_type)
-{
-	struct iommu_domain *smmu_domain;
-	int res;
-	phys_addr_t phys;
-	unsigned long va;
-	struct scatterlist *sg;
-	int count = 0;
-	size_t len;
-	int i;
-	struct page *page;
-
-	if (!VALID_IPA_SMMU_CB_TYPE(cb_type)) {
-		IPAERR("invalid cb_type\n");
-		return -EINVAL;
-	}
-
-	if (ipa3_ctx->s1_bypass_arr[cb_type]) {
-		IPADBG("CB# %d is set to s1 bypass\n", cb_type);
-		return 0;
-	}
-
-	smmu_domain = ipa3_get_smmu_domain_by_type(cb_type);
-	if (!smmu_domain) {
-		IPAERR("invalid smmu domain\n");
-		return -EINVAL;
-	}
-
-	/*
-	 * USB GSI driver would update sgt irrespective of USB S1
-	 * is enable or bypass.
-	 * If USB S1 is enabled using IOMMU, iova != pa.
-	 * If USB S1 is bypass, iova == pa.
-	 */
-	if (map) {
-		if (sgt != NULL) {
-			va = rounddown(iova, PAGE_SIZE);
-			for_each_sg(sgt->sgl, sg, sgt->nents, i) {
-				page = sg_page(sg);
-				phys = page_to_phys(page);
-				len = PAGE_ALIGN(sg->offset + sg->length);
-				res = ipa3_iommu_map(smmu_domain, va, phys,
-					len, IOMMU_READ | IOMMU_WRITE);
-				if (res) {
-					IPAERR("Fail to map pa=%pa\n", &phys);
-					return -EINVAL;
-				}
-				va += len;
-				count++;
-			}
-		} else {
-			res = ipa3_iommu_map(smmu_domain,
-				rounddown(iova, PAGE_SIZE),
-				rounddown(iova, PAGE_SIZE),
-				roundup(size + iova -
-					rounddown(iova, PAGE_SIZE),
-				PAGE_SIZE),
-				IOMMU_READ | IOMMU_WRITE);
-			if (res) {
-				IPAERR("Fail to map 0x%llx\n", iova);
-				return -EINVAL;
-			}
-		}
-	} else {
-		res = iommu_unmap(smmu_domain,
-		rounddown(iova, PAGE_SIZE),
-		roundup(size + iova - rounddown(iova, PAGE_SIZE),
-		PAGE_SIZE));
-		if (res != roundup(size + iova - rounddown(iova, PAGE_SIZE),
-			PAGE_SIZE)) {
-			IPAERR("Fail to unmap 0x%llx\n", iova);
-			return -EINVAL;
-		}
-	}
-	IPADBG("Peer buff %s 0x%llx\n", map ? "map" : "unmap", iova);
-	return 0;
-}
-
-static enum ipa_client_cb_type ipa_get_client_cb_type(
-					enum ipa_client_type client_type)
-{
-	enum ipa_client_cb_type client_cb;
-
-	if (client_type == IPA_CLIENT_USB_PROD ||
-			client_type == IPA_CLIENT_USB_CONS) {
-		IPADBG("USB Client registered\n");
-		client_cb = IPA_USB_CLNT;
-	} else if (client_type == IPA_CLIENT_MHI_PROD ||
-			client_type == IPA_CLIENT_MHI_CONS) {
-		IPADBG("MHI Client registered\n");
-		client_cb = IPA_MHI_CLNT;
-	} else {
-		IPAERR("Invalid IPA client\n");
-		client_cb = IPA_MAX_CLNT;
-	}
-
-	return client_cb;
-}
-void ipa3_register_client_callback(int (*client_cb)(bool is_lock),
-				bool (*teth_port_state)(void),
-				enum ipa_client_type client_type)
-{
-	enum ipa_client_cb_type client;
-
-	IPADBG("entry\n");
-
-	client = ipa_get_client_cb_type(client_type);
-	if (client == IPA_MAX_CLNT)
-		return;
-
-	if (client_cb == NULL) {
-		IPAERR("Bad Param");
-		return;
-	}
-
-	if (!ipa3_ctx->client_lock_unlock[client])
-		ipa3_ctx->client_lock_unlock[client] = client_cb;
-	if (!ipa3_ctx->get_teth_port_state[client])
-		ipa3_ctx->get_teth_port_state[client] = teth_port_state;
-	IPADBG("exit\n");
-}
-
-void ipa3_deregister_client_callback(enum ipa_client_type client_type)
-{
-	enum ipa_client_cb_type client_cb;
-
-	IPADBG("entry\n");
-
-	client_cb = ipa_get_client_cb_type(client_type);
-	if (client_cb == IPA_MAX_CLNT)
-		return;
-
-	if (ipa3_ctx->client_lock_unlock[client_cb] == NULL &&
-		ipa3_ctx->get_teth_port_state[client_cb] == NULL) {
-		IPAERR("client_lock_unlock is already NULL");
-		return;
-	}
-
-	ipa3_ctx->client_lock_unlock[client_cb] = NULL;
-	ipa3_ctx->get_teth_port_state[client_cb] = NULL;
-	IPADBG("exit\n");
-}
-
-static void client_lock_unlock_cb(enum ipa_client_type client, bool is_lock)
-{
-	enum ipa_client_cb_type client_cb;
-
-	IPADBG("entry\n");
-
-	client_cb = ipa_get_client_cb_type(client);
-	if (client_cb == IPA_MAX_CLNT)
-		return;
-
-	if (ipa3_ctx->client_lock_unlock[client_cb])
-		ipa3_ctx->client_lock_unlock[client_cb](is_lock);
-
-	IPADBG("exit\n");
-}
-
-int ipa3_request_gsi_channel(struct ipa_request_gsi_channel_params *params,
-			     struct ipa_req_chan_out_params *out_params)
-{
-	int ipa_ep_idx;
-	int result = -EFAULT;
-	struct ipa3_ep_context *ep;
-	struct ipahal_reg_ep_cfg_status ep_status;
-	unsigned long gsi_dev_hdl;
-	enum gsi_status gsi_res;
-	const struct ipa_gsi_ep_config *gsi_ep_cfg_ptr;
-
-	IPADBG("entry\n");
-	if (params == NULL || out_params == NULL ||
-		!ipa3_is_legal_params(params)) {
-		IPAERR("bad parameters\n");
-		return -EINVAL;
-	}
-
-	ipa_ep_idx = ipa3_get_ep_mapping(params->client);
-	if (ipa_ep_idx == -1) {
-		IPAERR("fail to alloc EP.\n");
-		goto fail;
-	}
-
-	ep = &ipa3_ctx->ep[ipa_ep_idx];
-
-	if (ep->valid) {
-		IPAERR("EP already allocated.\n");
-		goto fail;
-	}
-
-	memset(&ipa3_ctx->ep[ipa_ep_idx], 0, sizeof(struct ipa3_ep_context));
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	ep->skip_ep_cfg = params->skip_ep_cfg;
-	ep->valid = 1;
-	ep->client = params->client;
-	ep->client_notify = params->notify;
-	ep->priv = params->priv;
-	ep->keep_ipa_awake = params->keep_ipa_awake;
-
-
-	/* Config QMB for USB_CONS ep */
-	if (!IPA_CLIENT_IS_PROD(ep->client)) {
-		IPADBG("Configuring QMB on USB CONS pipe\n");
-		if (ipa_ep_idx >= ipa3_ctx->ipa_num_pipes ||
-			ipa3_ctx->ep[ipa_ep_idx].valid == 0) {
-			IPAERR("bad parm.\n");
-			return -EINVAL;
-		}
-		result = ipa3_cfg_ep_cfg(ipa_ep_idx, &params->ipa_ep_cfg.cfg);
-		if (result) {
-			IPAERR("fail to configure QMB.\n");
-			return result;
-		}
-	}
-
-	if (!ep->skip_ep_cfg) {
-		if (ipa3_cfg_ep(ipa_ep_idx, &params->ipa_ep_cfg)) {
-			IPAERR("fail to configure EP.\n");
-			goto ipa_cfg_ep_fail;
-		}
-		/* Setting EP status 0 */
-		memset(&ep_status, 0, sizeof(ep_status));
-		if (ipa3_cfg_ep_status(ipa_ep_idx, &ep_status)) {
-			IPAERR("fail to configure status of EP.\n");
-			goto ipa_cfg_ep_fail;
-		}
-		IPADBG("ep configuration successful\n");
-	} else {
-		IPADBG("Skipping endpoint configuration.\n");
-	}
-
-	out_params->clnt_hdl = ipa_ep_idx;
-
-	result = ipa3_enable_data_path(out_params->clnt_hdl);
-	if (result) {
-		IPAERR("enable data path failed res=%d clnt=%d.\n", result,
-				out_params->clnt_hdl);
-		goto ipa_cfg_ep_fail;
-	}
-
-	gsi_dev_hdl = ipa3_ctx->gsi_dev_hdl;
-	gsi_res = gsi_alloc_evt_ring(&params->evt_ring_params, gsi_dev_hdl,
-		&ep->gsi_evt_ring_hdl);
-	if (gsi_res != GSI_STATUS_SUCCESS) {
-		IPAERR("Error allocating event ring: %d\n", gsi_res);
-		result = -EFAULT;
-		goto ipa_cfg_ep_fail;
-	}
-
-	gsi_res = gsi_write_evt_ring_scratch(ep->gsi_evt_ring_hdl,
-		params->evt_scratch);
-	if (gsi_res != GSI_STATUS_SUCCESS) {
-		IPAERR("Error writing event ring scratch: %d\n", gsi_res);
-		result = -EFAULT;
-		goto write_evt_scratch_fail;
-	}
-
-	gsi_ep_cfg_ptr = ipa3_get_gsi_ep_info(ep->client);
-	if (gsi_ep_cfg_ptr == NULL) {
-		IPAERR("Error ipa3_get_gsi_ep_info ret NULL\n");
-		result = -EFAULT;
-		goto write_evt_scratch_fail;
-	}
-
-	params->chan_params.evt_ring_hdl = ep->gsi_evt_ring_hdl;
-	params->chan_params.ch_id = gsi_ep_cfg_ptr->ipa_gsi_chan_num;
-	params->chan_params.prefetch_mode = gsi_ep_cfg_ptr->prefetch_mode;
-	params->chan_params.empty_lvl_threshold =
-		gsi_ep_cfg_ptr->prefetch_threshold;
-	gsi_res = gsi_alloc_channel(&params->chan_params, gsi_dev_hdl,
-		&ep->gsi_chan_hdl);
-	if (gsi_res != GSI_STATUS_SUCCESS) {
-		IPAERR("Error allocating channel: %d, chan_id: %d\n", gsi_res,
-			params->chan_params.ch_id);
-		result = -EFAULT;
-		goto write_evt_scratch_fail;
-	}
-
-	memcpy(&ep->chan_scratch, &params->chan_scratch,
-		sizeof(union __packed gsi_channel_scratch));
-
-	/*
-	 * Update scratch for MCS smart prefetch:
-	 * Starting IPA4.5, smart prefetch implemented by H/W.
-	 * At IPA 4.0/4.1/4.2, we do not use MCS smart prefetch
-	 *  so keep the fields zero.
-	 */
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_0) {
-		ep->chan_scratch.xdci.max_outstanding_tre =
-		params->chan_params.re_size * gsi_ep_cfg_ptr->ipa_if_tlv;
-	}
-
-	gsi_res = gsi_write_channel_scratch(ep->gsi_chan_hdl,
-		params->chan_scratch);
-	if (gsi_res != GSI_STATUS_SUCCESS) {
-		IPAERR("Error writing channel scratch: %d\n", gsi_res);
-		result = -EFAULT;
-		goto write_chan_scratch_fail;
-	}
-
-	gsi_res = gsi_query_channel_db_addr(ep->gsi_chan_hdl,
-		&out_params->db_reg_phs_addr_lsb,
-		&out_params->db_reg_phs_addr_msb);
-	if (gsi_res != GSI_STATUS_SUCCESS) {
-		IPAERR("Error querying channel DB registers addresses: %d\n",
-			gsi_res);
-		result = -EFAULT;
-		goto write_chan_scratch_fail;
-	}
-
-	ep->gsi_mem_info.evt_ring_len = params->evt_ring_params.ring_len;
-	ep->gsi_mem_info.evt_ring_base_addr =
-		params->evt_ring_params.ring_base_addr;
-	ep->gsi_mem_info.evt_ring_base_vaddr =
-		params->evt_ring_params.ring_base_vaddr;
-	ep->gsi_mem_info.chan_ring_len = params->chan_params.ring_len;
-	ep->gsi_mem_info.chan_ring_base_addr =
-		params->chan_params.ring_base_addr;
-	ep->gsi_mem_info.chan_ring_base_vaddr =
-		params->chan_params.ring_base_vaddr;
-
-	ipa3_ctx->skip_ep_cfg_shadow[ipa_ep_idx] = ep->skip_ep_cfg;
-	if (!ep->skip_ep_cfg && IPA_CLIENT_IS_PROD(params->client))
-		ipa3_install_dflt_flt_rules(ipa_ep_idx);
-
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-
-	IPADBG("client %d (ep: %d) connected\n", params->client, ipa_ep_idx);
-	IPADBG("exit\n");
-
-	return 0;
-
-write_chan_scratch_fail:
-	gsi_dealloc_channel(ep->gsi_chan_hdl);
-write_evt_scratch_fail:
-	gsi_dealloc_evt_ring(ep->gsi_evt_ring_hdl);
-ipa_cfg_ep_fail:
-	memset(&ipa3_ctx->ep[ipa_ep_idx], 0, sizeof(struct ipa3_ep_context));
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-fail:
-	return result;
-}
-
-int ipa3_set_usb_max_packet_size(
-	enum ipa_usb_max_usb_packet_size usb_max_packet_size)
-{
-	struct gsi_device_scratch dev_scratch;
-	enum gsi_status gsi_res;
-
-	IPADBG("entry\n");
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	memset(&dev_scratch, 0, sizeof(struct gsi_device_scratch));
-	dev_scratch.mhi_base_chan_idx_valid = false;
-	dev_scratch.max_usb_pkt_size_valid = true;
-	dev_scratch.max_usb_pkt_size = usb_max_packet_size;
-
-	gsi_res = gsi_write_device_scratch(ipa3_ctx->gsi_dev_hdl,
-		&dev_scratch);
-	if (gsi_res != GSI_STATUS_SUCCESS) {
-		IPAERR("Error writing device scratch: %d\n", gsi_res);
-		return -EFAULT;
-	}
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-
-	IPADBG("exit\n");
-	return 0;
-}
-
-/* This function called as part of usb pipe resume */
-int ipa3_xdci_connect(u32 clnt_hdl)
-{
-	int result;
-	struct ipa3_ep_context *ep;
-
-	IPADBG("entry\n");
-
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-		ipa3_ctx->ep[clnt_hdl].valid == 0) {
-		IPAERR("Bad parameter.\n");
-		return -EINVAL;
-	}
-
-	ep = &ipa3_ctx->ep[clnt_hdl];
-	IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	result = ipa3_start_gsi_channel(clnt_hdl);
-	if (result) {
-		IPAERR("failed to start gsi channel clnt_hdl=%u\n", clnt_hdl);
-		goto exit;
-	}
-
-	result = ipa3_enable_data_path(clnt_hdl);
-	if (result) {
-		IPAERR("enable data path failed res=%d clnt_hdl=%d.\n", result,
-			clnt_hdl);
-		goto stop_ch;
-	}
-
-	IPADBG("exit\n");
-	goto exit;
-
-stop_ch:
-	(void)ipa3_stop_gsi_channel(clnt_hdl);
-exit:
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-	return result;
-}
-
-
-/* This function called as part of usb pipe connect */
-int ipa3_xdci_start(u32 clnt_hdl, u8 xferrscidx, bool xferrscidx_valid)
-{
-	struct ipa3_ep_context *ep;
-	int result = -EFAULT;
-	enum gsi_status gsi_res;
-	struct ipa_ep_cfg_ctrl ep_cfg_ctrl;
-
-	IPADBG("entry\n");
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes  ||
-		ipa3_ctx->ep[clnt_hdl].valid == 0 ||
-		xferrscidx > IPA_XFER_RSC_IDX_MAX) {
-		IPAERR("Bad parameters.\n");
-		return -EINVAL;
-	}
-
-	ep = &ipa3_ctx->ep[clnt_hdl];
-	IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	if (xferrscidx_valid) {
-		ep->chan_scratch.xdci.xferrscidx = xferrscidx;
-		gsi_res = gsi_write_channel_scratch(ep->gsi_chan_hdl,
-			ep->chan_scratch);
-		if (gsi_res != GSI_STATUS_SUCCESS) {
-			IPAERR("Error writing channel scratch: %d\n", gsi_res);
-			goto write_chan_scratch_fail;
-		}
-	}
-
-	if (IPA_CLIENT_IS_PROD(ep->client) && ep->skip_ep_cfg) {
-		memset(&ep_cfg_ctrl, 0, sizeof(struct ipa_ep_cfg_ctrl));
-		ep_cfg_ctrl.ipa_ep_delay = true;
-		ep->ep_delay_set = true;
-
-		result = ipa3_cfg_ep_ctrl(clnt_hdl, &ep_cfg_ctrl);
-		if (result)
-			IPAERR("client (ep: %d) failed result=%d\n",
-			clnt_hdl, result);
-		else
-			IPADBG("client (ep: %d) success\n", clnt_hdl);
-	} else {
-		ep->ep_delay_set = false;
-	}
-
-	gsi_res = gsi_start_channel(ep->gsi_chan_hdl);
-	if (gsi_res != GSI_STATUS_SUCCESS) {
-		IPAERR("Error starting channel: %d\n", gsi_res);
-		goto write_chan_scratch_fail;
-	}
-	ipa3_start_gsi_debug_monitor(clnt_hdl);
-	if (!ep->keep_ipa_awake)
-		IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	IPADBG("exit\n");
-	return 0;
-
-write_chan_scratch_fail:
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-	return result;
-}
-
-int ipa3_get_gsi_chan_info(struct gsi_chan_info *gsi_chan_info,
-	unsigned long chan_hdl)
-{
-	enum gsi_status gsi_res;
-
-	memset(gsi_chan_info, 0, sizeof(struct gsi_chan_info));
-	gsi_res = gsi_query_channel_info(chan_hdl, gsi_chan_info);
-	if (gsi_res != GSI_STATUS_SUCCESS) {
-		IPAERR("Error querying channel info: %d\n", gsi_res);
-		return -EFAULT;
-	}
-	if (!gsi_chan_info->evt_valid) {
-		IPAERR("Event info invalid\n");
-		return -EFAULT;
-	}
-
-	return 0;
-}
-
-static bool ipa3_is_xdci_channel_with_given_info_empty(
-	struct ipa3_ep_context *ep, struct gsi_chan_info *chan_info)
-{
-	bool is_empty = false;
-
-	if (!IPA_CLIENT_IS_CONS(ep->client)) {
-		/* For UL channel: chan.RP == chan.WP */
-		is_empty = (chan_info->rp == chan_info->wp);
-	} else {
-		/* For DL channel: */
-		if (chan_info->wp !=
-		    (ep->gsi_mem_info.chan_ring_base_addr +
-		     ep->gsi_mem_info.chan_ring_len -
-		     GSI_CHAN_RE_SIZE_16B)) {
-			/*  if chan.WP != LINK TRB: chan.WP == evt.RP */
-			is_empty = (chan_info->wp == chan_info->evt_rp);
-		} else {
-			/*
-			 * if chan.WP == LINK TRB: chan.base_xfer_ring_addr
-			 * == evt.RP
-			 */
-			is_empty = (ep->gsi_mem_info.chan_ring_base_addr ==
-				chan_info->evt_rp);
-		}
-	}
-
-	return is_empty;
-}
-
-static int ipa3_is_xdci_channel_empty(struct ipa3_ep_context *ep,
-	bool *is_empty)
-{
-	struct gsi_chan_info chan_info;
-	int res;
-
-	if (!ep || !is_empty || !ep->valid) {
-		IPAERR("Input Error\n");
-		return -EFAULT;
-	}
-
-	res = ipa3_get_gsi_chan_info(&chan_info, ep->gsi_chan_hdl);
-	if (res) {
-		IPAERR("Failed to get GSI channel info\n");
-		return -EFAULT;
-	}
-
-	*is_empty = ipa3_is_xdci_channel_with_given_info_empty(ep, &chan_info);
-
-	return 0;
-}
-
-int ipa3_enable_force_clear(u32 request_id, bool throttle_source,
-	u32 source_pipe_bitmask)
-{
-	struct ipa_enable_force_clear_datapath_req_msg_v01 req;
-	int result;
-
-	if (ipa3_ctx->platform_type == IPA_PLAT_TYPE_APQ) {
-		IPADBG("APQ platform - ignore force clear\n");
-		return 0;
-	}
-
-	memset(&req, 0, sizeof(req));
-	req.request_id = request_id;
-	req.source_pipe_bitmask = source_pipe_bitmask;
-	if (throttle_source) {
-		req.throttle_source_valid = 1;
-		req.throttle_source = 1;
-	}
-	result = ipa3_qmi_enable_force_clear_datapath_send(&req);
-	if (result) {
-		IPAERR("ipa3_qmi_enable_force_clear_datapath_send failed %d\n",
-			result);
-		return result;
-	}
-
-	return 0;
-}
-
-int ipa3_disable_force_clear(u32 request_id)
-{
-	struct ipa_disable_force_clear_datapath_req_msg_v01 req;
-	int result;
-
-	if (ipa3_ctx->platform_type == IPA_PLAT_TYPE_APQ) {
-		IPADBG("APQ platform - ignore force clear\n");
-		return 0;
-	}
-
-	memset(&req, 0, sizeof(req));
-	req.request_id = request_id;
-	result = ipa3_qmi_disable_force_clear_datapath_send(&req);
-	if (result) {
-		IPAERR("ipa3_qmi_disable_force_clear_datapath_send failed %d\n",
-			result);
-		return result;
-	}
-
-	return 0;
-}
-
-/* Clocks should be voted before invoking this function */
-static int ipa3_xdci_stop_gsi_channel(u32 clnt_hdl, bool *stop_in_proc)
-{
-	int res;
-
-	IPADBG("entry\n");
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-		ipa3_ctx->ep[clnt_hdl].valid == 0 ||
-		!stop_in_proc) {
-		IPAERR("Bad parameter.\n");
-		return -EINVAL;
-	}
-
-	res = ipa3_stop_gsi_channel(clnt_hdl);
-	if (res != 0 && res != -GSI_STATUS_AGAIN &&
-		res != -GSI_STATUS_TIMED_OUT) {
-		IPAERR("xDCI stop channel failed res=%d\n", res);
-		return -EFAULT;
-	}
-
-	if (res)
-		*stop_in_proc = true;
-	else
-		*stop_in_proc = false;
-
-	IPADBG("xDCI channel is %s (result=%d)\n",
-		res ? "STOP_IN_PROC/TimeOut" : "STOP", res);
-
-	IPADBG("exit\n");
-	return 0;
-}
-
-/* Clocks should be voted before invoking this function */
-static int ipa3_xdci_stop_gsi_ch_brute_force(u32 clnt_hdl,
-	bool *stop_in_proc)
-{
-	unsigned long jiffies_start;
-	unsigned long jiffies_timeout =
-		msecs_to_jiffies(IPA_CHANNEL_STOP_IN_PROC_TO_MSEC);
-	int res;
-
-	IPADBG("entry\n");
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-		ipa3_ctx->ep[clnt_hdl].valid == 0 ||
-		!stop_in_proc) {
-		IPAERR("Bad parameter.\n");
-		return -EINVAL;
-	}
-
-	jiffies_start = jiffies;
-	while (1) {
-		res = ipa3_xdci_stop_gsi_channel(clnt_hdl,
-			stop_in_proc);
-		if (res) {
-			IPAERR("failed to stop xDCI channel hdl=%d\n",
-				clnt_hdl);
-			return res;
-		}
-
-		if (!*stop_in_proc) {
-			IPADBG("xDCI channel STOP hdl=%d\n", clnt_hdl);
-			return res;
-		}
-
-		/*
-		 * Give chance to the previous stop request to be accomplished
-		 * before the retry
-		 */
-		udelay(IPA_CHANNEL_STOP_IN_PROC_SLEEP_USEC);
-
-		if (time_after(jiffies, jiffies_start + jiffies_timeout)) {
-			IPADBG("timeout waiting for xDCI channel emptiness\n");
-			return res;
-		}
-	}
-}
-
-/* Clocks should be voted for before invoking this function */
-static int ipa3_stop_ul_chan_with_data_drain(u32 qmi_req_id,
-		u32 source_pipe_bitmask, bool should_force_clear, u32 clnt_hdl,
-		bool remove_delay)
-{
-	int result;
-	bool is_empty = false;
-	int i;
-	bool stop_in_proc;
-	struct ipa3_ep_context *ep;
-	struct ipa_ep_cfg_ctrl ep_cfg_ctrl;
-
-	IPADBG("entry\n");
-
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-		ipa3_ctx->ep[clnt_hdl].valid == 0) {
-		IPAERR("Bad parameter.\n");
-		return -EINVAL;
-	}
-
-	ep = &ipa3_ctx->ep[clnt_hdl];
-
-	/* first try to stop the channel */
-	result = ipa3_xdci_stop_gsi_ch_brute_force(clnt_hdl,
-			&stop_in_proc);
-	if (result) {
-		IPAERR("fail to stop UL channel - hdl=%d clnt=%d\n",
-			clnt_hdl, ep->client);
-		goto exit;
-	}
-	if (!stop_in_proc)
-		goto exit;
-
-	if (remove_delay && ep->ep_delay_set) {
-		memset(&ep_cfg_ctrl, 0, sizeof(struct ipa_ep_cfg_ctrl));
-		ep_cfg_ctrl.ipa_ep_delay = false;
-		result = ipa3_cfg_ep_ctrl(clnt_hdl,
-			&ep_cfg_ctrl);
-		if (result) {
-			IPAERR
-			("client (ep: %d) failed to remove delay result=%d\n",
-				clnt_hdl, result);
-		} else {
-			IPADBG("client (ep: %d) delay removed\n",
-				clnt_hdl);
-			ep->ep_delay_set = false;
-		}
-	}
-
-	/* if stop_in_proc, lets wait for emptiness */
-	for (i = 0; i < IPA_POLL_FOR_EMPTINESS_NUM; i++) {
-		result = ipa3_is_xdci_channel_empty(ep, &is_empty);
-		if (result)
-			goto exit;
-		if (is_empty)
-			break;
-		udelay(IPA_POLL_FOR_EMPTINESS_SLEEP_USEC);
-	}
-	/* In case of empty, lets try to stop the channel again */
-	if (is_empty) {
-		result = ipa3_xdci_stop_gsi_ch_brute_force(clnt_hdl,
-			&stop_in_proc);
-		if (result) {
-			IPAERR("fail to stop UL channel - hdl=%d clnt=%d\n",
-				clnt_hdl, ep->client);
-			goto exit;
-		}
-		if (!stop_in_proc)
-			goto exit;
-	}
-	/* if still stop_in_proc or not empty, activate force clear */
-	if (should_force_clear) {
-		result = ipa3_enable_force_clear(qmi_req_id, false,
-			source_pipe_bitmask);
-		if (result) {
-			struct ipahal_ep_cfg_ctrl_scnd ep_ctrl_scnd = { 0 };
-
-			/*
-			 * assuming here modem SSR\shutdown, AP can remove
-			 * the delay in this case
-			 */
-			IPAERR(
-				"failed to force clear %d, remove delay from SCND reg\n"
-				, result);
-			ep_ctrl_scnd.endp_delay = false;
-			ipahal_write_reg_n_fields(
-				IPA_ENDP_INIT_CTRL_SCND_n, clnt_hdl,
-				&ep_ctrl_scnd);
-		}
-	}
-	/* with force clear, wait for emptiness */
-	for (i = 0; i < IPA_POLL_FOR_EMPTINESS_NUM; i++) {
-		result = ipa3_is_xdci_channel_empty(ep, &is_empty);
-		if (result)
-			goto disable_force_clear_and_exit;
-		if (is_empty)
-			break;
-
-		udelay(IPA_POLL_FOR_EMPTINESS_SLEEP_USEC);
-	}
-	/* try to stop for the last time */
-	result = ipa3_xdci_stop_gsi_ch_brute_force(clnt_hdl,
-		&stop_in_proc);
-	if (result) {
-		IPAERR("fail to stop UL channel - hdl=%d clnt=%d\n",
-			clnt_hdl, ep->client);
-		goto disable_force_clear_and_exit;
-	}
-	result = stop_in_proc ? -EFAULT : 0;
-
-disable_force_clear_and_exit:
-	if (should_force_clear)
-		ipa3_disable_force_clear(qmi_req_id);
-exit:
-	if (remove_delay && ep->ep_delay_set) {
-		memset(&ep_cfg_ctrl, 0, sizeof(struct ipa_ep_cfg_ctrl));
-		ep_cfg_ctrl.ipa_ep_delay = false;
-		result = ipa3_cfg_ep_ctrl(clnt_hdl,
-			&ep_cfg_ctrl);
-		if (result) {
-			IPAERR
-			("client (ep: %d) failed to remove delay result=%d\n",
-				clnt_hdl, result);
-		} else {
-			IPADBG("client (ep: %d) delay removed\n",
-				clnt_hdl);
-			ep->ep_delay_set = false;
-		}
-	}
-	IPADBG("exit\n");
-	return result;
-}
-
-/*
- * Set reset ep_delay for CLIENT PROD pipe
- * Clocks, should be voted before calling this API
- * locks should be taken before calling this API
- */
-
-int ipa3_set_reset_client_prod_pipe_delay(bool set_reset,
-		enum ipa_client_type client)
-{
-	int result = 0;
-	int pipe_idx;
-	struct ipa3_ep_context *ep;
-	struct ipa_ep_cfg_ctrl ep_ctrl;
-
-	memset(&ep_ctrl, 0, sizeof(struct ipa_ep_cfg_ctrl));
-	ep_ctrl.ipa_ep_delay = set_reset;
-
-	if (IPA_CLIENT_IS_CONS(client)) {
-		IPAERR("client (%d) not PROD\n", client);
-		return -EINVAL;
-	}
-
-	pipe_idx = ipa3_get_ep_mapping(client);
-
-	if (pipe_idx == IPA_EP_NOT_ALLOCATED) {
-		IPAERR("client (%d) not valid\n", client);
-		return -EINVAL;
-	}
-
-	ep = &ipa3_ctx->ep[pipe_idx];
-
-	/* Setting delay on USB_PROD with skip_ep_cfg */
-	client_lock_unlock_cb(client, true);
-	if (ep->valid && ep->skip_ep_cfg) {
-		ep->ep_delay_set = ep_ctrl.ipa_ep_delay;
-		result = ipa3_cfg_ep_ctrl(pipe_idx, &ep_ctrl);
-		if (result)
-			IPAERR("client (ep: %d) failed result=%d\n",
-				pipe_idx, result);
-		else
-			IPADBG("client (ep: %d) success\n", pipe_idx);
-	}
-	client_lock_unlock_cb(client, false);
-	return result;
-}
-
-static bool ipa3_get_teth_port_status(enum ipa_client_type client)
-{
-	enum ipa_client_cb_type client_cb;
-
-	client_cb = ipa_get_client_cb_type(client);
-	if (client_cb == IPA_MAX_CLNT)
-		return false;
-	if (ipa3_ctx->get_teth_port_state[client_cb])
-		return ipa3_ctx->get_teth_port_state[client_cb]();
-	return false;
-}
-
-/*
- * Start/stop the CLIENT PROD pipes in SSR scenarios
- */
-
-int ipa3_start_stop_client_prod_gsi_chnl(enum ipa_client_type client,
-		bool start_chnl)
-{
-	int result = 0;
-	int pipe_idx;
-	struct ipa3_ep_context *ep;
-
-	if (IPA_CLIENT_IS_CONS(client)) {
-		IPAERR("client (%d) not PROD\n", client);
-		return -EINVAL;
-	}
-
-	pipe_idx = ipa3_get_ep_mapping(client);
-
-	if (pipe_idx == IPA_EP_NOT_ALLOCATED) {
-		IPAERR("client (%d) not valid\n", client);
-		return -EINVAL;
-	}
-
-	client_lock_unlock_cb(client, true);
-	ep = &ipa3_ctx->ep[pipe_idx];
-	if (ep->valid && ep->skip_ep_cfg && ipa3_get_teth_port_status(client)) {
-		if (start_chnl)
-			result = ipa3_start_gsi_channel(pipe_idx);
-		else
-			result = ipa3_stop_gsi_channel(pipe_idx);
-	}
-	client_lock_unlock_cb(client, false);
-	return result;
-}
-int ipa3_set_reset_client_cons_pipe_sus_holb(bool set_reset,
-		enum ipa_client_type client)
-{
-	int pipe_idx;
-	struct ipa3_ep_context *ep;
-	struct ipa_ep_cfg_ctrl ep_suspend;
-	struct ipa_ep_cfg_holb ep_holb;
-
-	memset(&ep_suspend, 0, sizeof(ep_suspend));
-	memset(&ep_holb, 0, sizeof(ep_holb));
-
-	ep_suspend.ipa_ep_suspend = set_reset;
-	ep_holb.tmr_val = 0;
-	ep_holb.en = set_reset;
-
-	if (IPA_CLIENT_IS_PROD(client)) {
-		IPAERR("client (%d) not CONS\n", client);
-		return -EINVAL;
-	}
-
-	pipe_idx = ipa3_get_ep_mapping(client);
-
-	if (pipe_idx == IPA_EP_NOT_ALLOCATED) {
-		IPAERR("client (%d) not valid\n", client);
-		return -EINVAL;
-	}
-
-	ep = &ipa3_ctx->ep[pipe_idx];
-	/* Setting sus/holb on MHI_CONS with skip_ep_cfg */
-	client_lock_unlock_cb(client, true);
-	if (ep->valid && ep->skip_ep_cfg) {
-		if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_0)
-			ipahal_write_reg_n_fields(
-					IPA_ENDP_INIT_CTRL_n,
-					pipe_idx, &ep_suspend);
-		/*
-		 * ipa3_cfg_ep_holb is not used here because we are
-		 * setting HOLB on Q6 pipes, and from APPS perspective
-		 * they are not valid, therefore, the above function
-		 * will fail.
-		 */
-		ipahal_write_reg_n_fields(
-			IPA_ENDP_INIT_HOL_BLOCK_TIMER_n,
-			pipe_idx, &ep_holb);
-		ipahal_write_reg_n_fields(
-			IPA_ENDP_INIT_HOL_BLOCK_EN_n,
-			pipe_idx, &ep_holb);
-
-		/* IPA4.5 issue requires HOLB_EN to be written twice */
-		if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_5)
-			ipahal_write_reg_n_fields(
-				IPA_ENDP_INIT_HOL_BLOCK_EN_n,
-				pipe_idx, &ep_holb);
-	}
-	client_lock_unlock_cb(client, false);
-	return 0;
-}
-
-void ipa3_xdci_ep_delay_rm(u32 clnt_hdl)
-{
-	struct ipa3_ep_context *ep;
-	struct ipa_ep_cfg_ctrl ep_cfg_ctrl;
-	int result;
-
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-		ipa3_ctx->ep[clnt_hdl].valid == 0) {
-		IPAERR("bad parm.\n");
-		return;
-	}
-
-	ep = &ipa3_ctx->ep[clnt_hdl];
-
-	if (ep->ep_delay_set) {
-
-		memset(&ep_cfg_ctrl, 0, sizeof(struct ipa_ep_cfg_ctrl));
-		ep_cfg_ctrl.ipa_ep_delay = false;
-
-		if (!ep->keep_ipa_awake)
-			IPA_ACTIVE_CLIENTS_INC_EP
-				(ipa3_get_client_mapping(clnt_hdl));
-
-		result = ipa3_cfg_ep_ctrl(clnt_hdl,
-			&ep_cfg_ctrl);
-
-		if (!ep->keep_ipa_awake)
-			IPA_ACTIVE_CLIENTS_DEC_EP
-				(ipa3_get_client_mapping(clnt_hdl));
-
-		if (result) {
-			IPAERR
-			("client (ep: %d) failed to remove delay result=%d\n",
-				clnt_hdl, result);
-		} else {
-			IPADBG("client (ep: %d) delay removed\n",
-				clnt_hdl);
-			ep->ep_delay_set = false;
-		}
-	}
-}
-
-int ipa3_xdci_disconnect(u32 clnt_hdl, bool should_force_clear, u32 qmi_req_id)
-{
-	struct ipa3_ep_context *ep;
-	int result;
-	u32 source_pipe_bitmask = 0;
-
-	IPADBG("entry\n");
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-		ipa3_ctx->ep[clnt_hdl].valid == 0) {
-		IPAERR("Bad parameter.\n");
-		return -EINVAL;
-	}
-
-	ep = &ipa3_ctx->ep[clnt_hdl];
-
-	if (!ep->keep_ipa_awake)
-		IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	ipa3_disable_data_path(clnt_hdl);
-
-	if (!IPA_CLIENT_IS_CONS(ep->client)) {
-		IPADBG("Stopping PROD channel - hdl=%d clnt=%d\n",
-			clnt_hdl, ep->client);
-		source_pipe_bitmask = 1 <<
-			ipa3_get_ep_mapping(ep->client);
-		result = ipa3_stop_ul_chan_with_data_drain(qmi_req_id,
-			source_pipe_bitmask, should_force_clear, clnt_hdl,
-			true);
-		if (result) {
-			IPAERR("Fail to stop UL channel with data drain\n");
-			WARN_ON(1);
-			goto stop_chan_fail;
-		}
-	} else {
-		IPADBG("Stopping CONS channel - hdl=%d clnt=%d\n",
-			clnt_hdl, ep->client);
-		result = ipa3_stop_gsi_channel(clnt_hdl);
-		if (result) {
-			IPAERR("Error stopping channel (CONS client): %d\n",
-				result);
-			goto stop_chan_fail;
-		}
-	}
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	IPADBG("exit\n");
-	return 0;
-
-stop_chan_fail:
-	if (!ep->keep_ipa_awake)
-		IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-	return result;
-}
-
-int ipa3_release_gsi_channel(u32 clnt_hdl)
-{
-	struct ipa3_ep_context *ep;
-	int result = -EFAULT;
-	enum gsi_status gsi_res;
-
-	IPADBG("entry\n");
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-		ipa3_ctx->ep[clnt_hdl].valid == 0) {
-		IPAERR("Bad parameter.\n");
-		return -EINVAL;
-	}
-
-	ep = &ipa3_ctx->ep[clnt_hdl];
-
-	if (!ep->keep_ipa_awake)
-		IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	gsi_res = gsi_dealloc_channel(ep->gsi_chan_hdl);
-	if (gsi_res != GSI_STATUS_SUCCESS) {
-		IPAERR("Error deallocating channel: %d\n", gsi_res);
-		goto dealloc_chan_fail;
-	}
-
-	gsi_res = gsi_dealloc_evt_ring(ep->gsi_evt_ring_hdl);
-	if (gsi_res != GSI_STATUS_SUCCESS) {
-		IPAERR("Error deallocating event: %d\n", gsi_res);
-		goto dealloc_chan_fail;
-	}
-
-	if (!ep->skip_ep_cfg && IPA_CLIENT_IS_PROD(ep->client))
-		ipa3_delete_dflt_flt_rules(clnt_hdl);
-
-	if (!ep->keep_ipa_awake)
-		IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	memset(&ipa3_ctx->ep[clnt_hdl], 0, sizeof(struct ipa3_ep_context));
-
-	IPADBG("exit\n");
-	return 0;
-
-dealloc_chan_fail:
-	if (!ep->keep_ipa_awake)
-		IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-	return result;
-}
-
-int ipa3_xdci_suspend(u32 ul_clnt_hdl, u32 dl_clnt_hdl,
-	bool should_force_clear, u32 qmi_req_id, bool is_dpl)
-{
-	struct ipa3_ep_context *ul_ep = NULL;
-	struct ipa3_ep_context *dl_ep;
-	int result = -EFAULT;
-	u32 source_pipe_bitmask = 0;
-	bool dl_data_pending = true;
-	bool ul_data_pending = true;
-	int i;
-	bool is_empty = false;
-	struct gsi_chan_info ul_gsi_chan_info, dl_gsi_chan_info;
-	int aggr_active_bitmap = 0;
-	struct ipa_ep_cfg_ctrl ep_cfg_ctrl;
-
-	/* In case of DPL, dl is the DPL channel/client */
-
-	IPADBG("entry\n");
-	if (dl_clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-		ipa3_ctx->ep[dl_clnt_hdl].valid == 0 ||
-		(!is_dpl && (ul_clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-		ipa3_ctx->ep[ul_clnt_hdl].valid == 0))) {
-		IPAERR("Bad parameter.\n");
-		return -EINVAL;
-	}
-
-	dl_ep = &ipa3_ctx->ep[dl_clnt_hdl];
-	if (!is_dpl)
-		ul_ep = &ipa3_ctx->ep[ul_clnt_hdl];
-	IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(dl_clnt_hdl));
-
-	result = ipa3_get_gsi_chan_info(&dl_gsi_chan_info,
-		dl_ep->gsi_chan_hdl);
-	if (result)
-		goto disable_clk_and_exit;
-
-	if (!is_dpl) {
-		result = ipa3_get_gsi_chan_info(&ul_gsi_chan_info,
-			ul_ep->gsi_chan_hdl);
-		if (result)
-			goto disable_clk_and_exit;
-	}
-
-	for (i = 0; i < IPA_POLL_FOR_EMPTINESS_NUM; i++) {
-		if (!dl_data_pending && !ul_data_pending)
-			break;
-		result = ipa3_is_xdci_channel_empty(dl_ep, &is_empty);
-		if (result)
-			goto disable_clk_and_exit;
-		if (!is_empty) {
-			dl_data_pending = true;
-			break;
-		}
-		dl_data_pending = false;
-		if (!is_dpl) {
-			result = ipa3_is_xdci_channel_empty(ul_ep, &is_empty);
-			if (result)
-				goto disable_clk_and_exit;
-			ul_data_pending = !is_empty;
-		} else {
-			ul_data_pending = false;
-		}
-
-		udelay(IPA_POLL_FOR_EMPTINESS_SLEEP_USEC);
-	}
-
-	if (!dl_data_pending) {
-		aggr_active_bitmap = ipahal_read_reg(IPA_STATE_AGGR_ACTIVE);
-		if (aggr_active_bitmap & (1 << dl_clnt_hdl)) {
-			IPADBG("DL/DPL data pending due to open aggr. frame\n");
-			dl_data_pending = true;
-		}
-	}
-	if (dl_data_pending) {
-		IPAERR("DL/DPL data pending, can't suspend\n");
-		result = -EFAULT;
-		goto disable_clk_and_exit;
-	}
-
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_0) {
-		/* Suspend the DL/DPL EP */
-		memset(&ep_cfg_ctrl, 0, sizeof(struct ipa_ep_cfg_ctrl));
-		ep_cfg_ctrl.ipa_ep_suspend = true;
-		ipa3_cfg_ep_ctrl(dl_clnt_hdl, &ep_cfg_ctrl);
-	}
-
-	/*
-	 * Check if DL/DPL channel is empty again, data could enter the channel
-	 * before its IPA EP was suspended
-	 */
-	result = ipa3_is_xdci_channel_empty(dl_ep, &is_empty);
-	if (result)
-		goto unsuspend_dl_and_exit;
-	if (!is_empty) {
-		IPAERR("DL/DPL data pending, can't suspend\n");
-		result = -EFAULT;
-		goto unsuspend_dl_and_exit;
-	}
-
-	/* Stop DL channel */
-	result = ipa3_stop_gsi_channel(dl_clnt_hdl);
-	if (result) {
-		IPAERR("Error stopping DL/DPL channel: %d\n", result);
-		result = -EFAULT;
-		goto unsuspend_dl_and_exit;
-	}
-
-	/* STOP UL channel */
-	if (!is_dpl) {
-		source_pipe_bitmask = 1 << ipa3_get_ep_mapping(ul_ep->client);
-		result = ipa3_stop_ul_chan_with_data_drain(qmi_req_id,
-			source_pipe_bitmask, should_force_clear, ul_clnt_hdl,
-			false);
-		if (result) {
-			IPAERR("Error stopping UL channel: result = %d\n",
-				result);
-			goto start_dl_and_exit;
-		}
-	}
-
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(dl_clnt_hdl));
-
-	IPADBG("exit\n");
-	return 0;
-
-start_dl_and_exit:
-	gsi_start_channel(dl_ep->gsi_chan_hdl);
-	ipa3_start_gsi_debug_monitor(dl_clnt_hdl);
-unsuspend_dl_and_exit:
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_0) {
-		/* Unsuspend the DL EP */
-		memset(&ep_cfg_ctrl, 0, sizeof(struct ipa_ep_cfg_ctrl));
-		ep_cfg_ctrl.ipa_ep_suspend = false;
-		ipa3_cfg_ep_ctrl(dl_clnt_hdl, &ep_cfg_ctrl);
-	}
-disable_clk_and_exit:
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(dl_clnt_hdl));
-	return result;
-}
-
-int ipa3_start_gsi_channel(u32 clnt_hdl)
-{
-	struct ipa3_ep_context *ep;
-	int result = -EFAULT;
-	enum gsi_status gsi_res;
-	enum ipa_client_type client_type;
-
-	IPADBG("entry\n");
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes  ||
-		ipa3_ctx->ep[clnt_hdl].valid == 0) {
-		IPAERR("Bad parameters.\n");
-		return -EINVAL;
-	}
-
-	ep = &ipa3_ctx->ep[clnt_hdl];
-	client_type = ipa3_get_client_mapping(clnt_hdl);
-	if (!ep->keep_ipa_awake)
-		IPA_ACTIVE_CLIENTS_INC_EP(client_type);
-
-	gsi_res = gsi_start_channel(ep->gsi_chan_hdl);
-	if (gsi_res != GSI_STATUS_SUCCESS) {
-		IPAERR("Error starting channel: %d\n", gsi_res);
-		goto start_chan_fail;
-	}
-	ipa3_start_gsi_debug_monitor(clnt_hdl);
-
-	if (!ep->keep_ipa_awake)
-		IPA_ACTIVE_CLIENTS_DEC_EP(client_type);
-
-	IPADBG("exit\n");
-	return 0;
-
-start_chan_fail:
-	if (!ep->keep_ipa_awake)
-		IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-	return result;
-}
-
-int ipa3_xdci_resume(u32 ul_clnt_hdl, u32 dl_clnt_hdl, bool is_dpl)
-{
-	struct ipa3_ep_context *ul_ep = NULL;
-	struct ipa3_ep_context *dl_ep = NULL;
-	enum gsi_status gsi_res;
-	struct ipa_ep_cfg_ctrl ep_cfg_ctrl;
-
-	/* In case of DPL, dl is the DPL channel/client */
-
-	IPADBG("entry\n");
-	if (dl_clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-		ipa3_ctx->ep[dl_clnt_hdl].valid == 0 ||
-		(!is_dpl && (ul_clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-		ipa3_ctx->ep[ul_clnt_hdl].valid == 0))) {
-		IPAERR("Bad parameter.\n");
-		return -EINVAL;
-	}
-
-	dl_ep = &ipa3_ctx->ep[dl_clnt_hdl];
-	if (!is_dpl)
-		ul_ep = &ipa3_ctx->ep[ul_clnt_hdl];
-	IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(dl_clnt_hdl));
-
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_0) {
-		/* Unsuspend the DL/DPL EP */
-		memset(&ep_cfg_ctrl, 0, sizeof(struct ipa_ep_cfg_ctrl));
-		ep_cfg_ctrl.ipa_ep_suspend = false;
-		ipa3_cfg_ep_ctrl(dl_clnt_hdl, &ep_cfg_ctrl);
-	}
-
-	/* Start DL channel */
-	gsi_res = gsi_start_channel(dl_ep->gsi_chan_hdl);
-	if (gsi_res != GSI_STATUS_SUCCESS)
-		IPAERR("Error starting DL channel: %d\n", gsi_res);
-	ipa3_start_gsi_debug_monitor(dl_clnt_hdl);
-
-	/* Start UL channel */
-	if (!is_dpl) {
-		gsi_res = gsi_start_channel(ul_ep->gsi_chan_hdl);
-		if (gsi_res != GSI_STATUS_SUCCESS)
-			IPAERR("Error starting UL channel: %d\n", gsi_res);
-		ipa3_start_gsi_debug_monitor(ul_clnt_hdl);
-	}
-
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(dl_clnt_hdl));
-
-	IPADBG("exit\n");
-	return 0;
-}
-/**
- * ipa3_clear_endpoint_delay() - Remove ep delay set on the IPA pipe before
- * client disconnect.
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- *
- * Should be called by the driver of the peripheral that wants to remove
- * ep delay on IPA consumer ipe before disconnect in non GPI mode. this api
- * expects caller to take responsibility to free any needed headers, routing
- * and filtering tables and rules as needed.
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_clear_endpoint_delay(u32 clnt_hdl)
-{
-	struct ipa3_ep_context *ep;
-	struct ipa_ep_cfg_ctrl ep_ctrl = {0};
-	struct ipa_enable_force_clear_datapath_req_msg_v01 req = {0};
-	int res;
-
-	if (unlikely(!ipa3_ctx)) {
-		IPAERR("IPA driver was not initialized\n");
-		return -EINVAL;
-	}
-
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-		ipa3_ctx->ep[clnt_hdl].valid == 0) {
-		IPAERR("bad parm.\n");
-		return -EINVAL;
-	}
-
-	ep = &ipa3_ctx->ep[clnt_hdl];
-
-	if (!ipa3_ctx->tethered_flow_control) {
-		IPADBG("APPS flow control is not enabled\n");
-		/* Send a message to modem to disable flow control honoring. */
-		req.request_id = clnt_hdl;
-		req.source_pipe_bitmask = 1 << clnt_hdl;
-		res = ipa3_qmi_enable_force_clear_datapath_send(&req);
-		if (res) {
-			IPADBG("enable_force_clear_datapath failed %d\n",
-				res);
-		}
-		ep->qmi_request_sent = true;
-	}
-
-	IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-	/* Set disconnect in progress flag so further flow control events are
-	 * not honored.
-	 */
-	spin_lock(&ipa3_ctx->disconnect_lock);
-	ep->disconnect_in_progress = true;
-	spin_unlock(&ipa3_ctx->disconnect_lock);
-
-	/* If flow is disabled at this point, restore the ep state.*/
-	ep_ctrl.ipa_ep_delay = false;
-	ep_ctrl.ipa_ep_suspend = false;
-	ipa3_cfg_ep_ctrl(clnt_hdl, &ep_ctrl);
-
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	IPADBG("client (ep: %d) removed ep delay\n", clnt_hdl);
-
-	return 0;
-}

ipa/ipa_v3/ipa_debugfs.c

@@ -1,2557 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2012-2019, The Linux Foundation. All rights reserved.
- */
-
-#ifdef CONFIG_DEBUG_FS
-
-#include <linux/debugfs.h>
-#include <linux/kernel.h>
-#include <linux/stringify.h>
-#include "ipa_i.h"
-#include "../ipa_rm_i.h"
-#include "ipahal/ipahal_nat.h"
-#include "ipa_odl.h"
-#include "ipa_qmi_service.h"
-
-#define IPA_MAX_ENTRY_STRING_LEN 500
-#define IPA_MAX_MSG_LEN 4096
-#define IPA_DBG_MAX_RULE_IN_TBL 128
-#define IPA_DBG_ACTIVE_CLIENT_BUF_SIZE ((IPA3_ACTIVE_CLIENTS_LOG_LINE_LEN \
-	* IPA3_ACTIVE_CLIENTS_LOG_BUFFER_SIZE_LINES) + IPA_MAX_MSG_LEN)
-
-#define IPA_DUMP_STATUS_FIELD(f) \
-	pr_err(#f "=0x%x\n", status->f)
-
-#define IPA_READ_ONLY_MODE  0444
-#define IPA_READ_WRITE_MODE 0664
-#define IPA_WRITE_ONLY_MODE 0220
-
-struct ipa3_debugfs_file {
-	const char *name;
-	umode_t mode;
-	void *data;
-	const struct file_operations fops;
-};
-
-
-const char *ipa3_event_name[IPA_EVENT_MAX_NUM] = {
-	__stringify(WLAN_CLIENT_CONNECT),
-	__stringify(WLAN_CLIENT_DISCONNECT),
-	__stringify(WLAN_CLIENT_POWER_SAVE_MODE),
-	__stringify(WLAN_CLIENT_NORMAL_MODE),
-	__stringify(SW_ROUTING_ENABLE),
-	__stringify(SW_ROUTING_DISABLE),
-	__stringify(WLAN_AP_CONNECT),
-	__stringify(WLAN_AP_DISCONNECT),
-	__stringify(WLAN_STA_CONNECT),
-	__stringify(WLAN_STA_DISCONNECT),
-	__stringify(WLAN_CLIENT_CONNECT_EX),
-	__stringify(WLAN_SWITCH_TO_SCC),
-	__stringify(WLAN_SWITCH_TO_MCC),
-	__stringify(WLAN_WDI_ENABLE),
-	__stringify(WLAN_WDI_DISABLE),
-	__stringify(WAN_UPSTREAM_ROUTE_ADD),
-	__stringify(WAN_UPSTREAM_ROUTE_DEL),
-	__stringify(WAN_EMBMS_CONNECT),
-	__stringify(WAN_XLAT_CONNECT),
-	__stringify(ECM_CONNECT),
-	__stringify(ECM_DISCONNECT),
-	__stringify(IPA_TETHERING_STATS_UPDATE_STATS),
-	__stringify(IPA_TETHERING_STATS_UPDATE_NETWORK_STATS),
-	__stringify(IPA_QUOTA_REACH),
-	__stringify(IPA_SSR_BEFORE_SHUTDOWN),
-	__stringify(IPA_SSR_AFTER_POWERUP),
-	__stringify(ADD_VLAN_IFACE),
-	__stringify(DEL_VLAN_IFACE),
-	__stringify(ADD_L2TP_VLAN_MAPPING),
-	__stringify(DEL_L2TP_VLAN_MAPPING),
-	__stringify(IPA_PER_CLIENT_STATS_CONNECT_EVENT),
-	__stringify(IPA_PER_CLIENT_STATS_DISCONNECT_EVENT),
-	__stringify(ADD_BRIDGE_VLAN_MAPPING),
-	__stringify(DEL_BRIDGE_VLAN_MAPPING),
-	__stringify(WLAN_FWR_SSR_BEFORE_SHUTDOWN),
-	__stringify(IPA_GSB_CONNECT),
-	__stringify(IPA_GSB_DISCONNECT),
-	__stringify(IPA_COALESCE_ENABLE),
-	__stringify(IPA_COALESCE_DISABLE),
-	__stringify_1(WIGIG_CLIENT_CONNECT),
-	__stringify_1(WIGIG_FST_SWITCH),
-};
-
-const char *ipa3_hdr_l2_type_name[] = {
-	__stringify(IPA_HDR_L2_NONE),
-	__stringify(IPA_HDR_L2_ETHERNET_II),
-	__stringify(IPA_HDR_L2_802_3),
-};
-
-const char *ipa3_hdr_proc_type_name[] = {
-	__stringify(IPA_HDR_PROC_NONE),
-	__stringify(IPA_HDR_PROC_ETHII_TO_ETHII),
-	__stringify(IPA_HDR_PROC_ETHII_TO_802_3),
-	__stringify(IPA_HDR_PROC_802_3_TO_ETHII),
-	__stringify(IPA_HDR_PROC_802_3_TO_802_3),
-	__stringify(IPA_HDR_PROC_L2TP_HEADER_ADD),
-	__stringify(IPA_HDR_PROC_L2TP_HEADER_REMOVE),
-};
-
-static struct dentry *dent;
-static char dbg_buff[IPA_MAX_MSG_LEN];
-static char *active_clients_buf;
-
-static s8 ep_reg_idx;
-static void *ipa_ipc_low_buff;
-
-
-static ssize_t ipa3_read_gen_reg(struct file *file, char __user *ubuf,
-		size_t count, loff_t *ppos)
-{
-	int nbytes;
-	struct ipahal_reg_shared_mem_size smem_sz;
-
-	memset(&smem_sz, 0, sizeof(smem_sz));
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	ipahal_read_reg_fields(IPA_SHARED_MEM_SIZE, &smem_sz);
-	nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
-			"IPA_VERSION=0x%x\n"
-			"IPA_COMP_HW_VERSION=0x%x\n"
-			"IPA_ROUTE=0x%x\n"
-			"IPA_SHARED_MEM_RESTRICTED=0x%x\n"
-			"IPA_SHARED_MEM_SIZE=0x%x\n"
-			"IPA_QTIME_TIMESTAMP_CFG=0x%x\n"
-			"IPA_TIMERS_PULSE_GRAN_CFG=0x%x\n"
-			"IPA_TIMERS_XO_CLK_DIV_CFG=0x%x\n",
-			ipahal_read_reg(IPA_VERSION),
-			ipahal_read_reg(IPA_COMP_HW_VERSION),
-			ipahal_read_reg(IPA_ROUTE),
-			smem_sz.shared_mem_baddr,
-			smem_sz.shared_mem_sz,
-			ipahal_read_reg(IPA_QTIME_TIMESTAMP_CFG),
-			ipahal_read_reg(IPA_TIMERS_PULSE_GRAN_CFG),
-			ipahal_read_reg(IPA_TIMERS_XO_CLK_DIV_CFG));
-
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-
-	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, nbytes);
-}
-
-static ssize_t ipa3_write_ep_holb(struct file *file,
-		const char __user *buf, size_t count, loff_t *ppos)
-{
-	struct ipa_ep_cfg_holb holb;
-	u32 en;
-	u32 tmr_val;
-	u32 ep_idx;
-	unsigned long missing;
-	char *sptr, *token;
-
-	if (count >= sizeof(dbg_buff))
-		return -EFAULT;
-
-	missing = copy_from_user(dbg_buff, buf, count);
-	if (missing)
-		return -EFAULT;
-
-	dbg_buff[count] = '\0';
-
-	sptr = dbg_buff;
-
-	token = strsep(&sptr, " ");
-	if (!token)
-		return -EINVAL;
-	if (kstrtou32(token, 0, &ep_idx))
-		return -EINVAL;
-
-	token = strsep(&sptr, " ");
-	if (!token)
-		return -EINVAL;
-	if (kstrtou32(token, 0, &en))
-		return -EINVAL;
-
-	token = strsep(&sptr, " ");
-	if (!token)
-		return -EINVAL;
-	if (kstrtou32(token, 0, &tmr_val))
-		return -EINVAL;
-
-	holb.en = en;
-	holb.tmr_val = tmr_val;
-
-	ipa3_cfg_ep_holb(ep_idx, &holb);
-
-	return count;
-}
-
-static ssize_t ipa3_write_ep_reg(struct file *file, const char __user *buf,
-		size_t count, loff_t *ppos)
-{
-	s8 option;
-	int ret;
-
-	ret = kstrtos8_from_user(buf, count, 0, &option);
-	if (ret)
-		return ret;
-
-	if (option >= ipa3_ctx->ipa_num_pipes) {
-		IPAERR("bad pipe specified %u\n", option);
-		return count;
-	}
-
-	ep_reg_idx = option;
-
-	return count;
-}
-
-/**
- * _ipa_read_ep_reg_v3_0() - Reads and prints endpoint configuration registers
- *
- * Returns the number of characters printed
- */
-int _ipa_read_ep_reg_v3_0(char *buf, int max_len, int pipe)
-{
-	return scnprintf(
-		dbg_buff, IPA_MAX_MSG_LEN,
-		"IPA_ENDP_INIT_NAT_%u=0x%x\n"
-		"IPA_ENDP_INIT_HDR_%u=0x%x\n"
-		"IPA_ENDP_INIT_HDR_EXT_%u=0x%x\n"
-		"IPA_ENDP_INIT_MODE_%u=0x%x\n"
-		"IPA_ENDP_INIT_AGGR_%u=0x%x\n"
-		"IPA_ENDP_INIT_ROUTE_%u=0x%x\n"
-		"IPA_ENDP_INIT_CTRL_%u=0x%x\n"
-		"IPA_ENDP_INIT_HOL_EN_%u=0x%x\n"
-		"IPA_ENDP_INIT_HOL_TIMER_%u=0x%x\n"
-		"IPA_ENDP_INIT_DEAGGR_%u=0x%x\n"
-		"IPA_ENDP_INIT_CFG_%u=0x%x\n",
-		pipe, ipahal_read_reg_n(IPA_ENDP_INIT_NAT_n, pipe),
-		pipe, ipahal_read_reg_n(IPA_ENDP_INIT_HDR_n, pipe),
-		pipe, ipahal_read_reg_n(IPA_ENDP_INIT_HDR_EXT_n, pipe),
-		pipe, ipahal_read_reg_n(IPA_ENDP_INIT_MODE_n, pipe),
-		pipe, ipahal_read_reg_n(IPA_ENDP_INIT_AGGR_n, pipe),
-		pipe, ipahal_read_reg_n(IPA_ENDP_INIT_ROUTE_n, pipe),
-		pipe, ipahal_read_reg_n(IPA_ENDP_INIT_CTRL_n, pipe),
-		pipe, ipahal_read_reg_n(IPA_ENDP_INIT_HOL_BLOCK_EN_n, pipe),
-		pipe, ipahal_read_reg_n(IPA_ENDP_INIT_HOL_BLOCK_TIMER_n, pipe),
-		pipe, ipahal_read_reg_n(IPA_ENDP_INIT_DEAGGR_n, pipe),
-		pipe, ipahal_read_reg_n(IPA_ENDP_INIT_CFG_n, pipe));
-}
-
-/**
- * _ipa_read_ep_reg_v4_0() - Reads and prints endpoint configuration registers
- *
- * Returns the number of characters printed
- * Removed IPA_ENDP_INIT_ROUTE_n from v3
- */
-int _ipa_read_ep_reg_v4_0(char *buf, int max_len, int pipe)
-{
-	return scnprintf(
-		dbg_buff, IPA_MAX_MSG_LEN,
-		"IPA_ENDP_INIT_NAT_%u=0x%x\n"
-		"IPA_ENDP_INIT_CONN_TRACK_n%u=0x%x\n"
-		"IPA_ENDP_INIT_HDR_%u=0x%x\n"
-		"IPA_ENDP_INIT_HDR_EXT_%u=0x%x\n"
-		"IPA_ENDP_INIT_MODE_%u=0x%x\n"
-		"IPA_ENDP_INIT_AGGR_%u=0x%x\n"
-		"IPA_ENDP_INIT_CTRL_%u=0x%x\n"
-		"IPA_ENDP_INIT_HOL_EN_%u=0x%x\n"
-		"IPA_ENDP_INIT_HOL_TIMER_%u=0x%x\n"
-		"IPA_ENDP_INIT_DEAGGR_%u=0x%x\n"
-		"IPA_ENDP_INIT_CFG_%u=0x%x\n",
-		pipe, ipahal_read_reg_n(IPA_ENDP_INIT_NAT_n, pipe),
-		pipe, ipahal_read_reg_n(IPA_ENDP_INIT_CONN_TRACK_n, pipe),
-		pipe, ipahal_read_reg_n(IPA_ENDP_INIT_HDR_n, pipe),
-		pipe, ipahal_read_reg_n(IPA_ENDP_INIT_HDR_EXT_n, pipe),
-		pipe, ipahal_read_reg_n(IPA_ENDP_INIT_MODE_n, pipe),
-		pipe, ipahal_read_reg_n(IPA_ENDP_INIT_AGGR_n, pipe),
-		pipe, ipahal_read_reg_n(IPA_ENDP_INIT_CTRL_n, pipe),
-		pipe, ipahal_read_reg_n(IPA_ENDP_INIT_HOL_BLOCK_EN_n, pipe),
-		pipe, ipahal_read_reg_n(IPA_ENDP_INIT_HOL_BLOCK_TIMER_n, pipe),
-		pipe, ipahal_read_reg_n(IPA_ENDP_INIT_DEAGGR_n, pipe),
-		pipe, ipahal_read_reg_n(IPA_ENDP_INIT_CFG_n, pipe));
-}
-
-static ssize_t ipa3_read_ep_reg(struct file *file, char __user *ubuf,
-		size_t count, loff_t *ppos)
-{
-	int nbytes;
-	int i;
-	int start_idx;
-	int end_idx;
-	int size = 0;
-	int ret;
-	loff_t pos;
-
-	/* negative ep_reg_idx means all registers */
-	if (ep_reg_idx < 0) {
-		start_idx = 0;
-		end_idx = ipa3_ctx->ipa_num_pipes;
-	} else {
-		start_idx = ep_reg_idx;
-		end_idx = start_idx + 1;
-	}
-	pos = *ppos;
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-	for (i = start_idx; i < end_idx; i++) {
-
-		nbytes = ipa3_ctx->ctrl->ipa3_read_ep_reg(dbg_buff,
-				IPA_MAX_MSG_LEN, i);
-
-		*ppos = pos;
-		ret = simple_read_from_buffer(ubuf, count, ppos, dbg_buff,
-					      nbytes);
-		if (ret < 0) {
-			IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-			return ret;
-		}
-
-		size += ret;
-		ubuf += nbytes;
-		count -= nbytes;
-	}
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-
-	*ppos = pos + size;
-	return size;
-}
-
-static ssize_t ipa3_write_keep_awake(struct file *file, const char __user *buf,
-	size_t count, loff_t *ppos)
-{
-	s8 option = 0;
-	int ret;
-	uint32_t bw_mbps = 0;
-
-	ret = kstrtos8_from_user(buf, count, 0, &option);
-	if (ret)
-		return ret;
-
-	switch (option) {
-	case 0:
-		IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-		bw_mbps = 0;
-		break;
-	case 1:
-		IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-		bw_mbps = 0;
-		break;
-	case 2:
-		IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-		bw_mbps = 700;
-		break;
-	case 3:
-		IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-		bw_mbps = 3000;
-		break;
-	case 4:
-		IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-		bw_mbps = 7000;
-		break;
-	default:
-		pr_err("Not support this vote (%d)\n", option);
-		return -EFAULT;
-	}
-	if (ipa3_vote_for_bus_bw(&bw_mbps)) {
-		IPAERR("Failed to vote for bus BW (%u)\n", bw_mbps);
-		return -EFAULT;
-	}
-
-	return count;
-}
-
-static ssize_t ipa3_read_keep_awake(struct file *file, char __user *ubuf,
-	size_t count, loff_t *ppos)
-{
-	int nbytes;
-
-	mutex_lock(&ipa3_ctx->ipa3_active_clients.mutex);
-	if (atomic_read(&ipa3_ctx->ipa3_active_clients.cnt))
-		nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
-				"IPA APPS power state is ON\n");
-	else
-		nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
-				"IPA APPS power state is OFF\n");
-	mutex_unlock(&ipa3_ctx->ipa3_active_clients.mutex);
-
-	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, nbytes);
-}
-
-static ssize_t ipa3_read_hdr(struct file *file, char __user *ubuf, size_t count,
-		loff_t *ppos)
-{
-	int nbytes = 0;
-	int i = 0;
-	struct ipa3_hdr_entry *entry;
-
-	mutex_lock(&ipa3_ctx->lock);
-
-	if (ipa3_ctx->hdr_tbl_lcl)
-		pr_err("Table resides on local memory\n");
-	else
-		pr_err("Table resides on system (ddr) memory\n");
-
-	list_for_each_entry(entry, &ipa3_ctx->hdr_tbl.head_hdr_entry_list,
-			link) {
-		if (entry->cookie != IPA_HDR_COOKIE)
-			continue;
-		nbytes = scnprintf(
-			dbg_buff,
-			IPA_MAX_MSG_LEN,
-			"name:%s len=%d ref=%d partial=%d type=%s ",
-			entry->name,
-			entry->hdr_len,
-			entry->ref_cnt,
-			entry->is_partial,
-			ipa3_hdr_l2_type_name[entry->type]);
-
-		if (entry->is_hdr_proc_ctx) {
-			nbytes += scnprintf(
-				dbg_buff + nbytes,
-				IPA_MAX_MSG_LEN - nbytes,
-				"phys_base=0x%pa ",
-				&entry->phys_base);
-		} else {
-			nbytes += scnprintf(
-				dbg_buff + nbytes,
-				IPA_MAX_MSG_LEN - nbytes,
-				"ofst=%u ",
-				entry->offset_entry->offset >> 2);
-		}
-		for (i = 0; i < entry->hdr_len; i++) {
-			scnprintf(dbg_buff + nbytes + i * 2,
-				  IPA_MAX_MSG_LEN - nbytes - i * 2,
-				  "%02x", entry->hdr[i]);
-		}
-		scnprintf(dbg_buff + nbytes + entry->hdr_len * 2,
-			  IPA_MAX_MSG_LEN - nbytes - entry->hdr_len * 2,
-			  "\n");
-		pr_err("%s", dbg_buff);
-	}
-	mutex_unlock(&ipa3_ctx->lock);
-
-	return 0;
-}
-
-static int ipa3_attrib_dump(struct ipa_rule_attrib *attrib,
-		enum ipa_ip_type ip)
-{
-	uint32_t addr[4];
-	uint32_t mask[4];
-	int i;
-
-	if (attrib->attrib_mask & IPA_FLT_IS_PURE_ACK)
-		pr_err("is_pure_ack ");
-
-	if (attrib->attrib_mask & IPA_FLT_TOS)
-		pr_err("tos:%d ", attrib->u.v4.tos);
-
-	if (attrib->attrib_mask & IPA_FLT_TOS_MASKED) {
-		pr_err("tos_value:%d ", attrib->tos_value);
-		pr_err("tos_mask:%d ", attrib->tos_mask);
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_PROTOCOL)
-		pr_err("protocol:%d ", attrib->u.v4.protocol);
-
-	if (attrib->attrib_mask & IPA_FLT_SRC_ADDR) {
-		if (ip == IPA_IP_v4) {
-			addr[0] = htonl(attrib->u.v4.src_addr);
-			mask[0] = htonl(attrib->u.v4.src_addr_mask);
-			pr_err(
-					"src_addr:%pI4 src_addr_mask:%pI4 ",
-					addr + 0, mask + 0);
-		} else if (ip == IPA_IP_v6) {
-			for (i = 0; i < 4; i++) {
-				addr[i] = htonl(attrib->u.v6.src_addr[i]);
-				mask[i] = htonl(attrib->u.v6.src_addr_mask[i]);
-			}
-			pr_err(
-					   "src_addr:%pI6 src_addr_mask:%pI6 ",
-					   addr + 0, mask + 0);
-		}
-	}
-	if (attrib->attrib_mask & IPA_FLT_DST_ADDR) {
-		if (ip == IPA_IP_v4) {
-			addr[0] = htonl(attrib->u.v4.dst_addr);
-			mask[0] = htonl(attrib->u.v4.dst_addr_mask);
-			pr_err(
-					   "dst_addr:%pI4 dst_addr_mask:%pI4 ",
-					   addr + 0, mask + 0);
-		} else if (ip == IPA_IP_v6) {
-			for (i = 0; i < 4; i++) {
-				addr[i] = htonl(attrib->u.v6.dst_addr[i]);
-				mask[i] = htonl(attrib->u.v6.dst_addr_mask[i]);
-			}
-			pr_err(
-					   "dst_addr:%pI6 dst_addr_mask:%pI6 ",
-					   addr + 0, mask + 0);
-		}
-	}
-	if (attrib->attrib_mask & IPA_FLT_SRC_PORT_RANGE) {
-		pr_err("src_port_range:%u %u ",
-				   attrib->src_port_lo,
-			     attrib->src_port_hi);
-	}
-	if (attrib->attrib_mask & IPA_FLT_DST_PORT_RANGE) {
-		pr_err("dst_port_range:%u %u ",
-				   attrib->dst_port_lo,
-			     attrib->dst_port_hi);
-	}
-	if (attrib->attrib_mask & IPA_FLT_TYPE)
-		pr_err("type:%d ", attrib->type);
-
-	if (attrib->attrib_mask & IPA_FLT_CODE)
-		pr_err("code:%d ", attrib->code);
-
-	if (attrib->attrib_mask & IPA_FLT_SPI)
-		pr_err("spi:%x ", attrib->spi);
-
-	if (attrib->attrib_mask & IPA_FLT_SRC_PORT)
-		pr_err("src_port:%u ", attrib->src_port);
-
-	if (attrib->attrib_mask & IPA_FLT_DST_PORT)
-		pr_err("dst_port:%u ", attrib->dst_port);
-
-	if (attrib->attrib_mask & IPA_FLT_TC)
-		pr_err("tc:%d ", attrib->u.v6.tc);
-
-	if (attrib->attrib_mask & IPA_FLT_FLOW_LABEL)
-		pr_err("flow_label:%x ", attrib->u.v6.flow_label);
-
-	if (attrib->attrib_mask & IPA_FLT_NEXT_HDR)
-		pr_err("next_hdr:%d ", attrib->u.v6.next_hdr);
-
-	if (attrib->attrib_mask & IPA_FLT_META_DATA) {
-		pr_err(
-				   "metadata:%x metadata_mask:%x ",
-				   attrib->meta_data, attrib->meta_data_mask);
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_FRAGMENT)
-		pr_err("frg ");
-
-	if ((attrib->attrib_mask & IPA_FLT_MAC_SRC_ADDR_ETHER_II) ||
-		(attrib->attrib_mask & IPA_FLT_MAC_SRC_ADDR_802_3)) {
-		pr_err("src_mac_addr:%pM ", attrib->src_mac_addr);
-	}
-
-	if ((attrib->attrib_mask & IPA_FLT_MAC_DST_ADDR_ETHER_II) ||
-		(attrib->attrib_mask & IPA_FLT_MAC_DST_ADDR_802_3) ||
-		(attrib->attrib_mask & IPA_FLT_MAC_DST_ADDR_L2TP)) {
-		pr_err("dst_mac_addr:%pM ", attrib->dst_mac_addr);
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_MAC_ETHER_TYPE)
-		pr_err("ether_type:%x ", attrib->ether_type);
-
-	if (attrib->attrib_mask & IPA_FLT_TCP_SYN)
-		pr_err("tcp syn ");
-
-	if (attrib->attrib_mask & IPA_FLT_TCP_SYN_L2TP)
-		pr_err("tcp syn l2tp ");
-
-	if (attrib->attrib_mask & IPA_FLT_L2TP_INNER_IP_TYPE)
-		pr_err("l2tp inner ip type: %d ", attrib->type);
-
-	if (attrib->attrib_mask & IPA_FLT_L2TP_INNER_IPV4_DST_ADDR) {
-		addr[0] = htonl(attrib->u.v4.dst_addr);
-		mask[0] = htonl(attrib->u.v4.dst_addr_mask);
-		pr_err("dst_addr:%pI4 dst_addr_mask:%pI4 ", addr, mask);
-	}
-
-	pr_err("\n");
-	return 0;
-}
-
-static int ipa3_attrib_dump_eq(struct ipa_ipfltri_rule_eq *attrib)
-{
-	uint8_t addr[16];
-	uint8_t mask[16];
-	int i;
-	int j;
-
-	if (attrib->tos_eq_present) {
-		if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_5)
-			pr_err("pure_ack ");
-		else
-			pr_err("tos:%d ", attrib->tos_eq);
-	}
-
-	if (attrib->protocol_eq_present)
-		pr_err("protocol:%d ", attrib->protocol_eq);
-
-	if (attrib->tc_eq_present)
-		pr_err("tc:%d ", attrib->tc_eq);
-
-	if (attrib->num_offset_meq_128 > IPA_IPFLTR_NUM_MEQ_128_EQNS) {
-		IPAERR_RL("num_offset_meq_128  Max %d passed value %d\n",
-		IPA_IPFLTR_NUM_MEQ_128_EQNS, attrib->num_offset_meq_128);
-		return -EPERM;
-	}
-
-	for (i = 0; i < attrib->num_offset_meq_128; i++) {
-		for (j = 0; j < 16; j++) {
-			addr[j] = attrib->offset_meq_128[i].value[j];
-			mask[j] = attrib->offset_meq_128[i].mask[j];
-		}
-		pr_err(
-			"(ofst_meq128: ofst:%d mask:%pI6 val:%pI6) ",
-			attrib->offset_meq_128[i].offset,
-			mask, addr);
-	}
-
-	if (attrib->num_offset_meq_32 > IPA_IPFLTR_NUM_MEQ_32_EQNS) {
-		IPAERR_RL("num_offset_meq_32  Max %d passed value %d\n",
-		IPA_IPFLTR_NUM_MEQ_32_EQNS, attrib->num_offset_meq_32);
-		return -EPERM;
-	}
-
-	for (i = 0; i < attrib->num_offset_meq_32; i++)
-		pr_err(
-			   "(ofst_meq32: ofst:%u mask:0x%x val:0x%x) ",
-			   attrib->offset_meq_32[i].offset,
-			   attrib->offset_meq_32[i].mask,
-			   attrib->offset_meq_32[i].value);
-
-	if (attrib->num_ihl_offset_meq_32 > IPA_IPFLTR_NUM_IHL_MEQ_32_EQNS) {
-		IPAERR_RL("num_ihl_offset_meq_32  Max %d passed value %d\n",
-		IPA_IPFLTR_NUM_IHL_MEQ_32_EQNS, attrib->num_ihl_offset_meq_32);
-		return -EPERM;
-	}
-
-	for (i = 0; i < attrib->num_ihl_offset_meq_32; i++)
-		pr_err(
-			"(ihl_ofst_meq32: ofts:%d mask:0x%x val:0x%x) ",
-			attrib->ihl_offset_meq_32[i].offset,
-			attrib->ihl_offset_meq_32[i].mask,
-			attrib->ihl_offset_meq_32[i].value);
-
-	if (attrib->metadata_meq32_present)
-		pr_err(
-			"(metadata: ofst:%u mask:0x%x val:0x%x) ",
-			attrib->metadata_meq32.offset,
-			attrib->metadata_meq32.mask,
-			attrib->metadata_meq32.value);
-
-	if (attrib->num_ihl_offset_range_16 >
-			IPA_IPFLTR_NUM_IHL_RANGE_16_EQNS) {
-		IPAERR_RL("num_ihl_offset_range_16  Max %d passed value %d\n",
-			IPA_IPFLTR_NUM_IHL_RANGE_16_EQNS,
-			attrib->num_ihl_offset_range_16);
-		return -EPERM;
-	}
-
-	for (i = 0; i < attrib->num_ihl_offset_range_16; i++)
-		pr_err(
-			   "(ihl_ofst_range16: ofst:%u lo:%u hi:%u) ",
-			   attrib->ihl_offset_range_16[i].offset,
-			   attrib->ihl_offset_range_16[i].range_low,
-			   attrib->ihl_offset_range_16[i].range_high);
-
-	if (attrib->ihl_offset_eq_32_present)
-		pr_err(
-			"(ihl_ofst_eq32:%d val:0x%x) ",
-			attrib->ihl_offset_eq_32.offset,
-			attrib->ihl_offset_eq_32.value);
-
-	if (attrib->ihl_offset_eq_16_present)
-		pr_err(
-			"(ihl_ofst_eq16:%d val:0x%x) ",
-			attrib->ihl_offset_eq_16.offset,
-			attrib->ihl_offset_eq_16.value);
-
-	if (attrib->fl_eq_present)
-		pr_err("flow_label:%d ", attrib->fl_eq);
-
-	if (attrib->ipv4_frag_eq_present)
-		pr_err("frag ");
-
-	pr_err("\n");
-	return 0;
-}
-
-static int ipa3_open_dbg(struct inode *inode, struct file *file)
-{
-	file->private_data = inode->i_private;
-	return 0;
-}
-
-static ssize_t ipa3_read_rt(struct file *file, char __user *ubuf, size_t count,
-		loff_t *ppos)
-{
-	int i = 0;
-	struct ipa3_rt_tbl *tbl;
-	struct ipa3_rt_entry *entry;
-	struct ipa3_rt_tbl_set *set;
-	enum ipa_ip_type ip = (enum ipa_ip_type)file->private_data;
-	u32 ofst;
-	u32 ofst_words;
-
-	set = &ipa3_ctx->rt_tbl_set[ip];
-
-	mutex_lock(&ipa3_ctx->lock);
-
-	if (ip ==  IPA_IP_v6) {
-		if (ipa3_ctx->ip6_rt_tbl_hash_lcl)
-			pr_err("Hashable table resides on local memory\n");
-		else
-			pr_err("Hashable table resides on system (ddr) memory\n");
-		if (ipa3_ctx->ip6_rt_tbl_nhash_lcl)
-			pr_err("Non-Hashable table resides on local memory\n");
-		else
-			pr_err("Non-Hashable table resides on system (ddr) memory\n");
-	} else if (ip == IPA_IP_v4) {
-		if (ipa3_ctx->ip4_rt_tbl_hash_lcl)
-			pr_err("Hashable table resides on local memory\n");
-		else
-			pr_err("Hashable table resides on system (ddr) memory\n");
-		if (ipa3_ctx->ip4_rt_tbl_nhash_lcl)
-			pr_err("Non-Hashable table resides on local memory\n");
-		else
-			pr_err("Non-Hashable table resides on system (ddr) memory\n");
-	}
-
-	list_for_each_entry(tbl, &set->head_rt_tbl_list, link) {
-		i = 0;
-		list_for_each_entry(entry, &tbl->head_rt_rule_list, link) {
-			if (entry->proc_ctx) {
-				ofst = entry->proc_ctx->offset_entry->offset;
-				ofst_words =
-					(ofst +
-					ipa3_ctx->hdr_proc_ctx_tbl.start_offset)
-					>> 5;
-
-				pr_err("tbl_idx:%d tbl_name:%s tbl_ref:%u ",
-					entry->tbl->idx, entry->tbl->name,
-					entry->tbl->ref_cnt);
-				pr_err("rule_idx:%d dst:%d ep:%d S:%u ",
-					i, entry->rule.dst,
-					ipa3_get_ep_mapping(entry->rule.dst),
-					!ipa3_ctx->hdr_proc_ctx_tbl_lcl);
-				pr_err("proc_ctx[32B]:%u attrib_mask:%08x ",
-					ofst_words,
-					entry->rule.attrib.attrib_mask);
-				pr_err("rule_id:%u max_prio:%u prio:%u ",
-					entry->rule_id, entry->rule.max_prio,
-					entry->prio);
-				pr_err("enable_stats:%u counter_id:%u\n",
-					entry->rule.enable_stats,
-					entry->rule.cnt_idx);
-				pr_err("hashable:%u retain_hdr:%u ",
-					entry->rule.hashable,
-					entry->rule.retain_hdr);
-			} else {
-				if (entry->hdr)
-					ofst = entry->hdr->offset_entry->offset;
-				else
-					ofst = 0;
-
-				pr_err("tbl_idx:%d tbl_name:%s tbl_ref:%u ",
-					entry->tbl->idx, entry->tbl->name,
-					entry->tbl->ref_cnt);
-				pr_err("rule_idx:%d dst:%d ep:%d S:%u ",
-					i, entry->rule.dst,
-					ipa3_get_ep_mapping(entry->rule.dst),
-					!ipa3_ctx->hdr_tbl_lcl);
-				pr_err("hdr_ofst[words]:%u attrib_mask:%08x ",
-					ofst >> 2,
-					entry->rule.attrib.attrib_mask);
-				pr_err("rule_id:%u max_prio:%u prio:%u ",
-					entry->rule_id, entry->rule.max_prio,
-					entry->prio);
-				pr_err("enable_stats:%u counter_id:%u\n",
-					entry->rule.enable_stats,
-					entry->rule.cnt_idx);
-				pr_err("hashable:%u retain_hdr:%u ",
-					entry->rule.hashable,
-					entry->rule.retain_hdr);
-			}
-
-			ipa3_attrib_dump(&entry->rule.attrib, ip);
-			i++;
-		}
-	}
-	mutex_unlock(&ipa3_ctx->lock);
-
-	return 0;
-}
-
-static ssize_t ipa3_read_rt_hw(struct file *file, char __user *ubuf,
-	size_t count, loff_t *ppos)
-{
-	enum ipa_ip_type ip = (enum ipa_ip_type)file->private_data;
-	int tbls_num;
-	int rules_num;
-	int tbl;
-	int rl;
-	int res = 0;
-	struct ipahal_rt_rule_entry *rules = NULL;
-
-	switch (ip) {
-	case IPA_IP_v4:
-		tbls_num = IPA_MEM_PART(v4_rt_num_index);
-		break;
-	case IPA_IP_v6:
-		tbls_num = IPA_MEM_PART(v6_rt_num_index);
-		break;
-	default:
-		IPAERR("ip type error %d\n", ip);
-		return -EINVAL;
-	}
-
-	IPADBG("Tring to parse %d H/W routing tables - IP=%d\n", tbls_num, ip);
-
-	rules = kzalloc(sizeof(*rules) * IPA_DBG_MAX_RULE_IN_TBL, GFP_KERNEL);
-	if (!rules) {
-		IPAERR("failed to allocate mem for tbl rules\n");
-		return -ENOMEM;
-	}
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-	mutex_lock(&ipa3_ctx->lock);
-
-	for (tbl = 0 ; tbl < tbls_num ; tbl++) {
-		pr_err("=== Routing Table %d = Hashable Rules ===\n", tbl);
-		rules_num = IPA_DBG_MAX_RULE_IN_TBL;
-		res = ipa3_rt_read_tbl_from_hw(tbl, ip, true, rules,
-			&rules_num);
-		if (res) {
-			pr_err("ERROR - Check the logs\n");
-			IPAERR("failed reading tbl from hw\n");
-			goto bail;
-		}
-		if (!rules_num)
-			pr_err("-->No rules. Empty tbl or modem system table\n");
-
-		for (rl = 0 ; rl < rules_num ; rl++) {
-			pr_err("rule_idx:%d dst ep:%d L:%u ",
-				rl, rules[rl].dst_pipe_idx, rules[rl].hdr_lcl);
-
-			if (rules[rl].hdr_type == IPAHAL_RT_RULE_HDR_PROC_CTX)
-				pr_err("proc_ctx:%u attrib_mask:%08x ",
-					rules[rl].hdr_ofst,
-					rules[rl].eq_attrib.rule_eq_bitmap);
-			else
-				pr_err("hdr_ofst:%u attrib_mask:%08x ",
-					rules[rl].hdr_ofst,
-					rules[rl].eq_attrib.rule_eq_bitmap);
-
-			pr_err("rule_id:%u cnt_id:%hhu prio:%u retain_hdr:%u\n",
-				rules[rl].id, rules[rl].cnt_idx,
-				rules[rl].priority, rules[rl].retain_hdr);
-			res = ipa3_attrib_dump_eq(&rules[rl].eq_attrib);
-			if (res) {
-				IPAERR_RL("failed read attrib eq\n");
-				goto bail;
-			}
-		}
-
-		pr_err("=== Routing Table %d = Non-Hashable Rules ===\n", tbl);
-		rules_num = IPA_DBG_MAX_RULE_IN_TBL;
-		res = ipa3_rt_read_tbl_from_hw(tbl, ip, false, rules,
-			&rules_num);
-		if (res) {
-			pr_err("ERROR - Check the logs\n");
-			IPAERR("failed reading tbl from hw\n");
-			goto bail;
-		}
-		if (!rules_num)
-			pr_err("-->No rules. Empty tbl or modem system table\n");
-
-		for (rl = 0 ; rl < rules_num ; rl++) {
-			pr_err("rule_idx:%d dst ep:%d L:%u ",
-				rl, rules[rl].dst_pipe_idx, rules[rl].hdr_lcl);
-
-			if (rules[rl].hdr_type == IPAHAL_RT_RULE_HDR_PROC_CTX)
-				pr_err("proc_ctx:%u attrib_mask:%08x ",
-					rules[rl].hdr_ofst,
-					rules[rl].eq_attrib.rule_eq_bitmap);
-			else
-				pr_err("hdr_ofst:%u attrib_mask:%08x ",
-					rules[rl].hdr_ofst,
-					rules[rl].eq_attrib.rule_eq_bitmap);
-
-			pr_err("rule_id:%u cnt_id:%hhu prio:%u retain_hdr:%u\n",
-				rules[rl].id, rules[rl].cnt_idx,
-				rules[rl].priority, rules[rl].retain_hdr);
-			res = ipa3_attrib_dump_eq(&rules[rl].eq_attrib);
-			if (res) {
-				IPAERR_RL("failed read attrib eq\n");
-				goto bail;
-			}
-		}
-		pr_err("\n");
-	}
-
-bail:
-	mutex_unlock(&ipa3_ctx->lock);
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-	kfree(rules);
-	return res;
-}
-
-static ssize_t ipa3_read_proc_ctx(struct file *file, char __user *ubuf,
-		size_t count, loff_t *ppos)
-{
-	int nbytes = 0;
-	struct ipa3_hdr_proc_ctx_tbl *tbl;
-	struct ipa3_hdr_proc_ctx_entry *entry;
-	u32 ofst_words;
-
-	tbl = &ipa3_ctx->hdr_proc_ctx_tbl;
-
-	mutex_lock(&ipa3_ctx->lock);
-
-	if (ipa3_ctx->hdr_proc_ctx_tbl_lcl)
-		pr_info("Table resides on local memory\n");
-	else
-		pr_info("Table resides on system(ddr) memory\n");
-
-	list_for_each_entry(entry, &tbl->head_proc_ctx_entry_list, link) {
-		ofst_words = (entry->offset_entry->offset +
-			ipa3_ctx->hdr_proc_ctx_tbl.start_offset)
-			>> 5;
-		if (entry->hdr->is_hdr_proc_ctx) {
-			nbytes += scnprintf(dbg_buff + nbytes,
-				IPA_MAX_MSG_LEN - nbytes,
-				"id:%u hdr_proc_type:%s proc_ctx[32B]:%u ",
-				entry->id,
-				ipa3_hdr_proc_type_name[entry->type],
-				ofst_words);
-			nbytes += scnprintf(dbg_buff + nbytes,
-				IPA_MAX_MSG_LEN - nbytes,
-				"hdr_phys_base:0x%pa\n",
-				&entry->hdr->phys_base);
-		} else {
-			nbytes += scnprintf(dbg_buff + nbytes,
-				IPA_MAX_MSG_LEN - nbytes,
-				"id:%u hdr_proc_type:%s proc_ctx[32B]:%u ",
-				entry->id,
-				ipa3_hdr_proc_type_name[entry->type],
-				ofst_words);
-			nbytes += scnprintf(dbg_buff + nbytes,
-				IPA_MAX_MSG_LEN - nbytes,
-				"hdr[words]:%u\n",
-				entry->hdr->offset_entry->offset >> 2);
-		}
-	}
-	mutex_unlock(&ipa3_ctx->lock);
-
-	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, nbytes);
-}
-
-static ssize_t ipa3_read_flt(struct file *file, char __user *ubuf, size_t count,
-		loff_t *ppos)
-{
-	int i;
-	int j;
-	struct ipa3_flt_tbl *tbl;
-	struct ipa3_flt_entry *entry;
-	enum ipa_ip_type ip = (enum ipa_ip_type)file->private_data;
-	struct ipa3_rt_tbl *rt_tbl;
-	u32 rt_tbl_idx;
-	u32 bitmap;
-	bool eq;
-	int res = 0;
-
-	mutex_lock(&ipa3_ctx->lock);
-
-	for (j = 0; j < ipa3_ctx->ipa_num_pipes; j++) {
-		if (!ipa_is_ep_support_flt(j))
-			continue;
-		tbl = &ipa3_ctx->flt_tbl[j][ip];
-		i = 0;
-		list_for_each_entry(entry, &tbl->head_flt_rule_list, link) {
-			if (entry->cookie != IPA_FLT_COOKIE)
-				continue;
-			if (entry->rule.eq_attrib_type) {
-				rt_tbl_idx = entry->rule.rt_tbl_idx;
-				bitmap = entry->rule.eq_attrib.rule_eq_bitmap;
-				eq = true;
-			} else {
-				rt_tbl = ipa3_id_find(entry->rule.rt_tbl_hdl);
-				if (rt_tbl == NULL ||
-					rt_tbl->cookie != IPA_RT_TBL_COOKIE)
-					rt_tbl_idx =  ~0;
-				else
-					rt_tbl_idx = rt_tbl->idx;
-				bitmap = entry->rule.attrib.attrib_mask;
-				eq = false;
-			}
-			pr_err("ep_idx:%d rule_idx:%d act:%d rt_tbl_idx:%d ",
-				j, i, entry->rule.action, rt_tbl_idx);
-			pr_err("attrib_mask:%08x retain_hdr:%d eq:%d ",
-				bitmap, entry->rule.retain_hdr, eq);
-			pr_err("hashable:%u rule_id:%u max_prio:%u prio:%u ",
-				entry->rule.hashable, entry->rule_id,
-				entry->rule.max_prio, entry->prio);
-			pr_err("enable_stats:%u counter_id:%u\n",
-					entry->rule.enable_stats,
-					entry->rule.cnt_idx);
-			if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0)
-				pr_err("pdn index %d, set metadata %d ",
-					entry->rule.pdn_idx,
-					entry->rule.set_metadata);
-			if (eq) {
-				res = ipa3_attrib_dump_eq(
-						&entry->rule.eq_attrib);
-				if (res) {
-					IPAERR_RL("failed read attrib eq\n");
-					goto bail;
-				}
-			} else
-				ipa3_attrib_dump(
-					&entry->rule.attrib, ip);
-			i++;
-		}
-	}
-bail:
-	mutex_unlock(&ipa3_ctx->lock);
-
-	return res;
-}
-
-static ssize_t ipa3_read_flt_hw(struct file *file, char __user *ubuf,
-	size_t count, loff_t *ppos)
-{
-	int pipe;
-	int rl;
-	int rules_num;
-	struct ipahal_flt_rule_entry *rules;
-	enum ipa_ip_type ip = (enum ipa_ip_type)file->private_data;
-	u32 rt_tbl_idx;
-	u32 bitmap;
-	int res = 0;
-
-	IPADBG("Tring to parse %d H/W filtering tables - IP=%d\n",
-		ipa3_ctx->ep_flt_num, ip);
-
-	rules = kzalloc(sizeof(*rules) * IPA_DBG_MAX_RULE_IN_TBL, GFP_KERNEL);
-	if (!rules)
-		return -ENOMEM;
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-	mutex_lock(&ipa3_ctx->lock);
-	for (pipe = 0; pipe < ipa3_ctx->ipa_num_pipes; pipe++) {
-		if (!ipa_is_ep_support_flt(pipe))
-			continue;
-		pr_err("=== Filtering Table ep:%d = Hashable Rules ===\n",
-			pipe);
-		rules_num = IPA_DBG_MAX_RULE_IN_TBL;
-		res = ipa3_flt_read_tbl_from_hw(pipe, ip, true, rules,
-			&rules_num);
-		if (res) {
-			pr_err("ERROR - Check the logs\n");
-			IPAERR("failed reading tbl from hw\n");
-			goto bail;
-		}
-		if (!rules_num)
-			pr_err("-->No rules. Empty tbl or modem sys table\n");
-
-		for (rl = 0; rl < rules_num; rl++) {
-			rt_tbl_idx = rules[rl].rule.rt_tbl_idx;
-			bitmap = rules[rl].rule.eq_attrib.rule_eq_bitmap;
-			pr_err("ep_idx:%d rule_idx:%d act:%d rt_tbl_idx:%d ",
-				pipe, rl, rules[rl].rule.action, rt_tbl_idx);
-			pr_err("attrib_mask:%08x retain_hdr:%d ",
-				bitmap, rules[rl].rule.retain_hdr);
-			pr_err("rule_id:%u cnt_id:%hhu prio:%u\n",
-				rules[rl].id, rules[rl].cnt_idx,
-				rules[rl].priority);
-			if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0)
-				pr_err("pdn: %u, set_metadata: %u ",
-					rules[rl].rule.pdn_idx,
-					rules[rl].rule.set_metadata);
-			res = ipa3_attrib_dump_eq(&rules[rl].rule.eq_attrib);
-			if (res) {
-				IPAERR_RL("failed read attrib eq\n");
-				goto bail;
-			}
-		}
-
-		pr_err("=== Filtering Table ep:%d = Non-Hashable Rules ===\n",
-			pipe);
-		rules_num = IPA_DBG_MAX_RULE_IN_TBL;
-		res = ipa3_flt_read_tbl_from_hw(pipe, ip, false, rules,
-			&rules_num);
-		if (res) {
-			IPAERR("failed reading tbl from hw\n");
-			goto bail;
-		}
-		if (!rules_num)
-			pr_err("-->No rules. Empty tbl or modem sys table\n");
-		for (rl = 0; rl < rules_num; rl++) {
-			rt_tbl_idx = rules[rl].rule.rt_tbl_idx;
-			bitmap = rules[rl].rule.eq_attrib.rule_eq_bitmap;
-			pr_err("ep_idx:%d rule_idx:%d act:%d rt_tbl_idx:%d ",
-				pipe, rl, rules[rl].rule.action, rt_tbl_idx);
-			pr_err("attrib_mask:%08x retain_hdr:%d ",
-				bitmap, rules[rl].rule.retain_hdr);
-			pr_err("rule_id:%u cnt_id:%hhu prio:%u\n",
-				rules[rl].id, rules[rl].cnt_idx,
-				rules[rl].priority);
-			if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0)
-				pr_err("pdn: %u, set_metadata: %u ",
-					rules[rl].rule.pdn_idx,
-					rules[rl].rule.set_metadata);
-			res = ipa3_attrib_dump_eq(&rules[rl].rule.eq_attrib);
-			if (res) {
-				IPAERR_RL("failed read attrib eq\n");
-				goto bail;
-			}
-		}
-		pr_err("\n");
-	}
-
-bail:
-	mutex_unlock(&ipa3_ctx->lock);
-	kfree(rules);
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-	return res;
-}
-
-static ssize_t ipa3_read_stats(struct file *file, char __user *ubuf,
-		size_t count, loff_t *ppos)
-{
-	int nbytes;
-	int i;
-	int cnt = 0;
-	uint connect = 0;
-
-	for (i = 0; i < ipa3_ctx->ipa_num_pipes; i++)
-		connect |= (ipa3_ctx->ep[i].valid << i);
-
-	nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
-		"sw_tx=%u\n"
-		"hw_tx=%u\n"
-		"tx_non_linear=%u\n"
-		"tx_compl=%u\n"
-		"wan_rx=%u\n"
-		"stat_compl=%u\n"
-		"lan_aggr_close=%u\n"
-		"wan_aggr_close=%u\n"
-		"act_clnt=%u\n"
-		"con_clnt_bmap=0x%x\n"
-		"wan_rx_empty=%u\n"
-		"wan_repl_rx_empty=%u\n"
-		"lan_rx_empty=%u\n"
-		"lan_repl_rx_empty=%u\n"
-		"flow_enable=%u\n"
-		"flow_disable=%u\n",
-		ipa3_ctx->stats.tx_sw_pkts,
-		ipa3_ctx->stats.tx_hw_pkts,
-		ipa3_ctx->stats.tx_non_linear,
-		ipa3_ctx->stats.tx_pkts_compl,
-		ipa3_ctx->stats.rx_pkts,
-		ipa3_ctx->stats.stat_compl,
-		ipa3_ctx->stats.aggr_close,
-		ipa3_ctx->stats.wan_aggr_close,
-		atomic_read(&ipa3_ctx->ipa3_active_clients.cnt),
-		connect,
-		ipa3_ctx->stats.wan_rx_empty,
-		ipa3_ctx->stats.wan_repl_rx_empty,
-		ipa3_ctx->stats.lan_rx_empty,
-		ipa3_ctx->stats.lan_repl_rx_empty,
-		ipa3_ctx->stats.flow_enable,
-		ipa3_ctx->stats.flow_disable);
-	cnt += nbytes;
-
-	for (i = 0; i < IPAHAL_PKT_STATUS_EXCEPTION_MAX; i++) {
-		nbytes = scnprintf(dbg_buff + cnt,
-			IPA_MAX_MSG_LEN - cnt,
-			"lan_rx_excp[%u:%20s]=%u\n", i,
-			ipahal_pkt_status_exception_str(i),
-			ipa3_ctx->stats.rx_excp_pkts[i]);
-		cnt += nbytes;
-	}
-
-	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, cnt);
-}
-
-static ssize_t ipa3_read_odlstats(struct file *file, char __user *ubuf,
-		size_t count, loff_t *ppos)
-{
-	int nbytes;
-	int cnt = 0;
-
-	nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
-			"ODL received pkt =%u\n"
-			"ODL processed pkt to DIAG=%u\n"
-			"ODL dropped pkt =%u\n"
-			"ODL packet in queue  =%u\n",
-			ipa3_odl_ctx->stats.odl_rx_pkt,
-			ipa3_odl_ctx->stats.odl_tx_diag_pkt,
-			ipa3_odl_ctx->stats.odl_drop_pkt,
-			atomic_read(&ipa3_odl_ctx->stats.numer_in_queue));
-
-	cnt += nbytes;
-
-	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, cnt);
-}
-
-static ssize_t ipa3_read_wstats(struct file *file, char __user *ubuf,
-		size_t count, loff_t *ppos)
-{
-
-#define HEAD_FRMT_STR "%25s\n"
-#define FRMT_STR "%25s %10u\n"
-#define FRMT_STR1 "%25s %10u\n\n"
-
-	int cnt = 0;
-	int nbytes;
-	int ipa_ep_idx;
-	enum ipa_client_type client = IPA_CLIENT_WLAN1_PROD;
-	struct ipa3_ep_context *ep;
-
-	do {
-		nbytes = scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN - cnt,
-			HEAD_FRMT_STR, "Client IPA_CLIENT_WLAN1_PROD Stats:");
-		cnt += nbytes;
-
-		ipa_ep_idx = ipa3_get_ep_mapping(client);
-		if (ipa_ep_idx == -1) {
-			nbytes = scnprintf(dbg_buff + cnt,
-				IPA_MAX_MSG_LEN - cnt, HEAD_FRMT_STR, "Not up");
-			cnt += nbytes;
-			break;
-		}
-
-		ep = &ipa3_ctx->ep[ipa_ep_idx];
-		if (ep->valid != 1) {
-			nbytes = scnprintf(dbg_buff + cnt,
-				IPA_MAX_MSG_LEN - cnt, HEAD_FRMT_STR, "Not up");
-			cnt += nbytes;
-			break;
-		}
-
-		nbytes = scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN - cnt,
-			FRMT_STR, "Avail Fifo Desc:",
-			atomic_read(&ep->avail_fifo_desc));
-		cnt += nbytes;
-
-		nbytes = scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN - cnt,
-			FRMT_STR, "Rx Pkts Rcvd:", ep->wstats.rx_pkts_rcvd);
-		cnt += nbytes;
-
-		nbytes = scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN - cnt,
-			FRMT_STR, "Rx Pkts Status Rcvd:",
-			ep->wstats.rx_pkts_status_rcvd);
-		cnt += nbytes;
-
-		nbytes = scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN - cnt,
-			FRMT_STR, "Rx DH Rcvd:", ep->wstats.rx_hd_rcvd);
-		cnt += nbytes;
-
-		nbytes = scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN - cnt,
-			FRMT_STR, "Rx DH Processed:",
-			ep->wstats.rx_hd_processed);
-		cnt += nbytes;
-
-		nbytes = scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN - cnt,
-			FRMT_STR, "Rx DH Sent Back:", ep->wstats.rx_hd_reply);
-		cnt += nbytes;
-
-		nbytes = scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN - cnt,
-			FRMT_STR, "Rx Pkt Leak:", ep->wstats.rx_pkt_leak);
-		cnt += nbytes;
-
-		nbytes = scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN - cnt,
-			FRMT_STR1, "Rx DP Fail:", ep->wstats.rx_dp_fail);
-		cnt += nbytes;
-
-	} while (0);
-
-	client = IPA_CLIENT_WLAN1_CONS;
-	nbytes = scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN - cnt, HEAD_FRMT_STR,
-		"Client IPA_CLIENT_WLAN1_CONS Stats:");
-	cnt += nbytes;
-	while (1) {
-		ipa_ep_idx = ipa3_get_ep_mapping(client);
-		if (ipa_ep_idx == -1) {
-			nbytes = scnprintf(dbg_buff + cnt,
-				IPA_MAX_MSG_LEN - cnt, HEAD_FRMT_STR, "Not up");
-			cnt += nbytes;
-			goto nxt_clnt_cons;
-		}
-
-		ep = &ipa3_ctx->ep[ipa_ep_idx];
-		if (ep->valid != 1) {
-			nbytes = scnprintf(dbg_buff + cnt,
-				IPA_MAX_MSG_LEN - cnt, HEAD_FRMT_STR, "Not up");
-			cnt += nbytes;
-			goto nxt_clnt_cons;
-		}
-
-		nbytes = scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN - cnt,
-			FRMT_STR, "Tx Pkts Received:", ep->wstats.tx_pkts_rcvd);
-		cnt += nbytes;
-
-		nbytes = scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN - cnt,
-			FRMT_STR, "Tx Pkts Sent:", ep->wstats.tx_pkts_sent);
-		cnt += nbytes;
-
-		nbytes = scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN - cnt,
-			FRMT_STR1, "Tx Pkts Dropped:",
-			ep->wstats.tx_pkts_dropped);
-		cnt += nbytes;
-
-nxt_clnt_cons:
-			switch (client) {
-			case IPA_CLIENT_WLAN1_CONS:
-				client = IPA_CLIENT_WLAN2_CONS;
-				nbytes = scnprintf(dbg_buff + cnt,
-					IPA_MAX_MSG_LEN - cnt, HEAD_FRMT_STR,
-					"Client IPA_CLIENT_WLAN2_CONS Stats:");
-				cnt += nbytes;
-				continue;
-			case IPA_CLIENT_WLAN2_CONS:
-				client = IPA_CLIENT_WLAN3_CONS;
-				nbytes = scnprintf(dbg_buff + cnt,
-					IPA_MAX_MSG_LEN - cnt, HEAD_FRMT_STR,
-					"Client IPA_CLIENT_WLAN3_CONS Stats:");
-				cnt += nbytes;
-				continue;
-			case IPA_CLIENT_WLAN3_CONS:
-				client = IPA_CLIENT_WLAN4_CONS;
-				nbytes = scnprintf(dbg_buff + cnt,
-					IPA_MAX_MSG_LEN - cnt, HEAD_FRMT_STR,
-					"Client IPA_CLIENT_WLAN4_CONS Stats:");
-				cnt += nbytes;
-				continue;
-			case IPA_CLIENT_WLAN4_CONS:
-			default:
-				break;
-			}
-		break;
-	}
-
-	nbytes = scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN - cnt,
-		"\n"HEAD_FRMT_STR, "All Wlan Consumer pipes stats:");
-	cnt += nbytes;
-
-	nbytes = scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN - cnt, FRMT_STR,
-		"Tx Comm Buff Allocated:",
-		ipa3_ctx->wc_memb.wlan_comm_total_cnt);
-	cnt += nbytes;
-
-	nbytes = scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN - cnt, FRMT_STR,
-		"Tx Comm Buff Avail:", ipa3_ctx->wc_memb.wlan_comm_free_cnt);
-	cnt += nbytes;
-
-	nbytes = scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN - cnt, FRMT_STR1,
-		"Total Tx Pkts Freed:", ipa3_ctx->wc_memb.total_tx_pkts_freed);
-	cnt += nbytes;
-
-	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, cnt);
-}
-
-static ssize_t ipa3_read_ntn(struct file *file, char __user *ubuf,
-		size_t count, loff_t *ppos)
-{
-#define TX_STATS(y) \
-	ipa3_ctx->uc_ntn_ctx.ntn_uc_stats_mmio->tx_ch_stats[0].y
-#define RX_STATS(y) \
-	ipa3_ctx->uc_ntn_ctx.ntn_uc_stats_mmio->rx_ch_stats[0].y
-
-	struct Ipa3HwStatsNTNInfoData_t stats;
-	int nbytes;
-	int cnt = 0;
-
-	if (!ipa3_get_ntn_stats(&stats)) {
-		nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
-			"TX num_pkts_processed=%u\n"
-			"TX ringFull=%u\n"
-			"TX ringEmpty=%u\n"
-			"TX ringUsageHigh=%u\n"
-			"TX ringUsageLow=%u\n"
-			"TX RingUtilCount=%u\n"
-			"TX bamFifoFull=%u\n"
-			"TX bamFifoEmpty=%u\n"
-			"TX bamFifoUsageHigh=%u\n"
-			"TX bamFifoUsageLow=%u\n"
-			"TX bamUtilCount=%u\n"
-			"TX num_db=%u\n"
-			"TX num_qmb_int_handled=%u\n"
-			"TX ipa_pipe_number=%u\n",
-			TX_STATS(num_pkts_processed),
-			TX_STATS(ring_stats.ringFull),
-			TX_STATS(ring_stats.ringEmpty),
-			TX_STATS(ring_stats.ringUsageHigh),
-			TX_STATS(ring_stats.ringUsageLow),
-			TX_STATS(ring_stats.RingUtilCount),
-			TX_STATS(gsi_stats.bamFifoFull),
-			TX_STATS(gsi_stats.bamFifoEmpty),
-			TX_STATS(gsi_stats.bamFifoUsageHigh),
-			TX_STATS(gsi_stats.bamFifoUsageLow),
-			TX_STATS(gsi_stats.bamUtilCount),
-			TX_STATS(num_db),
-			TX_STATS(num_qmb_int_handled),
-			TX_STATS(ipa_pipe_number));
-		cnt += nbytes;
-		nbytes = scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN - cnt,
-			"RX num_pkts_processed=%u\n"
-			"RX ringFull=%u\n"
-			"RX ringEmpty=%u\n"
-			"RX ringUsageHigh=%u\n"
-			"RX ringUsageLow=%u\n"
-			"RX RingUtilCount=%u\n"
-			"RX bamFifoFull=%u\n"
-			"RX bamFifoEmpty=%u\n"
-			"RX bamFifoUsageHigh=%u\n"
-			"RX bamFifoUsageLow=%u\n"
-			"RX bamUtilCount=%u\n"
-			"RX num_db=%u\n"
-			"RX num_qmb_int_handled=%u\n"
-			"RX ipa_pipe_number=%u\n",
-			RX_STATS(num_pkts_processed),
-			RX_STATS(ring_stats.ringFull),
-			RX_STATS(ring_stats.ringEmpty),
-			RX_STATS(ring_stats.ringUsageHigh),
-			RX_STATS(ring_stats.ringUsageLow),
-			RX_STATS(ring_stats.RingUtilCount),
-			RX_STATS(gsi_stats.bamFifoFull),
-			RX_STATS(gsi_stats.bamFifoEmpty),
-			RX_STATS(gsi_stats.bamFifoUsageHigh),
-			RX_STATS(gsi_stats.bamFifoUsageLow),
-			RX_STATS(gsi_stats.bamUtilCount),
-			RX_STATS(num_db),
-			RX_STATS(num_qmb_int_handled),
-			RX_STATS(ipa_pipe_number));
-		cnt += nbytes;
-	} else {
-		nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
-				"Fail to read NTN stats\n");
-		cnt += nbytes;
-	}
-
-	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, cnt);
-}
-
-static ssize_t ipa3_read_wdi(struct file *file, char __user *ubuf,
-		size_t count, loff_t *ppos)
-{
-	struct IpaHwStatsWDIInfoData_t stats;
-	int nbytes;
-	int cnt = 0;
-	struct IpaHwStatsWDITxInfoData_t *tx_ch_ptr;
-
-	if (!ipa3_get_wdi_stats(&stats)) {
-		tx_ch_ptr = &stats.tx_ch_stats;
-		nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
-			"TX num_pkts_processed=%u\n"
-			"TX copy_engine_doorbell_value=%u\n"
-			"TX num_db_fired=%u\n"
-			"TX ringFull=%u\n"
-			"TX ringEmpty=%u\n"
-			"TX ringUsageHigh=%u\n"
-			"TX ringUsageLow=%u\n"
-			"TX RingUtilCount=%u\n"
-			"TX bamFifoFull=%u\n"
-			"TX bamFifoEmpty=%u\n"
-			"TX bamFifoUsageHigh=%u\n"
-			"TX bamFifoUsageLow=%u\n"
-			"TX bamUtilCount=%u\n"
-			"TX num_db=%u\n"
-			"TX num_unexpected_db=%u\n"
-			"TX num_bam_int_handled=%u\n"
-			"TX num_bam_int_in_non_running_state=%u\n"
-			"TX num_qmb_int_handled=%u\n"
-			"TX num_bam_int_handled_while_wait_for_bam=%u\n",
-			tx_ch_ptr->num_pkts_processed,
-			tx_ch_ptr->copy_engine_doorbell_value,
-			tx_ch_ptr->num_db_fired,
-			tx_ch_ptr->tx_comp_ring_stats.ringFull,
-			tx_ch_ptr->tx_comp_ring_stats.ringEmpty,
-			tx_ch_ptr->tx_comp_ring_stats.ringUsageHigh,
-			tx_ch_ptr->tx_comp_ring_stats.ringUsageLow,
-			tx_ch_ptr->tx_comp_ring_stats.RingUtilCount,
-			tx_ch_ptr->bam_stats.bamFifoFull,
-			tx_ch_ptr->bam_stats.bamFifoEmpty,
-			tx_ch_ptr->bam_stats.bamFifoUsageHigh,
-			tx_ch_ptr->bam_stats.bamFifoUsageLow,
-			tx_ch_ptr->bam_stats.bamUtilCount,
-			tx_ch_ptr->num_db,
-			tx_ch_ptr->num_unexpected_db,
-			tx_ch_ptr->num_bam_int_handled,
-			tx_ch_ptr->num_bam_int_in_non_running_state,
-			tx_ch_ptr->num_qmb_int_handled,
-			tx_ch_ptr->num_bam_int_handled_while_wait_for_bam);
-		cnt += nbytes;
-		nbytes = scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN - cnt,
-			"RX max_outstanding_pkts=%u\n"
-			"RX num_pkts_processed=%u\n"
-			"RX rx_ring_rp_value=%u\n"
-			"RX ringFull=%u\n"
-			"RX ringEmpty=%u\n"
-			"RX ringUsageHigh=%u\n"
-			"RX ringUsageLow=%u\n"
-			"RX RingUtilCount=%u\n"
-			"RX bamFifoFull=%u\n"
-			"RX bamFifoEmpty=%u\n"
-			"RX bamFifoUsageHigh=%u\n"
-			"RX bamFifoUsageLow=%u\n"
-			"RX bamUtilCount=%u\n"
-			"RX num_bam_int_handled=%u\n"
-			"RX num_db=%u\n"
-			"RX num_unexpected_db=%u\n"
-			"RX num_pkts_in_dis_uninit_state=%u\n"
-			"RX num_ic_inj_vdev_change=%u\n"
-			"RX num_ic_inj_fw_desc_change=%u\n"
-			"RX num_qmb_int_handled=%u\n"
-			"RX reserved1=%u\n"
-			"RX reserved2=%u\n",
-			stats.rx_ch_stats.max_outstanding_pkts,
-			stats.rx_ch_stats.num_pkts_processed,
-			stats.rx_ch_stats.rx_ring_rp_value,
-			stats.rx_ch_stats.rx_ind_ring_stats.ringFull,
-			stats.rx_ch_stats.rx_ind_ring_stats.ringEmpty,
-			stats.rx_ch_stats.rx_ind_ring_stats.ringUsageHigh,
-			stats.rx_ch_stats.rx_ind_ring_stats.ringUsageLow,
-			stats.rx_ch_stats.rx_ind_ring_stats.RingUtilCount,
-			stats.rx_ch_stats.bam_stats.bamFifoFull,
-			stats.rx_ch_stats.bam_stats.bamFifoEmpty,
-			stats.rx_ch_stats.bam_stats.bamFifoUsageHigh,
-			stats.rx_ch_stats.bam_stats.bamFifoUsageLow,
-			stats.rx_ch_stats.bam_stats.bamUtilCount,
-			stats.rx_ch_stats.num_bam_int_handled,
-			stats.rx_ch_stats.num_db,
-			stats.rx_ch_stats.num_unexpected_db,
-			stats.rx_ch_stats.num_pkts_in_dis_uninit_state,
-			stats.rx_ch_stats.num_ic_inj_vdev_change,
-			stats.rx_ch_stats.num_ic_inj_fw_desc_change,
-			stats.rx_ch_stats.num_qmb_int_handled,
-			stats.rx_ch_stats.reserved1,
-			stats.rx_ch_stats.reserved2);
-		cnt += nbytes;
-	} else {
-		nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
-				"Fail to read WDI stats\n");
-		cnt += nbytes;
-	}
-
-	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, cnt);
-}
-
-static ssize_t ipa3_write_dbg_cnt(struct file *file, const char __user *buf,
-		size_t count, loff_t *ppos)
-{
-	u32 option = 0;
-	struct ipahal_reg_debug_cnt_ctrl dbg_cnt_ctrl;
-	int ret;
-
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0) {
-		IPAERR("IPA_DEBUG_CNT_CTRL is not supported in IPA 4.0\n");
-		return -EPERM;
-	}
-
-	ret = kstrtou32_from_user(buf, count, 0, &option);
-	if (ret)
-		return ret;
-
-	memset(&dbg_cnt_ctrl, 0, sizeof(dbg_cnt_ctrl));
-	dbg_cnt_ctrl.type = DBG_CNT_TYPE_GENERAL;
-	dbg_cnt_ctrl.product = true;
-	dbg_cnt_ctrl.src_pipe = 0xff;
-	dbg_cnt_ctrl.rule_idx_pipe_rule = false;
-	dbg_cnt_ctrl.rule_idx = 0;
-	if (option == 1)
-		dbg_cnt_ctrl.en = true;
-	else
-		dbg_cnt_ctrl.en = false;
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-	ipahal_write_reg_n_fields(IPA_DEBUG_CNT_CTRL_n, 0, &dbg_cnt_ctrl);
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-
-	return count;
-}
-
-static ssize_t ipa3_read_dbg_cnt(struct file *file, char __user *ubuf,
-		size_t count, loff_t *ppos)
-{
-	int nbytes;
-	u32 regval;
-
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0) {
-		IPAERR("IPA_DEBUG_CNT_REG is not supported in IPA 4.0\n");
-		return -EPERM;
-	}
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-	regval =
-		ipahal_read_reg_n(IPA_DEBUG_CNT_REG_n, 0);
-	nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
-			"IPA_DEBUG_CNT_REG_0=0x%x\n", regval);
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-
-	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, nbytes);
-}
-
-static ssize_t ipa3_read_msg(struct file *file, char __user *ubuf,
-		size_t count, loff_t *ppos)
-{
-	int nbytes;
-	int cnt = 0;
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(ipa3_event_name); i++) {
-		nbytes = scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN - cnt,
-				"msg[%u:%27s] W:%u R:%u\n", i,
-				ipa3_event_name[i],
-				ipa3_ctx->stats.msg_w[i],
-				ipa3_ctx->stats.msg_r[i]);
-		cnt += nbytes;
-	}
-
-	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, cnt);
-}
-
-static void ipa3_read_table(
-	char *table_addr, u32 table_size,
-	u32 *total_num_entries,
-	u32 *rule_id,
-	enum ipahal_nat_type nat_type)
-{
-	int result;
-	char *entry;
-	size_t entry_size;
-	bool entry_zeroed;
-	bool entry_valid;
-	u32 i, num_entries = 0, id = *rule_id;
-	char *buff;
-	size_t buff_size = 2 * IPA_MAX_ENTRY_STRING_LEN;
-
-	IPADBG("\n");
-	if (table_addr == NULL) {
-		pr_err("NULL NAT table\n");
-		return;
-	}
-
-	result = ipahal_nat_entry_size(nat_type, &entry_size);
-	if (result) {
-		IPAERR("Failed to retrieve size of %s entry\n",
-			ipahal_nat_type_str(nat_type));
-		return;
-	}
-
-	buff = kzalloc(buff_size, GFP_KERNEL);
-	if (!buff) {
-		IPAERR("Out of memory\n");
-		return;
-	}
-
-	for (i = 0, entry = table_addr;
-		i < table_size;
-		++i, ++id, entry += entry_size) {
-		result = ipahal_nat_is_entry_zeroed(nat_type, entry,
-			&entry_zeroed);
-		if (result) {
-			IPAERR(
-				"Failed to determine whether the %s entry is definitely zero\n"
-					, ipahal_nat_type_str(nat_type));
-			goto bail;
-		}
-		if (entry_zeroed)
-			continue;
-
-		result = ipahal_nat_is_entry_valid(nat_type, entry,
-			&entry_valid);
-		if (result) {
-			IPAERR(
-				"Failed to determine whether the %s entry is valid\n"
-					, ipahal_nat_type_str(nat_type));
-			goto bail;
-		}
-
-		if (entry_valid) {
-			++num_entries;
-			pr_err("\tEntry_Index=%d\n", id);
-		} else
-			pr_err("\tEntry_Index=%d - Invalid Entry\n", id);
-
-		ipahal_nat_stringify_entry(nat_type, entry,
-			buff, buff_size);
-		pr_err("%s\n", buff);
-		memset(buff, 0, buff_size);
-	}
-
-	if (num_entries)
-		pr_err("\n");
-	else
-		pr_err("\tEmpty\n\n");
-	IPADBG("return\n");
-bail:
-	kfree(buff);
-	*rule_id = id;
-	*total_num_entries += num_entries;
-}
-
-static void ipa3_start_read_memory_device(
-	struct ipa3_nat_ipv6ct_common_mem *dev,
-	enum ipahal_nat_type nat_type,
-	u32 *num_entries)
-{
-	u32 rule_id = 0;
-
-	IPADBG("\n");
-
-	pr_err("%s_Table_Size=%d\n",
-		dev->name, dev->table_entries + 1);
-
-	pr_err("%s_Expansion_Table_Size=%d\n",
-		dev->name, dev->expn_table_entries);
-
-	if (!dev->is_sys_mem)
-		pr_err("Not supported for local(shared) memory\n");
-
-	pr_err("\n%s Base Table:\n", dev->name);
-	ipa3_read_table(dev->base_table_addr, dev->table_entries + 1,
-		num_entries, &rule_id, nat_type);
-
-	pr_err("%s Expansion Table:\n", dev->name);
-	ipa3_read_table(
-		dev->expansion_table_addr, dev->expn_table_entries,
-		num_entries,
-		&rule_id,
-		nat_type);
-
-	IPADBG("return\n");
-}
-
-static void ipa3_finish_read_memory_device(
-	struct ipa3_nat_ipv6ct_common_mem *dev,
-	u32 num_entries)
-{
-	IPADBG("\n");
-	pr_err("Overall number %s entries: %d\n\n", dev->name, num_entries);
-	IPADBG("return\n");
-}
-
-static void ipa3_read_pdn_table(void)
-{
-	int i, result;
-	char *pdn_entry;
-	size_t pdn_entry_size;
-	bool entry_zeroed;
-	bool entry_valid;
-	char *buff;
-	size_t buff_size = 128;
-
-	IPADBG("\n");
-
-	result = ipahal_nat_entry_size(IPAHAL_NAT_IPV4_PDN, &pdn_entry_size);
-	if (result) {
-		IPAERR("Failed to retrieve size of PDN entry");
-		return;
-	}
-
-	buff = kzalloc(buff_size, GFP_KERNEL);
-	if (!buff) {
-		IPAERR("Out of memory\n");
-		return;
-	}
-
-	for (i = 0, pdn_entry = ipa3_ctx->nat_mem.pdn_mem.base;
-		i < IPA_MAX_PDN_NUM;
-		++i, pdn_entry += pdn_entry_size) {
-		result = ipahal_nat_is_entry_zeroed(IPAHAL_NAT_IPV4_PDN,
-			pdn_entry, &entry_zeroed);
-		if (result) {
-			IPAERR(
-				"Failed to determine whether the PDN entry is definitely zero\n");
-			goto bail;
-		}
-		if (entry_zeroed)
-			continue;
-
-		result = ipahal_nat_is_entry_valid(IPAHAL_NAT_IPV4_PDN,
-			pdn_entry, &entry_valid);
-		if (result) {
-			IPAERR(
-				"Failed to determine whether the PDN entry is valid\n");
-			goto bail;
-		}
-		if (entry_valid)
-			pr_err("PDN %d:\n", i);
-		else
-			pr_err("PDN %d - Invalid:\n", i);
-
-		ipahal_nat_stringify_entry(IPAHAL_NAT_IPV4_PDN,
-				pdn_entry, buff, buff_size);
-		pr_err("%s\n", buff);
-		memset(buff, 0, buff_size);
-	}
-	pr_err("\n");
-bail:
-	kfree(buff);
-	IPADBG("return\n");
-}
-
-static ssize_t ipa3_read_nat4(struct file *file,
-		char __user *ubuf, size_t count,
-		loff_t *ppos)
-{
-	u32 rule_id = 0, num_entries = 0, index_num_entries = 0;
-
-	pr_err("IPA3 NAT stats\n");
-	if (!ipa3_ctx->nat_mem.dev.is_dev_init) {
-		pr_err("NAT hasn't been initialized or not supported\n");
-		goto ret;
-	}
-
-	mutex_lock(&ipa3_ctx->nat_mem.dev.lock);
-
-	if (!ipa3_ctx->nat_mem.dev.is_hw_init) {
-		pr_err("NAT H/W hasn't been initialized\n");
-		goto bail;
-	}
-
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0) {
-		ipa3_read_pdn_table();
-	} else {
-		pr_err("NAT Table IP Address=%pI4h\n\n",
-			&ipa3_ctx->nat_mem.public_ip_addr);
-	}
-
-	ipa3_start_read_memory_device(&ipa3_ctx->nat_mem.dev,
-		IPAHAL_NAT_IPV4, &num_entries);
-
-	/* Print Index tables */
-	pr_err("ipaNatTable Index Table:\n");
-	ipa3_read_table(
-		ipa3_ctx->nat_mem.index_table_addr,
-		ipa3_ctx->nat_mem.dev.table_entries + 1,
-		&index_num_entries,
-		&rule_id,
-		IPAHAL_NAT_IPV4_INDEX);
-
-	pr_err("ipaNatTable Expansion Index Table:\n");
-	ipa3_read_table(
-		ipa3_ctx->nat_mem.index_table_expansion_addr,
-		ipa3_ctx->nat_mem.dev.expn_table_entries,
-		&index_num_entries,
-		&rule_id,
-		IPAHAL_NAT_IPV4_INDEX);
-
-	if (num_entries != index_num_entries)
-		IPAERR(
-			"The NAT table number of entries %d is different from index table number of entries %d\n",
-			num_entries, index_num_entries);
-
-	ipa3_finish_read_memory_device(&ipa3_ctx->nat_mem.dev, num_entries);
-
-	IPADBG("return\n");
-bail:
-	mutex_unlock(&ipa3_ctx->nat_mem.dev.lock);
-ret:
-	return 0;
-}
-
-static ssize_t ipa3_read_ipv6ct(struct file *file,
-	char __user *ubuf, size_t count,
-	loff_t *ppos)
-{
-	u32 num_entries = 0;
-
-	pr_err("\n");
-
-	if (!ipa3_ctx->ipv6ct_mem.dev.is_dev_init) {
-		pr_err("IPv6 Conntrack not initialized or not supported\n");
-		return 0;
-	}
-
-	mutex_lock(&ipa3_ctx->ipv6ct_mem.dev.lock);
-
-	if (!ipa3_ctx->ipv6ct_mem.dev.is_hw_init) {
-		pr_err("IPv6 connection tracking H/W hasn't been initialized\n");
-		goto bail;
-	}
-
-	ipa3_start_read_memory_device(&ipa3_ctx->ipv6ct_mem.dev,
-		IPAHAL_NAT_IPV6CT, &num_entries);
-	ipa3_finish_read_memory_device(&ipa3_ctx->ipv6ct_mem.dev,
-		num_entries);
-
-	IPADBG("return\n");
-bail:
-	mutex_unlock(&ipa3_ctx->ipv6ct_mem.dev.lock);
-	return 0;
-}
-
-static ssize_t ipa3_pm_read_stats(struct file *file, char __user *ubuf,
-		size_t count, loff_t *ppos)
-{
-	int result, cnt = 0;
-
-	result = ipa_pm_stat(dbg_buff, IPA_MAX_MSG_LEN);
-	if (result < 0) {
-		cnt += scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN - cnt,
-				"Error in printing PM stat %d\n", result);
-		goto ret;
-	}
-	cnt += result;
-ret:
-	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, cnt);
-}
-
-static ssize_t ipa3_pm_ex_read_stats(struct file *file, char __user *ubuf,
-		size_t count, loff_t *ppos)
-{
-	int result, cnt = 0;
-
-	result = ipa_pm_exceptions_stat(dbg_buff, IPA_MAX_MSG_LEN);
-	if (result < 0) {
-		cnt += scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN - cnt,
-				"Error in printing PM stat %d\n", result);
-		goto ret;
-	}
-	cnt += result;
-ret:
-	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, cnt);
-}
-
-static ssize_t ipa3_read_ipahal_regs(struct file *file, char __user *ubuf,
-		size_t count, loff_t *ppos)
-{
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-	ipahal_print_all_regs(true);
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-
-	return 0;
-}
-
-static ssize_t ipa3_read_wdi_gsi_stats(struct file *file,
-		char __user *ubuf, size_t count, loff_t *ppos)
-{
-	struct ipa3_uc_dbg_ring_stats stats;
-	int nbytes;
-	int cnt = 0;
-
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_5) {
-		nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
-				"This feature only support on IPA4.5+\n");
-		cnt += nbytes;
-		goto done;
-	}
-
-	if (!ipa3_get_wdi_gsi_stats(&stats)) {
-		nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
-			"TX ringFull=%u\n"
-			"TX ringEmpty=%u\n"
-			"TX ringUsageHigh=%u\n"
-			"TX ringUsageLow=%u\n"
-			"TX RingUtilCount=%u\n",
-			stats.ring[1].ringFull,
-			stats.ring[1].ringEmpty,
-			stats.ring[1].ringUsageHigh,
-			stats.ring[1].ringUsageLow,
-			stats.ring[1].RingUtilCount);
-		cnt += nbytes;
-		nbytes = scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN - cnt,
-			"RX ringFull=%u\n"
-			"RX ringEmpty=%u\n"
-			"RX ringUsageHigh=%u\n"
-			"RX ringUsageLow=%u\n"
-			"RX RingUtilCount=%u\n",
-			stats.ring[0].ringFull,
-			stats.ring[0].ringEmpty,
-			stats.ring[0].ringUsageHigh,
-			stats.ring[0].ringUsageLow,
-			stats.ring[0].RingUtilCount);
-		cnt += nbytes;
-	} else {
-		nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
-				"Fail to read WDI GSI stats\n");
-		cnt += nbytes;
-	}
-done:
-	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, cnt);
-}
-
-static ssize_t ipa3_read_wdi3_gsi_stats(struct file *file,
-		char __user *ubuf, size_t count, loff_t *ppos)
-{
-	struct ipa3_uc_dbg_ring_stats stats;
-	int nbytes;
-	int cnt = 0;
-
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_5) {
-		nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
-				"This feature only support on IPA4.5+\n");
-		cnt += nbytes;
-		goto done;
-	}
-	if (!ipa3_get_wdi3_gsi_stats(&stats)) {
-		nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
-			"TX ringFull=%u\n"
-			"TX ringEmpty=%u\n"
-			"TX ringUsageHigh=%u\n"
-			"TX ringUsageLow=%u\n"
-			"TX RingUtilCount=%u\n",
-			stats.ring[1].ringFull,
-			stats.ring[1].ringEmpty,
-			stats.ring[1].ringUsageHigh,
-			stats.ring[1].ringUsageLow,
-			stats.ring[1].RingUtilCount);
-		cnt += nbytes;
-		nbytes = scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN - cnt,
-			"RX ringFull=%u\n"
-			"RX ringEmpty=%u\n"
-			"RX ringUsageHigh=%u\n"
-			"RX ringUsageLow=%u\n"
-			"RX RingUtilCount=%u\n",
-			stats.ring[0].ringFull,
-			stats.ring[0].ringEmpty,
-			stats.ring[0].ringUsageHigh,
-			stats.ring[0].ringUsageLow,
-			stats.ring[0].RingUtilCount);
-		cnt += nbytes;
-	} else {
-		nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
-				"Fail to read WDI GSI stats\n");
-		cnt += nbytes;
-	}
-
-done:
-	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, cnt);
-}
-
-static ssize_t ipa3_read_11ad_gsi_stats(struct file *file,
-		char __user *ubuf, size_t count, loff_t *ppos)
-{
-	int nbytes;
-	int cnt = 0;
-
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_5) {
-		nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
-				"This feature only support on IPA4.5+\n");
-		cnt += nbytes;
-		goto done;
-	}
-	return 0;
-done:
-	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, cnt);
-}
-
-static ssize_t ipa3_read_aqc_gsi_stats(struct file *file,
-		char __user *ubuf, size_t count, loff_t *ppos)
-{
-	int nbytes;
-	int cnt = 0;
-
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_5) {
-		nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
-				"This feature only support on IPA4.5+\n");
-		cnt += nbytes;
-		goto done;
-	}
-	return 0;
-done:
-	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, cnt);
-}
-
-static ssize_t ipa3_read_mhip_gsi_stats(struct file *file,
-	char __user *ubuf, size_t count, loff_t *ppos)
-{
-	struct ipa3_uc_dbg_ring_stats stats;
-	int nbytes;
-	int cnt = 0;
-
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_5) {
-		nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
-			"This feature only support on IPA4.5+\n");
-		cnt += nbytes;
-		goto done;
-	}
-	if (!ipa3_get_mhip_gsi_stats(&stats)) {
-		nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
-			"IPA_CLIENT_MHI_PRIME_TETH_CONS ringFull=%u\n"
-			"IPA_CLIENT_MHI_PRIME_TETH_CONS ringEmpty=%u\n"
-			"IPA_CLIENT_MHI_PRIME_TETH_CONS ringUsageHigh=%u\n"
-			"IPA_CLIENT_MHI_PRIME_TETH_CONS ringUsageLow=%u\n"
-			"IPA_CLIENT_MHI_PRIME_TETH_CONS RingUtilCount=%u\n",
-			stats.ring[1].ringFull,
-			stats.ring[1].ringEmpty,
-			stats.ring[1].ringUsageHigh,
-			stats.ring[1].ringUsageLow,
-			stats.ring[1].RingUtilCount);
-		cnt += nbytes;
-		nbytes = scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN - cnt,
-			"IPA_CLIENT_MHI_PRIME_TETH_PROD ringFull=%u\n"
-			"IPA_CLIENT_MHI_PRIME_TETH_PROD ringEmpty=%u\n"
-			"IPA_CLIENT_MHI_PRIME_TETH_PROD ringUsageHigh=%u\n"
-			"IPA_CLIENT_MHI_PRIME_TETH_PROD ringUsageLow=%u\n"
-			"IPA_CLIENT_MHI_PRIME_TETH_PROD RingUtilCount=%u\n",
-			stats.ring[0].ringFull,
-			stats.ring[0].ringEmpty,
-			stats.ring[0].ringUsageHigh,
-			stats.ring[0].ringUsageLow,
-			stats.ring[0].RingUtilCount);
-		cnt += nbytes;
-		nbytes = scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN - cnt,
-			"IPA_CLIENT_MHI_PRIME_RMNET_CONS ringFull=%u\n"
-			"IPA_CLIENT_MHI_PRIME_RMNET_CONS ringEmpty=%u\n"
-			"IPA_CLIENT_MHI_PRIME_RMNET_CONS ringUsageHigh=%u\n"
-			"IPA_CLIENT_MHI_PRIME_RMNET_CONS ringUsageLow=%u\n"
-			"IPA_CLIENT_MHI_PRIME_RMNET_CONS RingUtilCount=%u\n",
-			stats.ring[3].ringFull,
-			stats.ring[3].ringEmpty,
-			stats.ring[3].ringUsageHigh,
-			stats.ring[3].ringUsageLow,
-			stats.ring[3].RingUtilCount);
-		cnt += nbytes;
-		nbytes = scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN - cnt,
-			"IPA_CLIENT_MHI_PRIME_RMNET_PROD ringFull=%u\n"
-			"IPA_CLIENT_MHI_PRIME_RMNET_PROD ringEmpty=%u\n"
-			"IPA_CLIENT_MHI_PRIME_RMNET_PROD ringUsageHigh=%u\n"
-			"IPA_CLIENT_MHI_PRIME_RMNET_PROD ringUsageLow=%u\n"
-			"IPA_CLIENT_MHI_PRIME_RMNET_PROD RingUtilCount=%u\n",
-			stats.ring[2].ringFull,
-			stats.ring[2].ringEmpty,
-			stats.ring[2].ringUsageHigh,
-			stats.ring[2].ringUsageLow,
-			stats.ring[2].RingUtilCount);
-		cnt += nbytes;
-	} else {
-		nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
-			"Fail to read WDI GSI stats\n");
-		cnt += nbytes;
-	}
-
-done:
-	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, cnt);
-}
-
-static ssize_t ipa3_read_usb_gsi_stats(struct file *file,
-	char __user *ubuf, size_t count, loff_t *ppos)
-{
-	struct ipa3_uc_dbg_ring_stats stats;
-	int nbytes;
-	int cnt = 0;
-
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_5) {
-		nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
-			"This feature only support on IPA4.5+\n");
-		cnt += nbytes;
-		goto done;
-	}
-	if (!ipa3_get_usb_gsi_stats(&stats)) {
-		nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
-			"TX ringFull=%u\n"
-			"TX ringEmpty=%u\n"
-			"TX ringUsageHigh=%u\n"
-			"TX ringUsageLow=%u\n"
-			"TX RingUtilCount=%u\n",
-			stats.ring[1].ringFull,
-			stats.ring[1].ringEmpty,
-			stats.ring[1].ringUsageHigh,
-			stats.ring[1].ringUsageLow,
-			stats.ring[1].RingUtilCount);
-		cnt += nbytes;
-		nbytes = scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN - cnt,
-			"RX ringFull=%u\n"
-			"RX ringEmpty=%u\n"
-			"RX ringUsageHigh=%u\n"
-			"RX ringUsageLow=%u\n"
-			"RX RingUtilCount=%u\n",
-			stats.ring[0].ringFull,
-			stats.ring[0].ringEmpty,
-			stats.ring[0].ringUsageHigh,
-			stats.ring[0].ringUsageLow,
-			stats.ring[0].RingUtilCount);
-		cnt += nbytes;
-	} else {
-		nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
-			"Fail to read WDI GSI stats\n");
-		cnt += nbytes;
-	}
-
-done:
-	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, cnt);
-}
-
-static void ipa_dump_status(struct ipahal_pkt_status *status)
-{
-	IPA_DUMP_STATUS_FIELD(status_opcode);
-	IPA_DUMP_STATUS_FIELD(exception);
-	IPA_DUMP_STATUS_FIELD(status_mask);
-	IPA_DUMP_STATUS_FIELD(pkt_len);
-	IPA_DUMP_STATUS_FIELD(endp_src_idx);
-	IPA_DUMP_STATUS_FIELD(endp_dest_idx);
-	IPA_DUMP_STATUS_FIELD(metadata);
-	IPA_DUMP_STATUS_FIELD(flt_local);
-	IPA_DUMP_STATUS_FIELD(flt_hash);
-	IPA_DUMP_STATUS_FIELD(flt_global);
-	IPA_DUMP_STATUS_FIELD(flt_ret_hdr);
-	IPA_DUMP_STATUS_FIELD(flt_miss);
-	IPA_DUMP_STATUS_FIELD(flt_rule_id);
-	IPA_DUMP_STATUS_FIELD(rt_local);
-	IPA_DUMP_STATUS_FIELD(rt_hash);
-	IPA_DUMP_STATUS_FIELD(ucp);
-	IPA_DUMP_STATUS_FIELD(rt_tbl_idx);
-	IPA_DUMP_STATUS_FIELD(rt_miss);
-	IPA_DUMP_STATUS_FIELD(rt_rule_id);
-	IPA_DUMP_STATUS_FIELD(nat_hit);
-	IPA_DUMP_STATUS_FIELD(nat_entry_idx);
-	IPA_DUMP_STATUS_FIELD(nat_type);
-	pr_err("tag = 0x%llx\n", (u64)status->tag_info & 0xFFFFFFFFFFFF);
-	IPA_DUMP_STATUS_FIELD(seq_num);
-	IPA_DUMP_STATUS_FIELD(time_of_day_ctr);
-	IPA_DUMP_STATUS_FIELD(hdr_local);
-	IPA_DUMP_STATUS_FIELD(hdr_offset);
-	IPA_DUMP_STATUS_FIELD(frag_hit);
-	IPA_DUMP_STATUS_FIELD(frag_rule);
-}
-
-static ssize_t ipa_status_stats_read(struct file *file, char __user *ubuf,
-		size_t count, loff_t *ppos)
-{
-	struct ipa3_status_stats *stats;
-	int i, j;
-
-	stats = kzalloc(sizeof(*stats), GFP_KERNEL);
-	if (!stats)
-		return -EFAULT;
-
-	for (i = 0; i < ipa3_ctx->ipa_num_pipes; i++) {
-		if (!ipa3_ctx->ep[i].sys || !ipa3_ctx->ep[i].sys->status_stat)
-			continue;
-
-		memcpy(stats, ipa3_ctx->ep[i].sys->status_stat, sizeof(*stats));
-		pr_err("Statuses for pipe %d\n", i);
-		for (j = 0; j < IPA_MAX_STATUS_STAT_NUM; j++) {
-			pr_err("curr=%d\n", stats->curr);
-			ipa_dump_status(&stats->status[stats->curr]);
-			pr_err("\n\n\n");
-			stats->curr = (stats->curr + 1) %
-				IPA_MAX_STATUS_STAT_NUM;
-		}
-	}
-
-	kfree(stats);
-	return 0;
-}
-
-static ssize_t ipa3_print_active_clients_log(struct file *file,
-		char __user *ubuf, size_t count, loff_t *ppos)
-{
-	int cnt;
-	int table_size;
-
-	if (active_clients_buf == NULL) {
-		IPAERR("Active Clients buffer is not allocated");
-		return 0;
-	}
-	memset(active_clients_buf, 0, IPA_DBG_ACTIVE_CLIENT_BUF_SIZE);
-	mutex_lock(&ipa3_ctx->ipa3_active_clients.mutex);
-	cnt = ipa3_active_clients_log_print_buffer(active_clients_buf,
-			IPA_DBG_ACTIVE_CLIENT_BUF_SIZE - IPA_MAX_MSG_LEN);
-	table_size = ipa3_active_clients_log_print_table(active_clients_buf
-			+ cnt, IPA_MAX_MSG_LEN);
-	mutex_unlock(&ipa3_ctx->ipa3_active_clients.mutex);
-
-	return simple_read_from_buffer(ubuf, count, ppos,
-			active_clients_buf, cnt + table_size);
-}
-
-static ssize_t ipa3_clear_active_clients_log(struct file *file,
-		const char __user *ubuf, size_t count, loff_t *ppos)
-{
-	ipa3_active_clients_log_clear();
-
-	return count;
-}
-
-static ssize_t ipa3_enable_ipc_low(struct file *file,
-	const char __user *ubuf, size_t count, loff_t *ppos)
-{
-	s8 option = 0;
-	int ret;
-
-	ret = kstrtos8_from_user(ubuf, count, 0, &option);
-	if (ret)
-		return ret;
-
-	mutex_lock(&ipa3_ctx->lock);
-	if (option) {
-		if (!ipa_ipc_low_buff) {
-			ipa_ipc_low_buff =
-				ipc_log_context_create(IPA_IPC_LOG_PAGES,
-					"ipa_low", 0);
-		}
-			if (ipa_ipc_low_buff == NULL)
-				IPADBG("failed to get logbuf_low\n");
-		ipa3_ctx->logbuf_low = ipa_ipc_low_buff;
-	} else {
-		ipa3_ctx->logbuf_low = NULL;
-	}
-	mutex_unlock(&ipa3_ctx->lock);
-
-	return count;
-}
-
-static const struct ipa3_debugfs_file debugfs_files[] = {
-	{
-		"gen_reg", IPA_READ_ONLY_MODE, NULL, {
-			.read = ipa3_read_gen_reg
-		}
-	}, {
-		"active_clients", IPA_READ_WRITE_MODE, NULL, {
-			.read = ipa3_print_active_clients_log,
-			.write = ipa3_clear_active_clients_log
-		}
-	}, {
-		"ep_reg", IPA_READ_WRITE_MODE, NULL, {
-			.read = ipa3_read_ep_reg,
-			.write = ipa3_write_ep_reg,
-		}
-	}, {
-		"keep_awake", IPA_READ_WRITE_MODE, NULL, {
-			.read = ipa3_read_keep_awake,
-			.write = ipa3_write_keep_awake,
-		}
-	}, {
-		"holb", IPA_WRITE_ONLY_MODE, NULL, {
-			.write = ipa3_write_ep_holb,
-		}
-	}, {
-		"hdr", IPA_READ_ONLY_MODE, NULL, {
-			.read = ipa3_read_hdr,
-		}
-	}, {
-		"proc_ctx", IPA_READ_ONLY_MODE, NULL, {
-			.read = ipa3_read_proc_ctx,
-		}
-	}, {
-		"ip4_rt", IPA_READ_ONLY_MODE, (void *)IPA_IP_v4, {
-			.read = ipa3_read_rt,
-			.open = ipa3_open_dbg,
-		}
-	}, {
-		"ip4_rt_hw", IPA_READ_ONLY_MODE, (void *)IPA_IP_v4, {
-			.read = ipa3_read_rt_hw,
-			.open = ipa3_open_dbg,
-		}
-	}, {
-		"ip6_rt", IPA_READ_ONLY_MODE, (void *)IPA_IP_v6, {
-			.read = ipa3_read_rt,
-			.open = ipa3_open_dbg,
-		}
-	}, {
-		"ip6_rt_hw", IPA_READ_ONLY_MODE, (void *)IPA_IP_v6, {
-			.read = ipa3_read_rt_hw,
-			.open = ipa3_open_dbg,
-		}
-	}, {
-		"ip4_flt", IPA_READ_ONLY_MODE, (void *)IPA_IP_v4, {
-			.read = ipa3_read_flt,
-			.open = ipa3_open_dbg,
-		}
-	}, {
-		"ip4_flt_hw", IPA_READ_ONLY_MODE, (void *)IPA_IP_v4, {
-			.read = ipa3_read_flt_hw,
-			.open = ipa3_open_dbg,
-		}
-	}, {
-		"ip6_flt", IPA_READ_ONLY_MODE, (void *)IPA_IP_v6, {
-			.read = ipa3_read_flt,
-			.open = ipa3_open_dbg,
-		}
-	}, {
-		"ip6_flt_hw", IPA_READ_ONLY_MODE, (void *)IPA_IP_v6, {
-			.read = ipa3_read_flt_hw,
-			.open = ipa3_open_dbg,
-		}
-	}, {
-		"stats", IPA_READ_ONLY_MODE, NULL, {
-			.read = ipa3_read_stats,
-		}
-	}, {
-		"wstats", IPA_READ_ONLY_MODE, NULL, {
-			.read = ipa3_read_wstats,
-		}
-	}, {
-		"odlstats", IPA_READ_ONLY_MODE, NULL, {
-			.read = ipa3_read_odlstats,
-		}
-	}, {
-		"wdi", IPA_READ_ONLY_MODE, NULL, {
-			.read = ipa3_read_wdi,
-		}
-	}, {
-		"ntn", IPA_READ_ONLY_MODE, NULL, {
-			.read = ipa3_read_ntn,
-		}
-	}, {
-		"dbg_cnt", IPA_READ_WRITE_MODE, NULL, {
-			.read = ipa3_read_dbg_cnt,
-			.write = ipa3_write_dbg_cnt,
-		}
-	}, {
-		"msg", IPA_READ_ONLY_MODE, NULL, {
-			.read = ipa3_read_msg,
-		}
-	}, {
-		"ip4_nat", IPA_READ_ONLY_MODE, NULL, {
-			.read = ipa3_read_nat4,
-		}
-	}, {
-		"ipv6ct", IPA_READ_ONLY_MODE, NULL, {
-			.read = ipa3_read_ipv6ct,
-		}
-	}, {
-		"pm_stats", IPA_READ_ONLY_MODE, NULL, {
-			.read = ipa3_pm_read_stats,
-		}
-	}, {
-		"pm_ex_stats", IPA_READ_ONLY_MODE, NULL, {
-			.read = ipa3_pm_ex_read_stats,
-		}
-	}, {
-		"status_stats", IPA_READ_ONLY_MODE, NULL, {
-			.read = ipa_status_stats_read,
-		}
-	}, {
-		"enable_low_prio_print", IPA_WRITE_ONLY_MODE, NULL, {
-			.write = ipa3_enable_ipc_low,
-		}
-	}, {
-		"ipa_dump_regs", IPA_READ_ONLY_MODE, NULL, {
-			.read = ipa3_read_ipahal_regs,
-		}
-	}, {
-		"wdi_gsi_stats", IPA_READ_ONLY_MODE, NULL, {
-			.read = ipa3_read_wdi_gsi_stats,
-		}
-	}, {
-		"wdi3_gsi_stats", IPA_READ_ONLY_MODE, NULL, {
-			.read = ipa3_read_wdi3_gsi_stats,
-		}
-	}, {
-		"11ad_gsi_stats", IPA_READ_ONLY_MODE, NULL, {
-			.read = ipa3_read_11ad_gsi_stats,
-		}
-	}, {
-		"aqc_gsi_stats", IPA_READ_ONLY_MODE, NULL, {
-			.read = ipa3_read_aqc_gsi_stats,
-		}
-	}, {
-		"mhip_gsi_stats", IPA_READ_ONLY_MODE, NULL, {
-			.read = ipa3_read_mhip_gsi_stats,
-		}
-	}, {
-		"usb_gsi_stats", IPA_READ_ONLY_MODE, NULL, {
-			.read = ipa3_read_usb_gsi_stats,
-		}
-	}
-};
-
-void ipa3_debugfs_init(void)
-{
-	const size_t debugfs_files_num =
-		sizeof(debugfs_files) / sizeof(struct ipa3_debugfs_file);
-	size_t i;
-	struct dentry *file;
-
-	dent = debugfs_create_dir("ipa", 0);
-	if (IS_ERR(dent)) {
-		IPAERR("fail to create folder in debug_fs.\n");
-		return;
-	}
-
-	file = debugfs_create_u32("hw_type", IPA_READ_ONLY_MODE,
-		dent, &ipa3_ctx->ipa_hw_type);
-	if (!file) {
-		IPAERR("could not create hw_type file\n");
-		goto fail;
-	}
-
-
-	for (i = 0; i < debugfs_files_num; ++i) {
-		const struct ipa3_debugfs_file *curr = &debugfs_files[i];
-
-		file = debugfs_create_file(curr->name, curr->mode, dent,
-			curr->data, &curr->fops);
-		if (!file || IS_ERR(file)) {
-			IPAERR("fail to create file for debug_fs %s\n",
-				curr->name);
-			goto fail;
-		}
-	}
-
-	active_clients_buf = NULL;
-	active_clients_buf = kzalloc(IPA_DBG_ACTIVE_CLIENT_BUF_SIZE,
-			GFP_KERNEL);
-	if (active_clients_buf == NULL)
-		goto fail;
-
-	file = debugfs_create_u32("enable_clock_scaling", IPA_READ_WRITE_MODE,
-		dent, &ipa3_ctx->enable_clock_scaling);
-	if (!file) {
-		IPAERR("could not create enable_clock_scaling file\n");
-		goto fail;
-	}
-
-	file = debugfs_create_u32("enable_napi_chain", IPA_READ_WRITE_MODE,
-		dent, &ipa3_ctx->enable_napi_chain);
-	if (!file) {
-		IPAERR("could not create enable_napi_chain file\n");
-		goto fail;
-	}
-
-	file = debugfs_create_u32("clock_scaling_bw_threshold_nominal_mbps",
-		IPA_READ_WRITE_MODE, dent,
-		&ipa3_ctx->ctrl->clock_scaling_bw_threshold_nominal);
-	if (!file) {
-		IPAERR("could not create bw_threshold_nominal_mbps\n");
-		goto fail;
-	}
-
-	file = debugfs_create_u32("clock_scaling_bw_threshold_turbo_mbps",
-			IPA_READ_WRITE_MODE, dent,
-			&ipa3_ctx->ctrl->clock_scaling_bw_threshold_turbo);
-	if (!file) {
-		IPAERR("could not create bw_threshold_turbo_mbps\n");
-		goto fail;
-	}
-
-	file = debugfs_create_u32("clk_rate", IPA_READ_ONLY_MODE,
-		dent, &ipa3_ctx->curr_ipa_clk_rate);
-	if (!file) {
-		IPAERR("could not create clk_rate file\n");
-		goto fail;
-	}
-
-	ipa_debugfs_init_stats(dent);
-
-	ipa3_wigig_init_debugfs_i(dent);
-
-	return;
-
-fail:
-	debugfs_remove_recursive(dent);
-}
-
-void ipa3_debugfs_remove(void)
-{
-	if (IS_ERR(dent)) {
-		IPAERR("Debugfs:folder was not created.\n");
-		return;
-	}
-	if (active_clients_buf != NULL) {
-		kfree(active_clients_buf);
-		active_clients_buf = NULL;
-	}
-	debugfs_remove_recursive(dent);
-}
-
-struct dentry *ipa_debugfs_get_root(void)
-{
-	return dent;
-}
-EXPORT_SYMBOL(ipa_debugfs_get_root);
-
-#else /* !CONFIG_DEBUG_FS */
-void ipa3_debugfs_init(void) {}
-void ipa3_debugfs_remove(void) {}
-#endif

ipa/ipa_v3/ipa_defs.h

@@ -1,94 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
- */
-
-#ifndef _IPA_DEFS_H_
-#define _IPA_DEFS_H_
-#include <linux/ipa.h>
-
-/**
- * struct ipa_rt_rule_i - attributes of a routing rule
- * @dst: dst "client"
- * @hdr_hdl: handle to the dynamic header
-	it is not an index or an offset
- * @hdr_proc_ctx_hdl: handle to header processing context. if it is provided
-	hdr_hdl shall be 0
- * @attrib: attributes of the rule
- * @max_prio: bool switch. is this rule with Max priority? meaning on rule hit,
- *  IPA will use the rule and will not look for other rules that may have
- *  higher priority
- * @hashable: bool switch. is this rule hashable or not?
- *  ipa uses hashable rules to cache their hit results to be used in
- *  consecutive packets
- * @retain_hdr: bool switch to instruct IPA core to add back to the packet
- *  the header removed as part of header removal
- * @coalesce: bool to decide whether packets should be coalesced or not
- * @enable_stats: is true when we want to enable stats for this
- * rt rule.
- * @cnt_idx: if enable_stats is 1 and cnt_idx is 0, then cnt_idx
- * will be assigned by ipa driver.
- */
-struct ipa_rt_rule_i {
-	enum ipa_client_type dst;
-	u32 hdr_hdl;
-	u32 hdr_proc_ctx_hdl;
-	struct ipa_rule_attrib attrib;
-	u8 max_prio;
-	u8 hashable;
-	u8 retain_hdr;
-	u8 coalesce;
-	u8 enable_stats;
-	u8 cnt_idx;
-};
-
-/**
- * struct ipa_flt_rule_i - attributes of a filtering rule
- * @retain_hdr: bool switch to instruct IPA core to add back to the packet
- *  the header removed as part of header removal
- * @to_uc: bool switch to pass packet to micro-controller
- * @action: action field
- * @rt_tbl_hdl: handle of table from "get"
- * @attrib: attributes of the rule
- * @eq_attrib: attributes of the rule in equation form (valid when
- * eq_attrib_type is true)
- * @rt_tbl_idx: index of RT table referred to by filter rule (valid when
- * eq_attrib_type is true and non-exception action)
- * @eq_attrib_type: true if equation level form used to specify attributes
- * @max_prio: bool switch. is this rule with Max priority? meaning on rule hit,
- *  IPA will use the rule and will not look for other rules that may have
- *  higher priority
- * @hashable: bool switch. is this rule hashable or not?
- *  ipa uses hashable rules to cache their hit results to be used in
- *  consecutive packets
- * @rule_id: rule_id to be assigned to the filter rule. In case client specifies
- *  rule_id as 0 the driver will assign a new rule_id
- * @set_metadata: bool switch. should metadata replacement at the NAT block
- *  take place?
- * @pdn_idx: if action is "pass to source\destination NAT" then a comparison
- * against the PDN index in the matching PDN entry will take place as an
- * additional condition for NAT hit.
- * @enable_stats: is true when we want to enable stats for this
- * flt rule.
- * @cnt_idx: if 0 means disable, otherwise use for index.
- * will be assigned by ipa driver.
- */
-struct ipa_flt_rule_i {
-	u8 retain_hdr;
-	u8 to_uc;
-	enum ipa_flt_action action;
-	u32 rt_tbl_hdl;
-	struct ipa_rule_attrib attrib;
-	struct ipa_ipfltri_rule_eq eq_attrib;
-	u32 rt_tbl_idx;
-	u8 eq_attrib_type;
-	u8 max_prio;
-	u8 hashable;
-	u16 rule_id;
-	u8 set_metadata;
-	u8 pdn_idx;
-	u8 enable_stats;
-	u8 cnt_idx;
-};
-
-#endif /* _IPA_DEFS_H_ */

ipa/ipa_v3/ipa_dma.c

@@ -1,1243 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2015-2019, The Linux Foundation. All rights reserved.
- */
-
-
-#include <linux/debugfs.h>
-#include <linux/export.h>
-#include <linux/delay.h>
-#include <linux/kernel.h>
-#include <linux/msm_ipa.h>
-#include <linux/mutex.h>
-#include <linux/ipa.h>
-#include "linux/msm_gsi.h"
-#include <linux/dmapool.h>
-#include "ipa_i.h"
-
-#define IPA_DMA_POLLING_MIN_SLEEP_RX 1010
-#define IPA_DMA_POLLING_MAX_SLEEP_RX 1050
-#define IPA_DMA_SYS_DESC_MAX_FIFO_SZ 0x7FF8
-#define IPA_DMA_MAX_PKT_SZ 0xFFFF
-#define IPA_DMA_DUMMY_BUFF_SZ 8
-#define IPA_DMA_PREFETCH_WA_THRESHOLD 9
-
-#define IPADMA_DRV_NAME "ipa_dma"
-
-#define IPADMA_DBG(fmt, args...) \
-	do { \
-		pr_debug(IPADMA_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-			IPADMA_DRV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			IPADMA_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-#define IPADMA_DBG_LOW(fmt, args...) \
-	do { \
-		pr_debug(IPADMA_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			IPADMA_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-#define IPADMA_ERR(fmt, args...) \
-	do { \
-		pr_err(IPADMA_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-			IPADMA_DRV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			IPADMA_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-#define IPADMA_FUNC_ENTRY() \
-	IPADMA_DBG_LOW("ENTRY\n")
-
-#define IPADMA_FUNC_EXIT() \
-	IPADMA_DBG_LOW("EXIT\n")
-
-#ifdef CONFIG_DEBUG_FS
-#define IPADMA_MAX_MSG_LEN 1024
-static char dbg_buff[IPADMA_MAX_MSG_LEN];
-static void ipa3_dma_debugfs_init(void);
-static void ipa3_dma_debugfs_destroy(void);
-#else
-static void ipa3_dma_debugfs_init(void) {}
-static void ipa3_dma_debugfs_destroy(void) {}
-#endif
-
-/**
- * struct ipa3_dma_ctx -IPADMA driver context information
- * @enable_ref_cnt: ipa dma enable reference count
- * @destroy_pending: destroy ipa_dma after handling all pending memcpy
- * @ipa_dma_xfer_wrapper_cache: cache of ipa3_dma_xfer_wrapper structs
- * @sync_lock: lock for synchronisation in sync_memcpy
- * @async_lock: lock for synchronisation in async_memcpy
- * @enable_lock: lock for is_enabled
- * @pending_lock: lock for synchronize is_enable and pending_cnt
- * @done: no pending works-ipadma can be destroyed
- * @ipa_dma_sync_prod_hdl: handle of sync memcpy producer
- * @ipa_dma_async_prod_hdl:handle of async memcpy producer
- * @ipa_dma_sync_cons_hdl: handle of sync memcpy consumer
- * @sync_memcpy_pending_cnt: number of pending sync memcopy operations
- * @async_memcpy_pending_cnt: number of pending async memcopy operations
- * @uc_memcpy_pending_cnt: number of pending uc memcopy operations
- * @total_sync_memcpy: total number of sync memcpy (statistics)
- * @total_async_memcpy: total number of async memcpy (statistics)
- * @total_uc_memcpy: total number of uc memcpy (statistics)
- */
-struct ipa3_dma_ctx {
-	unsigned int enable_ref_cnt;
-	bool destroy_pending;
-	struct kmem_cache *ipa_dma_xfer_wrapper_cache;
-	struct mutex sync_lock;
-	spinlock_t async_lock;
-	struct mutex enable_lock;
-	spinlock_t pending_lock;
-	struct completion done;
-	u32 ipa_dma_sync_prod_hdl;
-	u32 ipa_dma_async_prod_hdl;
-	u32 ipa_dma_sync_cons_hdl;
-	u32 ipa_dma_async_cons_hdl;
-	atomic_t sync_memcpy_pending_cnt;
-	atomic_t async_memcpy_pending_cnt;
-	atomic_t uc_memcpy_pending_cnt;
-	atomic_t total_sync_memcpy;
-	atomic_t total_async_memcpy;
-	atomic_t total_uc_memcpy;
-	struct ipa_mem_buffer ipa_dma_dummy_src_sync;
-	struct ipa_mem_buffer ipa_dma_dummy_dst_sync;
-	struct ipa_mem_buffer ipa_dma_dummy_src_async;
-	struct ipa_mem_buffer ipa_dma_dummy_dst_async;
-};
-static struct ipa3_dma_ctx *ipa3_dma_ctx;
-
-/**
- * struct ipa3_dma_init_refcnt_ctrl -IPADMA driver init control information
- * @ref_cnt: reference count for initialization operations
- * @lock: lock for the reference count
- */
-struct ipa3_dma_init_refcnt_ctrl {
-	unsigned int ref_cnt;
-	struct mutex lock;
-};
-static struct ipa3_dma_init_refcnt_ctrl *ipa3_dma_init_refcnt_ctrl;
-
-/**
- * ipa3_dma_setup() - One time setup for IPA DMA
- *
- * This function should be called once to setup ipa dma
- *  by creating the init reference count controller
- *
- * Return codes: 0: success
- *		 Negative value: failure
- */
-int ipa3_dma_setup(void)
-{
-	IPADMA_FUNC_ENTRY();
-
-	if (ipa3_dma_init_refcnt_ctrl) {
-		IPADMA_ERR("Setup already done\n");
-		return -EFAULT;
-	}
-
-	ipa3_dma_init_refcnt_ctrl =
-		kzalloc(sizeof(*(ipa3_dma_init_refcnt_ctrl)), GFP_KERNEL);
-
-	if (!ipa3_dma_init_refcnt_ctrl) {
-		IPADMA_ERR("kzalloc error.\n");
-		return -ENOMEM;
-	}
-
-	mutex_init(&ipa3_dma_init_refcnt_ctrl->lock);
-
-	IPADMA_FUNC_EXIT();
-	return 0;
-}
-
-/**
- * ipa3_dma_shutdown() - Clear setup operations.
- *
- * Cleanup for the setup function.
- * Should be called during IPA driver unloading.
- * It assumes all ipa_dma operations are done and ipa_dma is destroyed.
- *
- * Return codes: None.
- */
-void ipa3_dma_shutdown(void)
-{
-	IPADMA_FUNC_ENTRY();
-
-	if (!ipa3_dma_init_refcnt_ctrl)
-		return;
-
-	kfree(ipa3_dma_init_refcnt_ctrl);
-	ipa3_dma_init_refcnt_ctrl = NULL;
-
-	IPADMA_FUNC_EXIT();
-}
-
-/**
- * ipa3_dma_init() -Initialize IPADMA.
- *
- * This function initialize all IPADMA internal data and connect in dma:
- *	MEMCPY_DMA_SYNC_PROD ->MEMCPY_DMA_SYNC_CONS
- *	MEMCPY_DMA_ASYNC_PROD->MEMCPY_DMA_SYNC_CONS
- *
- * Can be executed several times (re-entrant)
- *
- * Return codes: 0: success
- *		-EFAULT: Mismatch between context existence and init ref_cnt
- *		-EINVAL: IPA driver is not initialized
- *		-ENOMEM: allocating memory error
- *		-EPERM: pipe connection failed
- */
-int ipa3_dma_init(void)
-{
-	struct ipa3_dma_ctx *ipa_dma_ctx_t;
-	struct ipa_sys_connect_params sys_in;
-	int res = 0;
-	int sync_sz;
-	int async_sz;
-
-	IPADMA_FUNC_ENTRY();
-
-	if (!ipa3_dma_init_refcnt_ctrl) {
-		IPADMA_ERR("Setup isn't done yet!\n");
-		return -EINVAL;
-	}
-
-	mutex_lock(&ipa3_dma_init_refcnt_ctrl->lock);
-	if (ipa3_dma_init_refcnt_ctrl->ref_cnt > 0) {
-		IPADMA_DBG("Already initialized refcnt=%d\n",
-			ipa3_dma_init_refcnt_ctrl->ref_cnt);
-		if (!ipa3_dma_ctx) {
-			IPADMA_ERR("Context missing. refcnt=%d\n",
-				ipa3_dma_init_refcnt_ctrl->ref_cnt);
-			res = -EFAULT;
-		} else {
-			ipa3_dma_init_refcnt_ctrl->ref_cnt++;
-		}
-		goto init_unlock;
-	}
-
-	if (ipa3_dma_ctx) {
-		IPADMA_ERR("Context already exist\n");
-		res = -EFAULT;
-		goto init_unlock;
-	}
-
-	if (!ipa3_is_ready()) {
-		IPADMA_ERR("IPA is not ready yet\n");
-		res = -EINVAL;
-		goto init_unlock;
-	}
-
-	ipa_dma_ctx_t = kzalloc(sizeof(*(ipa3_dma_ctx)), GFP_KERNEL);
-
-	if (!ipa_dma_ctx_t) {
-		res = -ENOMEM;
-		goto init_unlock;
-	}
-
-	ipa_dma_ctx_t->ipa_dma_xfer_wrapper_cache =
-		kmem_cache_create("IPA DMA XFER WRAPPER",
-			sizeof(struct ipa3_dma_xfer_wrapper), 0, 0, NULL);
-	if (!ipa_dma_ctx_t->ipa_dma_xfer_wrapper_cache) {
-		IPAERR(":failed to create ipa dma xfer wrapper cache.\n");
-		res = -ENOMEM;
-		goto fail_mem_ctrl;
-	}
-
-	mutex_init(&ipa_dma_ctx_t->enable_lock);
-	spin_lock_init(&ipa_dma_ctx_t->async_lock);
-	mutex_init(&ipa_dma_ctx_t->sync_lock);
-	spin_lock_init(&ipa_dma_ctx_t->pending_lock);
-	init_completion(&ipa_dma_ctx_t->done);
-	ipa_dma_ctx_t->enable_ref_cnt = 0;
-	ipa_dma_ctx_t->destroy_pending = false;
-	atomic_set(&ipa_dma_ctx_t->async_memcpy_pending_cnt, 0);
-	atomic_set(&ipa_dma_ctx_t->sync_memcpy_pending_cnt, 0);
-	atomic_set(&ipa_dma_ctx_t->uc_memcpy_pending_cnt, 0);
-	atomic_set(&ipa_dma_ctx_t->total_async_memcpy, 0);
-	atomic_set(&ipa_dma_ctx_t->total_sync_memcpy, 0);
-	atomic_set(&ipa_dma_ctx_t->total_uc_memcpy, 0);
-
-	sync_sz = IPA_SYS_DESC_FIFO_SZ;
-	async_sz = IPA_DMA_SYS_DESC_MAX_FIFO_SZ;
-	/*
-	 * for ipav3.5 we need to double the rings and allocate dummy buffers
-	 * in order to apply the prefetch WA
-	 */
-	if (ipa_get_hw_type() == IPA_HW_v3_5) {
-		sync_sz *= 2;
-		async_sz *= 2;
-
-		ipa_dma_ctx_t->ipa_dma_dummy_src_sync.base =
-			dma_alloc_coherent(ipa3_ctx->pdev,
-			IPA_DMA_DUMMY_BUFF_SZ * 4,
-			&ipa_dma_ctx_t->ipa_dma_dummy_src_sync.phys_base,
-			GFP_KERNEL);
-
-		if (!ipa_dma_ctx_t->ipa_dma_dummy_src_sync.base) {
-			IPAERR("DMA alloc fail %d bytes for prefetch WA\n",
-				IPA_DMA_DUMMY_BUFF_SZ);
-			res = -ENOMEM;
-			goto fail_alloc_dummy;
-		}
-
-		ipa_dma_ctx_t->ipa_dma_dummy_dst_sync.base =
-			ipa_dma_ctx_t->ipa_dma_dummy_src_sync.base +
-			IPA_DMA_DUMMY_BUFF_SZ;
-		ipa_dma_ctx_t->ipa_dma_dummy_dst_sync.phys_base =
-			ipa_dma_ctx_t->ipa_dma_dummy_src_sync.phys_base +
-			IPA_DMA_DUMMY_BUFF_SZ;
-		ipa_dma_ctx_t->ipa_dma_dummy_src_async.base =
-			ipa_dma_ctx_t->ipa_dma_dummy_dst_sync.base +
-			IPA_DMA_DUMMY_BUFF_SZ;
-		ipa_dma_ctx_t->ipa_dma_dummy_src_async.phys_base =
-			ipa_dma_ctx_t->ipa_dma_dummy_dst_sync.phys_base +
-			IPA_DMA_DUMMY_BUFF_SZ;
-		ipa_dma_ctx_t->ipa_dma_dummy_dst_async.base =
-			ipa_dma_ctx_t->ipa_dma_dummy_src_async.base +
-			IPA_DMA_DUMMY_BUFF_SZ;
-		ipa_dma_ctx_t->ipa_dma_dummy_dst_async.phys_base =
-			ipa_dma_ctx_t->ipa_dma_dummy_src_async.phys_base +
-			IPA_DMA_DUMMY_BUFF_SZ;
-	}
-
-	/* IPADMA SYNC PROD-source for sync memcpy */
-	memset(&sys_in, 0, sizeof(struct ipa_sys_connect_params));
-	sys_in.client = IPA_CLIENT_MEMCPY_DMA_SYNC_PROD;
-	sys_in.desc_fifo_sz = sync_sz;
-	sys_in.ipa_ep_cfg.mode.mode = IPA_DMA;
-	sys_in.ipa_ep_cfg.mode.dst = IPA_CLIENT_MEMCPY_DMA_SYNC_CONS;
-	sys_in.skip_ep_cfg = false;
-	if (ipa3_setup_sys_pipe(&sys_in,
-		&ipa_dma_ctx_t->ipa_dma_sync_prod_hdl)) {
-		IPADMA_ERR(":setup sync prod pipe failed\n");
-		res = -EPERM;
-		goto fail_sync_prod;
-	}
-
-	/* IPADMA SYNC CONS-destination for sync memcpy */
-	memset(&sys_in, 0, sizeof(struct ipa_sys_connect_params));
-	sys_in.client = IPA_CLIENT_MEMCPY_DMA_SYNC_CONS;
-	sys_in.desc_fifo_sz = sync_sz;
-	sys_in.skip_ep_cfg = false;
-	sys_in.ipa_ep_cfg.mode.mode = IPA_BASIC;
-	sys_in.notify = NULL;
-	sys_in.priv = NULL;
-	if (ipa3_setup_sys_pipe(&sys_in,
-		&ipa_dma_ctx_t->ipa_dma_sync_cons_hdl)) {
-		IPADMA_ERR(":setup sync cons pipe failed.\n");
-		res = -EPERM;
-		goto fail_sync_cons;
-	}
-
-	IPADMA_DBG("SYNC MEMCPY pipes are connected\n");
-
-	/* IPADMA ASYNC PROD-source for sync memcpy */
-	memset(&sys_in, 0, sizeof(struct ipa_sys_connect_params));
-	sys_in.client = IPA_CLIENT_MEMCPY_DMA_ASYNC_PROD;
-	sys_in.desc_fifo_sz = async_sz;
-	sys_in.ipa_ep_cfg.mode.mode = IPA_DMA;
-	sys_in.ipa_ep_cfg.mode.dst = IPA_CLIENT_MEMCPY_DMA_ASYNC_CONS;
-	sys_in.skip_ep_cfg = false;
-	sys_in.notify = NULL;
-	if (ipa3_setup_sys_pipe(&sys_in,
-		&ipa_dma_ctx_t->ipa_dma_async_prod_hdl)) {
-		IPADMA_ERR(":setup async prod pipe failed.\n");
-		res = -EPERM;
-		goto fail_async_prod;
-	}
-
-	/* IPADMA ASYNC CONS-destination for sync memcpy */
-	memset(&sys_in, 0, sizeof(struct ipa_sys_connect_params));
-	sys_in.client = IPA_CLIENT_MEMCPY_DMA_ASYNC_CONS;
-	sys_in.desc_fifo_sz = async_sz;
-	sys_in.skip_ep_cfg = false;
-	sys_in.ipa_ep_cfg.mode.mode = IPA_BASIC;
-	sys_in.notify = ipa3_dma_async_memcpy_notify_cb;
-	sys_in.priv = NULL;
-	if (ipa3_setup_sys_pipe(&sys_in,
-		&ipa_dma_ctx_t->ipa_dma_async_cons_hdl)) {
-		IPADMA_ERR(":setup async cons pipe failed.\n");
-		res = -EPERM;
-		goto fail_async_cons;
-	}
-	ipa3_dma_debugfs_init();
-	ipa3_dma_ctx = ipa_dma_ctx_t;
-	ipa3_dma_init_refcnt_ctrl->ref_cnt = 1;
-	IPADMA_DBG("ASYNC MEMCPY pipes are connected\n");
-
-	IPADMA_FUNC_EXIT();
-	goto init_unlock;
-
-fail_async_cons:
-	ipa3_teardown_sys_pipe(ipa_dma_ctx_t->ipa_dma_async_prod_hdl);
-fail_async_prod:
-	ipa3_teardown_sys_pipe(ipa_dma_ctx_t->ipa_dma_sync_cons_hdl);
-fail_sync_cons:
-	ipa3_teardown_sys_pipe(ipa_dma_ctx_t->ipa_dma_sync_prod_hdl);
-fail_sync_prod:
-	dma_free_coherent(ipa3_ctx->pdev, IPA_DMA_DUMMY_BUFF_SZ * 4,
-		ipa_dma_ctx_t->ipa_dma_dummy_src_sync.base,
-		ipa_dma_ctx_t->ipa_dma_dummy_src_sync.phys_base);
-fail_alloc_dummy:
-	kmem_cache_destroy(ipa_dma_ctx_t->ipa_dma_xfer_wrapper_cache);
-fail_mem_ctrl:
-	kfree(ipa_dma_ctx_t);
-	ipa3_dma_ctx = NULL;
-init_unlock:
-	mutex_unlock(&ipa3_dma_init_refcnt_ctrl->lock);
-	return res;
-
-}
-
-/**
- * ipa3_dma_enable() -Vote for IPA clocks.
- *
- * Can be executed several times (re-entrant)
- *
- *Return codes: 0: success
- *		-EINVAL: IPADMA is not initialized
- */
-int ipa3_dma_enable(void)
-{
-	IPADMA_FUNC_ENTRY();
-	if ((ipa3_dma_ctx == NULL) ||
-		(ipa3_dma_init_refcnt_ctrl->ref_cnt < 1)) {
-		IPADMA_ERR("IPADMA isn't initialized, can't enable\n");
-		return -EINVAL;
-	}
-	mutex_lock(&ipa3_dma_ctx->enable_lock);
-	if (ipa3_dma_ctx->enable_ref_cnt > 0) {
-		IPADMA_ERR("Already enabled refcnt=%d\n",
-			ipa3_dma_ctx->enable_ref_cnt);
-		ipa3_dma_ctx->enable_ref_cnt++;
-		mutex_unlock(&ipa3_dma_ctx->enable_lock);
-		return 0;
-	}
-	IPA_ACTIVE_CLIENTS_INC_SPECIAL("DMA");
-	ipa3_dma_ctx->enable_ref_cnt = 1;
-	mutex_unlock(&ipa3_dma_ctx->enable_lock);
-
-	IPADMA_FUNC_EXIT();
-	return 0;
-}
-
-static bool ipa3_dma_work_pending(void)
-{
-	if (atomic_read(&ipa3_dma_ctx->sync_memcpy_pending_cnt)) {
-		IPADMA_DBG("pending sync\n");
-		return true;
-	}
-	if (atomic_read(&ipa3_dma_ctx->async_memcpy_pending_cnt)) {
-		IPADMA_DBG("pending async\n");
-		return true;
-	}
-	if (atomic_read(&ipa3_dma_ctx->uc_memcpy_pending_cnt)) {
-		IPADMA_DBG("pending uc\n");
-		return true;
-	}
-	IPADMA_DBG_LOW("no pending work\n");
-	return false;
-}
-
-/**
- * ipa3_dma_disable()- Unvote for IPA clocks.
- *
- * enter to power save mode.
- *
- * Return codes: 0: success
- *		-EINVAL: IPADMA is not initialized
- *		-EPERM: Operation not permitted as ipa_dma is already
- *			diabled
- *		-EFAULT: can not disable ipa_dma as there are pending
- *			memcopy works
- */
-int ipa3_dma_disable(void)
-{
-	unsigned long flags;
-	int res = 0;
-	bool dec_clks = false;
-
-	IPADMA_FUNC_ENTRY();
-	if ((ipa3_dma_ctx == NULL) ||
-		(ipa3_dma_init_refcnt_ctrl->ref_cnt < 1)) {
-		IPADMA_ERR("IPADMA isn't initialized, can't disable\n");
-		return -EINVAL;
-	}
-	mutex_lock(&ipa3_dma_ctx->enable_lock);
-	spin_lock_irqsave(&ipa3_dma_ctx->pending_lock, flags);
-	if (ipa3_dma_ctx->enable_ref_cnt > 1) {
-		IPADMA_DBG("Multiple enablement done. refcnt=%d\n",
-			ipa3_dma_ctx->enable_ref_cnt);
-		ipa3_dma_ctx->enable_ref_cnt--;
-		goto completed;
-	}
-
-	if (ipa3_dma_ctx->enable_ref_cnt == 0) {
-		IPADMA_ERR("Already disabled\n");
-		res = -EPERM;
-		goto completed;
-	}
-
-	if (ipa3_dma_work_pending()) {
-		IPADMA_ERR("There is pending work, can't disable.\n");
-		res = -EFAULT;
-		goto completed;
-	}
-	ipa3_dma_ctx->enable_ref_cnt = 0;
-	dec_clks = true;
-	IPADMA_FUNC_EXIT();
-
-completed:
-	spin_unlock_irqrestore(&ipa3_dma_ctx->pending_lock, flags);
-	if (dec_clks)
-		IPA_ACTIVE_CLIENTS_DEC_SPECIAL("DMA");
-	mutex_unlock(&ipa3_dma_ctx->enable_lock);
-	return res;
-}
-
-/**
- * ipa3_dma_sync_memcpy()- Perform synchronous memcpy using IPA.
- *
- * @dest: physical address to store the copied data.
- * @src: physical address of the source data to copy.
- * @len: number of bytes to copy.
- *
- * Return codes: 0: success
- *		-EINVAL: invalid params
- *		-EPERM: operation not permitted as ipa_dma isn't enable or
- *			initialized
- *		-gsi_status : on GSI failures
- *		-EFAULT: other
- */
-int ipa3_dma_sync_memcpy(u64 dest, u64 src, int len)
-{
-	int ep_idx;
-	int res;
-	int i = 0;
-	struct ipa3_sys_context *cons_sys;
-	struct ipa3_sys_context *prod_sys;
-	struct ipa3_dma_xfer_wrapper *xfer_descr = NULL;
-	struct ipa3_dma_xfer_wrapper *head_descr = NULL;
-	struct gsi_xfer_elem prod_xfer_elem;
-	struct gsi_xfer_elem cons_xfer_elem;
-	struct gsi_chan_xfer_notify gsi_notify;
-	unsigned long flags;
-	bool stop_polling = false;
-	bool prefetch_wa = false;
-
-	IPADMA_FUNC_ENTRY();
-	IPADMA_DBG_LOW("dest =  0x%llx, src = 0x%llx, len = %d\n",
-		dest, src, len);
-	if (ipa3_dma_ctx == NULL) {
-		IPADMA_ERR("IPADMA isn't initialized, can't memcpy\n");
-		return -EPERM;
-	}
-	if ((max(src, dest) - min(src, dest)) < len) {
-		IPADMA_ERR("invalid addresses - overlapping buffers\n");
-		return -EINVAL;
-	}
-	if (len > IPA_DMA_MAX_PKT_SZ || len <= 0) {
-		IPADMA_ERR("invalid len, %d\n", len);
-		return	-EINVAL;
-	}
-	spin_lock_irqsave(&ipa3_dma_ctx->pending_lock, flags);
-	if (!ipa3_dma_ctx->enable_ref_cnt) {
-		IPADMA_ERR("can't memcpy, IPADMA isn't enabled\n");
-		spin_unlock_irqrestore(&ipa3_dma_ctx->pending_lock, flags);
-		return -EPERM;
-	}
-	atomic_inc(&ipa3_dma_ctx->sync_memcpy_pending_cnt);
-	spin_unlock_irqrestore(&ipa3_dma_ctx->pending_lock, flags);
-
-	ep_idx = ipa3_get_ep_mapping(IPA_CLIENT_MEMCPY_DMA_SYNC_CONS);
-	if (-1 == ep_idx) {
-		IPADMA_ERR("Client %u is not mapped\n",
-			IPA_CLIENT_MEMCPY_DMA_SYNC_CONS);
-		return -EFAULT;
-	}
-	cons_sys = ipa3_ctx->ep[ep_idx].sys;
-
-	ep_idx = ipa3_get_ep_mapping(IPA_CLIENT_MEMCPY_DMA_SYNC_PROD);
-	if (-1 == ep_idx) {
-		IPADMA_ERR("Client %u is not mapped\n",
-			IPA_CLIENT_MEMCPY_DMA_SYNC_PROD);
-		return -EFAULT;
-	}
-	prod_sys = ipa3_ctx->ep[ep_idx].sys;
-
-	xfer_descr = kmem_cache_zalloc(ipa3_dma_ctx->ipa_dma_xfer_wrapper_cache,
-					GFP_KERNEL);
-	if (!xfer_descr) {
-		IPADMA_ERR("failed to alloc xfer descr wrapper\n");
-		res = -ENOMEM;
-		goto fail_mem_alloc;
-	}
-	xfer_descr->phys_addr_dest = dest;
-	xfer_descr->phys_addr_src = src;
-	xfer_descr->len = len;
-	init_completion(&xfer_descr->xfer_done);
-
-	mutex_lock(&ipa3_dma_ctx->sync_lock);
-	list_add_tail(&xfer_descr->link, &cons_sys->head_desc_list);
-	cons_sys->len++;
-	cons_xfer_elem.addr = dest;
-	cons_xfer_elem.len = len;
-	cons_xfer_elem.type = GSI_XFER_ELEM_DATA;
-	cons_xfer_elem.flags = GSI_XFER_FLAG_EOT;
-
-	prod_xfer_elem.addr = src;
-	prod_xfer_elem.len = len;
-	prod_xfer_elem.type = GSI_XFER_ELEM_DATA;
-	prod_xfer_elem.xfer_user_data = NULL;
-
-	/*
-	 * when copy is less than 9B we need to chain another dummy
-	 * copy so the total size will be larger (for ipav3.5)
-	 * for the consumer we have to prepare an additional credit
-	 */
-	prefetch_wa = ((ipa_get_hw_type() == IPA_HW_v3_5) &&
-		len < IPA_DMA_PREFETCH_WA_THRESHOLD);
-	if (prefetch_wa) {
-		cons_xfer_elem.xfer_user_data = NULL;
-		res = gsi_queue_xfer(cons_sys->ep->gsi_chan_hdl, 1,
-			&cons_xfer_elem, false);
-		if (res) {
-			IPADMA_ERR(
-				"Failed: gsi_queue_xfer dest descr res:%d\n",
-				res);
-			goto fail_send;
-		}
-		cons_xfer_elem.addr =
-			ipa3_dma_ctx->ipa_dma_dummy_dst_sync.phys_base;
-		cons_xfer_elem.len = IPA_DMA_DUMMY_BUFF_SZ;
-		cons_xfer_elem.type = GSI_XFER_ELEM_DATA;
-		cons_xfer_elem.flags = GSI_XFER_FLAG_EOT;
-		cons_xfer_elem.xfer_user_data = xfer_descr;
-		res = gsi_queue_xfer(cons_sys->ep->gsi_chan_hdl, 1,
-			&cons_xfer_elem, true);
-		if (res) {
-			IPADMA_ERR(
-				"Failed: gsi_queue_xfer dummy dest descr res:%d\n",
-				res);
-			goto fail_send;
-		}
-		prod_xfer_elem.flags = GSI_XFER_FLAG_CHAIN;
-		res = gsi_queue_xfer(prod_sys->ep->gsi_chan_hdl, 1,
-			&prod_xfer_elem, false);
-		if (res) {
-			IPADMA_ERR(
-				"Failed: gsi_queue_xfer src descr res:%d\n",
-				res);
-			ipa_assert();
-			goto fail_send;
-		}
-		prod_xfer_elem.addr =
-			ipa3_dma_ctx->ipa_dma_dummy_src_sync.phys_base;
-		prod_xfer_elem.len = IPA_DMA_DUMMY_BUFF_SZ;
-		prod_xfer_elem.type = GSI_XFER_ELEM_DATA;
-		prod_xfer_elem.flags = GSI_XFER_FLAG_EOT;
-		prod_xfer_elem.xfer_user_data = NULL;
-		res = gsi_queue_xfer(prod_sys->ep->gsi_chan_hdl, 1,
-				&prod_xfer_elem, true);
-		if (res) {
-			IPADMA_ERR(
-					"Failed: gsi_queue_xfer dummy src descr res:%d\n",
-					res);
-				ipa_assert();
-				goto fail_send;
-			}
-	} else {
-		cons_xfer_elem.xfer_user_data = xfer_descr;
-		res = gsi_queue_xfer(cons_sys->ep->gsi_chan_hdl, 1,
-			&cons_xfer_elem, true);
-		if (res) {
-			IPADMA_ERR(
-				"Failed: gsi_queue_xfer dest descr res:%d\n",
-				res);
-			goto fail_send;
-		}
-		prod_xfer_elem.flags = GSI_XFER_FLAG_EOT;
-		res = gsi_queue_xfer(prod_sys->ep->gsi_chan_hdl, 1,
-			&prod_xfer_elem, true);
-		if (res) {
-			IPADMA_ERR(
-			"Failed: gsi_queue_xfer src descr res:%d\n",
-			 res);
-			ipa_assert();
-			goto fail_send;
-		}
-	}
-	head_descr = list_first_entry(&cons_sys->head_desc_list,
-				struct ipa3_dma_xfer_wrapper, link);
-
-	/* in case we are not the head of the list, wait for head to wake us */
-	if (xfer_descr != head_descr) {
-		mutex_unlock(&ipa3_dma_ctx->sync_lock);
-		wait_for_completion(&xfer_descr->xfer_done);
-		mutex_lock(&ipa3_dma_ctx->sync_lock);
-		head_descr = list_first_entry(&cons_sys->head_desc_list,
-					struct ipa3_dma_xfer_wrapper, link);
-		/* Unexpected transfer sent from HW */
-		ipa_assert_on(xfer_descr != head_descr);
-	}
-	mutex_unlock(&ipa3_dma_ctx->sync_lock);
-
-	do {
-		/* wait for transfer to complete */
-		res = gsi_poll_channel(cons_sys->ep->gsi_chan_hdl,
-			&gsi_notify);
-		if (res == GSI_STATUS_SUCCESS)
-			stop_polling = true;
-		else if (res != GSI_STATUS_POLL_EMPTY)
-			IPADMA_ERR(
-				"Failed: gsi_poll_chanel, returned %d loop#:%d\n",
-				res, i);
-		usleep_range(IPA_DMA_POLLING_MIN_SLEEP_RX,
-			IPA_DMA_POLLING_MAX_SLEEP_RX);
-		i++;
-	} while (!stop_polling);
-
-	/* for prefetch WA we will receive the length of the dummy
-	 * transfer in the event (because it is the second element)
-	 */
-	if (prefetch_wa)
-		ipa_assert_on(gsi_notify.bytes_xfered !=
-			IPA_DMA_DUMMY_BUFF_SZ);
-	else
-		ipa_assert_on(len != gsi_notify.bytes_xfered);
-
-	ipa_assert_on(dest != ((struct ipa3_dma_xfer_wrapper *)
-			(gsi_notify.xfer_user_data))->phys_addr_dest);
-
-	mutex_lock(&ipa3_dma_ctx->sync_lock);
-	list_del(&head_descr->link);
-	cons_sys->len--;
-	kmem_cache_free(ipa3_dma_ctx->ipa_dma_xfer_wrapper_cache, xfer_descr);
-	/* wake the head of the list */
-	if (!list_empty(&cons_sys->head_desc_list)) {
-		head_descr = list_first_entry(&cons_sys->head_desc_list,
-				struct ipa3_dma_xfer_wrapper, link);
-		complete(&head_descr->xfer_done);
-	}
-	mutex_unlock(&ipa3_dma_ctx->sync_lock);
-
-	atomic_inc(&ipa3_dma_ctx->total_sync_memcpy);
-	atomic_dec(&ipa3_dma_ctx->sync_memcpy_pending_cnt);
-	if (ipa3_dma_ctx->destroy_pending && !ipa3_dma_work_pending())
-		complete(&ipa3_dma_ctx->done);
-
-	IPADMA_FUNC_EXIT();
-	return res;
-
-fail_send:
-	list_del(&xfer_descr->link);
-	cons_sys->len--;
-	mutex_unlock(&ipa3_dma_ctx->sync_lock);
-	kmem_cache_free(ipa3_dma_ctx->ipa_dma_xfer_wrapper_cache, xfer_descr);
-fail_mem_alloc:
-	atomic_dec(&ipa3_dma_ctx->sync_memcpy_pending_cnt);
-	if (ipa3_dma_ctx->destroy_pending && !ipa3_dma_work_pending())
-		complete(&ipa3_dma_ctx->done);
-	return res;
-}
-
-/**
- * ipa3_dma_async_memcpy()- Perform asynchronous memcpy using IPA.
- *
- * @dest: physical address to store the copied data.
- * @src: physical address of the source data to copy.
- * @len: number of bytes to copy.
- * @user_cb: callback function to notify the client when the copy was done.
- * @user_param: cookie for user_cb.
- *
- * Return codes: 0: success
- *		-EINVAL: invalid params
- *		-EPERM: operation not permitted as ipa_dma isn't enable or
- *			initialized
- *		-gsi_status : on GSI failures
- *		-EFAULT: descr fifo is full.
- */
-int ipa3_dma_async_memcpy(u64 dest, u64 src, int len,
-		void (*user_cb)(void *user1), void *user_param)
-{
-	int ep_idx;
-	int res = 0;
-	struct ipa3_dma_xfer_wrapper *xfer_descr = NULL;
-	struct ipa3_sys_context *prod_sys;
-	struct ipa3_sys_context *cons_sys;
-	struct gsi_xfer_elem xfer_elem_cons, xfer_elem_prod;
-	unsigned long flags;
-
-	IPADMA_FUNC_ENTRY();
-	IPADMA_DBG_LOW("dest =  0x%llx, src = 0x%llx, len = %d\n",
-		dest, src, len);
-	if (ipa3_dma_ctx == NULL) {
-		IPADMA_ERR("IPADMA isn't initialized, can't memcpy\n");
-		return -EPERM;
-	}
-	if ((max(src, dest) - min(src, dest)) < len) {
-		IPADMA_ERR("invalid addresses - overlapping buffers\n");
-		return -EINVAL;
-	}
-	if (len > IPA_DMA_MAX_PKT_SZ || len <= 0) {
-		IPADMA_ERR("invalid len, %d\n", len);
-		return	-EINVAL;
-	}
-	if (!user_cb) {
-		IPADMA_ERR("null pointer: user_cb\n");
-		return -EINVAL;
-	}
-	spin_lock_irqsave(&ipa3_dma_ctx->pending_lock, flags);
-	if (!ipa3_dma_ctx->enable_ref_cnt) {
-		IPADMA_ERR("can't memcpy, IPA_DMA isn't enabled\n");
-		spin_unlock_irqrestore(&ipa3_dma_ctx->pending_lock, flags);
-		return -EPERM;
-	}
-	atomic_inc(&ipa3_dma_ctx->async_memcpy_pending_cnt);
-	spin_unlock_irqrestore(&ipa3_dma_ctx->pending_lock, flags);
-
-	ep_idx = ipa3_get_ep_mapping(IPA_CLIENT_MEMCPY_DMA_ASYNC_CONS);
-	if (-1 == ep_idx) {
-		IPADMA_ERR("Client %u is not mapped\n",
-			IPA_CLIENT_MEMCPY_DMA_ASYNC_CONS);
-		return -EFAULT;
-	}
-	cons_sys = ipa3_ctx->ep[ep_idx].sys;
-
-	ep_idx = ipa3_get_ep_mapping(IPA_CLIENT_MEMCPY_DMA_ASYNC_PROD);
-	if (-1 == ep_idx) {
-		IPADMA_ERR("Client %u is not mapped\n",
-			IPA_CLIENT_MEMCPY_DMA_ASYNC_PROD);
-		return -EFAULT;
-	}
-	prod_sys = ipa3_ctx->ep[ep_idx].sys;
-
-	xfer_descr = kmem_cache_zalloc(ipa3_dma_ctx->ipa_dma_xfer_wrapper_cache,
-					GFP_KERNEL);
-	if (!xfer_descr) {
-		res = -ENOMEM;
-		goto fail_mem_alloc;
-	}
-	xfer_descr->phys_addr_dest = dest;
-	xfer_descr->phys_addr_src = src;
-	xfer_descr->len = len;
-	xfer_descr->callback = user_cb;
-	xfer_descr->user1 = user_param;
-
-	spin_lock_irqsave(&ipa3_dma_ctx->async_lock, flags);
-	list_add_tail(&xfer_descr->link, &cons_sys->head_desc_list);
-	cons_sys->len++;
-	/*
-	 * when copy is less than 9B we need to chain another dummy
-	 * copy so the total size will be larger (for ipav3.5)
-	 */
-	if ((ipa_get_hw_type() == IPA_HW_v3_5) && len <
-		IPA_DMA_PREFETCH_WA_THRESHOLD) {
-		xfer_elem_cons.addr = dest;
-		xfer_elem_cons.len = len;
-		xfer_elem_cons.type = GSI_XFER_ELEM_DATA;
-		xfer_elem_cons.flags = GSI_XFER_FLAG_EOT;
-		xfer_elem_cons.xfer_user_data = NULL;
-		res = gsi_queue_xfer(cons_sys->ep->gsi_chan_hdl, 1,
-			&xfer_elem_cons, false);
-		if (res) {
-			IPADMA_ERR(
-				"Failed: gsi_queue_xfer on dest descr res: %d\n",
-				res);
-			goto fail_send;
-		}
-		xfer_elem_cons.addr =
-			ipa3_dma_ctx->ipa_dma_dummy_dst_async.phys_base;
-		xfer_elem_cons.len = IPA_DMA_DUMMY_BUFF_SZ;
-		xfer_elem_cons.type = GSI_XFER_ELEM_DATA;
-		xfer_elem_cons.flags = GSI_XFER_FLAG_EOT;
-		xfer_elem_cons.xfer_user_data = xfer_descr;
-		res = gsi_queue_xfer(cons_sys->ep->gsi_chan_hdl, 1,
-			&xfer_elem_cons, true);
-		if (res) {
-			IPADMA_ERR(
-				"Failed: gsi_queue_xfer on dummy dest descr res: %d\n",
-				res);
-			goto fail_send;
-		}
-
-		xfer_elem_prod.addr = src;
-		xfer_elem_prod.len = len;
-		xfer_elem_prod.type = GSI_XFER_ELEM_DATA;
-		xfer_elem_prod.flags = GSI_XFER_FLAG_CHAIN;
-		xfer_elem_prod.xfer_user_data = NULL;
-		res = gsi_queue_xfer(prod_sys->ep->gsi_chan_hdl, 1,
-			&xfer_elem_prod, false);
-		if (res) {
-			IPADMA_ERR(
-				"Failed: gsi_queue_xfer on src descr res: %d\n",
-				res);
-			ipa_assert();
-			goto fail_send;
-		}
-		xfer_elem_prod.addr =
-			ipa3_dma_ctx->ipa_dma_dummy_src_async.phys_base;
-		xfer_elem_prod.len = IPA_DMA_DUMMY_BUFF_SZ;
-		xfer_elem_prod.type = GSI_XFER_ELEM_DATA;
-		xfer_elem_prod.flags = GSI_XFER_FLAG_EOT;
-		xfer_elem_prod.xfer_user_data = NULL;
-		res = gsi_queue_xfer(prod_sys->ep->gsi_chan_hdl, 1,
-			&xfer_elem_prod, true);
-		if (res) {
-			IPADMA_ERR(
-				"Failed: gsi_queue_xfer on dummy src descr res: %d\n",
-				res);
-			ipa_assert();
-			goto fail_send;
-		}
-	} else {
-
-		xfer_elem_cons.addr = dest;
-		xfer_elem_cons.len = len;
-		xfer_elem_cons.type = GSI_XFER_ELEM_DATA;
-		xfer_elem_cons.flags = GSI_XFER_FLAG_EOT;
-		xfer_elem_cons.xfer_user_data = xfer_descr;
-		res = gsi_queue_xfer(cons_sys->ep->gsi_chan_hdl, 1,
-			&xfer_elem_cons, true);
-		if (res) {
-			IPADMA_ERR(
-					"Failed: gsi_queue_xfer on dummy dest descr res: %d\n",
-				res);
-				ipa_assert();
-			goto fail_send;
-		}
-		xfer_elem_prod.addr = src;
-		xfer_elem_prod.len = len;
-		xfer_elem_prod.type = GSI_XFER_ELEM_DATA;
-		xfer_elem_prod.flags = GSI_XFER_FLAG_EOT;
-		xfer_elem_prod.xfer_user_data = NULL;
-		res = gsi_queue_xfer(prod_sys->ep->gsi_chan_hdl, 1,
-			&xfer_elem_prod, true);
-		if (res) {
-			IPADMA_ERR(
-					"Failed: gsi_queue_xfer on dummy src descr res: %d\n",
-				res);
-			ipa_assert();
-			goto fail_send;
-		}
-
-	}
-	spin_unlock_irqrestore(&ipa3_dma_ctx->async_lock, flags);
-	IPADMA_FUNC_EXIT();
-	return res;
-
-fail_send:
-	list_del(&xfer_descr->link);
-	spin_unlock_irqrestore(&ipa3_dma_ctx->async_lock, flags);
-	kmem_cache_free(ipa3_dma_ctx->ipa_dma_xfer_wrapper_cache, xfer_descr);
-fail_mem_alloc:
-	atomic_dec(&ipa3_dma_ctx->async_memcpy_pending_cnt);
-	if (ipa3_dma_ctx->destroy_pending && !ipa3_dma_work_pending())
-		complete(&ipa3_dma_ctx->done);
-	return res;
-}
-
-/**
- * ipa3_dma_uc_memcpy() - Perform a memcpy action using IPA uC
- * @dest: physical address to store the copied data.
- * @src: physical address of the source data to copy.
- * @len: number of bytes to copy.
- *
- * Return codes: 0: success
- *		-EINVAL: invalid params
- *		-EPERM: operation not permitted as ipa_dma isn't enable or
- *			initialized
- *		-EBADF: IPA uC is not loaded
- */
-int ipa3_dma_uc_memcpy(phys_addr_t dest, phys_addr_t src, int len)
-{
-	int res;
-	unsigned long flags;
-
-	IPADMA_FUNC_ENTRY();
-	if (ipa3_dma_ctx == NULL) {
-		IPADMA_ERR("IPADMA isn't initialized, can't memcpy\n");
-		return -EPERM;
-	}
-	if ((max(src, dest) - min(src, dest)) < len) {
-		IPADMA_ERR("invalid addresses - overlapping buffers\n");
-		return -EINVAL;
-	}
-	if (len > IPA_DMA_MAX_PKT_SZ || len <= 0) {
-		IPADMA_ERR("invalid len, %d\n", len);
-		return	-EINVAL;
-	}
-
-	spin_lock_irqsave(&ipa3_dma_ctx->pending_lock, flags);
-	if (!ipa3_dma_ctx->enable_ref_cnt) {
-		IPADMA_ERR("can't memcpy, IPADMA isn't enabled\n");
-		spin_unlock_irqrestore(&ipa3_dma_ctx->pending_lock, flags);
-		return -EPERM;
-	}
-	atomic_inc(&ipa3_dma_ctx->uc_memcpy_pending_cnt);
-	spin_unlock_irqrestore(&ipa3_dma_ctx->pending_lock, flags);
-
-	res = ipa3_uc_memcpy(dest, src, len);
-	if (res) {
-		IPADMA_ERR("ipa3_uc_memcpy failed %d\n", res);
-		goto dec_and_exit;
-	}
-
-	atomic_inc(&ipa3_dma_ctx->total_uc_memcpy);
-	res = 0;
-dec_and_exit:
-	atomic_dec(&ipa3_dma_ctx->uc_memcpy_pending_cnt);
-	if (ipa3_dma_ctx->destroy_pending && !ipa3_dma_work_pending())
-		complete(&ipa3_dma_ctx->done);
-	IPADMA_FUNC_EXIT();
-	return res;
-}
-
-/**
- * ipa3_dma_destroy() -teardown IPADMA pipes and release ipadma.
- *
- * this is a blocking function, returns just after destroying IPADMA.
- */
-void ipa3_dma_destroy(void)
-{
-	int res = 0;
-
-	IPADMA_FUNC_ENTRY();
-
-	if (!ipa3_dma_init_refcnt_ctrl) {
-		IPADMA_ERR("Setup isn't done\n");
-		return;
-	}
-
-	mutex_lock(&ipa3_dma_init_refcnt_ctrl->lock);
-	if (ipa3_dma_init_refcnt_ctrl->ref_cnt > 1) {
-		IPADMA_DBG("Multiple initialization done. refcnt=%d\n",
-			ipa3_dma_init_refcnt_ctrl->ref_cnt);
-		ipa3_dma_init_refcnt_ctrl->ref_cnt--;
-		goto completed;
-	}
-
-	if ((!ipa3_dma_ctx) || (ipa3_dma_init_refcnt_ctrl->ref_cnt == 0)) {
-		IPADMA_ERR("IPADMA isn't initialized ctx=%pK\n", ipa3_dma_ctx);
-		goto completed;
-	}
-
-	if (ipa3_dma_work_pending()) {
-		ipa3_dma_ctx->destroy_pending = true;
-		IPADMA_DBG("There are pending memcpy, wait for completion\n");
-		wait_for_completion(&ipa3_dma_ctx->done);
-	}
-
-	if (ipa3_dma_ctx->enable_ref_cnt > 0) {
-		IPADMA_ERR("IPADMA still enabled\n");
-		goto completed;
-	}
-
-	res = ipa3_teardown_sys_pipe(ipa3_dma_ctx->ipa_dma_async_cons_hdl);
-	if (res)
-		IPADMA_ERR("teardown IPADMA ASYNC CONS failed\n");
-	ipa3_dma_ctx->ipa_dma_async_cons_hdl = 0;
-	res = ipa3_teardown_sys_pipe(ipa3_dma_ctx->ipa_dma_sync_cons_hdl);
-	if (res)
-		IPADMA_ERR("teardown IPADMA SYNC CONS failed\n");
-	ipa3_dma_ctx->ipa_dma_sync_cons_hdl = 0;
-	res = ipa3_teardown_sys_pipe(ipa3_dma_ctx->ipa_dma_async_prod_hdl);
-	if (res)
-		IPADMA_ERR("teardown IPADMA ASYNC PROD failed\n");
-	ipa3_dma_ctx->ipa_dma_async_prod_hdl = 0;
-	res = ipa3_teardown_sys_pipe(ipa3_dma_ctx->ipa_dma_sync_prod_hdl);
-	if (res)
-		IPADMA_ERR("teardown IPADMA SYNC PROD failed\n");
-	ipa3_dma_ctx->ipa_dma_sync_prod_hdl = 0;
-
-	ipa3_dma_debugfs_destroy();
-	kmem_cache_destroy(ipa3_dma_ctx->ipa_dma_xfer_wrapper_cache);
-	dma_free_coherent(ipa3_ctx->pdev, IPA_DMA_DUMMY_BUFF_SZ * 4,
-		ipa3_dma_ctx->ipa_dma_dummy_src_sync.base,
-		ipa3_dma_ctx->ipa_dma_dummy_src_sync.phys_base);
-	kfree(ipa3_dma_ctx);
-	ipa3_dma_ctx = NULL;
-
-	ipa3_dma_init_refcnt_ctrl->ref_cnt = 0;
-	IPADMA_FUNC_EXIT();
-
-completed:
-	mutex_unlock(&ipa3_dma_init_refcnt_ctrl->lock);
-}
-
-/**
- * ipa3_dma_async_memcpy_notify_cb() - Callback function which will be called
- * by IPA driver after getting notify on Rx operation is completed (data was
- * written to dest descriptor on async_cons ep).
- *
- * @priv -not in use.
- * @evt - event name - IPA_RECIVE.
- * @data -the ipa_mem_buffer.
- */
-void ipa3_dma_async_memcpy_notify_cb(void *priv
-			, enum ipa_dp_evt_type evt, unsigned long data)
-{
-	int ep_idx = 0;
-	struct ipa3_dma_xfer_wrapper *xfer_descr_expected;
-	struct ipa3_sys_context *sys;
-	unsigned long flags;
-
-	IPADMA_FUNC_ENTRY();
-
-	ep_idx = ipa3_get_ep_mapping(IPA_CLIENT_MEMCPY_DMA_ASYNC_CONS);
-	if (ep_idx < 0) {
-		IPADMA_ERR("IPA Client mapping failed\n");
-		return;
-	}
-	sys = ipa3_ctx->ep[ep_idx].sys;
-
-	spin_lock_irqsave(&ipa3_dma_ctx->async_lock, flags);
-	xfer_descr_expected = list_first_entry(&sys->head_desc_list,
-				 struct ipa3_dma_xfer_wrapper, link);
-	list_del(&xfer_descr_expected->link);
-	sys->len--;
-	spin_unlock_irqrestore(&ipa3_dma_ctx->async_lock, flags);
-	atomic_inc(&ipa3_dma_ctx->total_async_memcpy);
-	atomic_dec(&ipa3_dma_ctx->async_memcpy_pending_cnt);
-	xfer_descr_expected->callback(xfer_descr_expected->user1);
-
-	kmem_cache_free(ipa3_dma_ctx->ipa_dma_xfer_wrapper_cache,
-		xfer_descr_expected);
-
-	if (ipa3_dma_ctx->destroy_pending && !ipa3_dma_work_pending())
-		complete(&ipa3_dma_ctx->done);
-
-	IPADMA_FUNC_EXIT();
-}
-
-#ifdef CONFIG_DEBUG_FS
-static struct dentry *dent;
-static struct dentry *dfile_info;
-
-static ssize_t ipa3_dma_debugfs_read(struct file *file, char __user *ubuf,
-				 size_t count, loff_t *ppos)
-{
-	int nbytes = 0;
-
-	if (!ipa3_dma_init_refcnt_ctrl) {
-		nbytes += scnprintf(&dbg_buff[nbytes],
-			IPADMA_MAX_MSG_LEN - nbytes,
-			"Setup was not done\n");
-		goto completed;
-
-	}
-
-	if (!ipa3_dma_ctx) {
-		nbytes += scnprintf(&dbg_buff[nbytes],
-			IPADMA_MAX_MSG_LEN - nbytes,
-			"Status:\n	Not initialized (ref_cnt=%d)\n",
-			ipa3_dma_init_refcnt_ctrl->ref_cnt);
-	} else {
-		nbytes += scnprintf(&dbg_buff[nbytes],
-			IPADMA_MAX_MSG_LEN - nbytes,
-			"Status:\n	Initialized (ref_cnt=%d)\n",
-			ipa3_dma_init_refcnt_ctrl->ref_cnt);
-		nbytes += scnprintf(&dbg_buff[nbytes],
-			IPADMA_MAX_MSG_LEN - nbytes,
-			"	%s (ref_cnt=%d)\n",
-			(ipa3_dma_ctx->enable_ref_cnt > 0) ?
-			"Enabled" : "Disabled",
-			ipa3_dma_ctx->enable_ref_cnt);
-		nbytes += scnprintf(&dbg_buff[nbytes],
-			IPADMA_MAX_MSG_LEN - nbytes,
-			"Statistics:\n	total sync memcpy: %d\n	",
-			atomic_read(&ipa3_dma_ctx->total_sync_memcpy));
-		nbytes += scnprintf(&dbg_buff[nbytes],
-			IPADMA_MAX_MSG_LEN - nbytes,
-			"total async memcpy: %d\n	",
-			atomic_read(&ipa3_dma_ctx->total_async_memcpy));
-		nbytes += scnprintf(&dbg_buff[nbytes],
-			IPADMA_MAX_MSG_LEN - nbytes,
-			"total uc memcpy: %d\n	",
-			atomic_read(&ipa3_dma_ctx->total_uc_memcpy));
-		nbytes += scnprintf(&dbg_buff[nbytes],
-			IPADMA_MAX_MSG_LEN - nbytes,
-			"pending sync memcpy jobs: %d\n	",
-			atomic_read(&ipa3_dma_ctx->sync_memcpy_pending_cnt));
-		nbytes += scnprintf(&dbg_buff[nbytes],
-			IPADMA_MAX_MSG_LEN - nbytes,
-			"pending async memcpy jobs: %d\n	",
-			atomic_read(&ipa3_dma_ctx->async_memcpy_pending_cnt));
-		nbytes += scnprintf(&dbg_buff[nbytes],
-			IPADMA_MAX_MSG_LEN - nbytes,
-			"pending uc memcpy jobs: %d\n",
-			atomic_read(&ipa3_dma_ctx->uc_memcpy_pending_cnt));
-	}
-
-completed:
-	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, nbytes);
-}
-
-static ssize_t ipa3_dma_debugfs_reset_statistics(struct file *file,
-					const char __user *ubuf,
-					size_t count,
-					loff_t *ppos)
-{
-	s8 in_num = 0;
-	int ret;
-
-	ret = kstrtos8_from_user(ubuf, count, 0, &in_num);
-	if (ret)
-		return ret;
-
-	switch (in_num) {
-	case 0:
-		if (ipa3_dma_work_pending())
-			IPADMA_ERR("Note, there are pending memcpy\n");
-
-		atomic_set(&ipa3_dma_ctx->total_async_memcpy, 0);
-		atomic_set(&ipa3_dma_ctx->total_sync_memcpy, 0);
-		break;
-	default:
-		IPADMA_ERR("invalid argument: To reset statistics echo 0\n");
-		break;
-	}
-	return count;
-}
-
-const struct file_operations ipa3_ipadma_stats_ops = {
-	.read = ipa3_dma_debugfs_read,
-	.write = ipa3_dma_debugfs_reset_statistics,
-};
-
-static void ipa3_dma_debugfs_init(void)
-{
-	const mode_t read_write_mode = 0666;
-
-	dent = debugfs_create_dir("ipa_dma", 0);
-	if (IS_ERR(dent)) {
-		IPADMA_ERR("fail to create folder ipa_dma\n");
-		return;
-	}
-
-	dfile_info =
-		debugfs_create_file("info", read_write_mode, dent,
-				 0, &ipa3_ipadma_stats_ops);
-	if (!dfile_info || IS_ERR(dfile_info)) {
-		IPADMA_ERR("fail to create file stats\n");
-		goto fail;
-	}
-	return;
-fail:
-	debugfs_remove_recursive(dent);
-}
-
-static void ipa3_dma_debugfs_destroy(void)
-{
-	debugfs_remove_recursive(dent);
-}
-
-#endif /* !CONFIG_DEBUG_FS */

ipa/ipa_v3/ipa_dp.c

@@ -1,4474 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2012-2019, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/delay.h>
-#include <linux/device.h>
-#include <linux/dmapool.h>
-#include <linux/list.h>
-#include <linux/netdevice.h>
-#include <linux/msm_gsi.h>
-#include "ipa_i.h"
-#include "ipa_trace.h"
-#include "ipahal/ipahal.h"
-#include "ipahal/ipahal_fltrt.h"
-
-#define IPA_WAN_AGGR_PKT_CNT 5
-#define IPA_WAN_NAPI_MAX_FRAMES (NAPI_WEIGHT / IPA_WAN_AGGR_PKT_CNT)
-#define IPA_LAST_DESC_CNT 0xFFFF
-#define POLLING_INACTIVITY_RX 40
-#define POLLING_MIN_SLEEP_RX 1010
-#define POLLING_MAX_SLEEP_RX 1050
-#define POLLING_INACTIVITY_TX 40
-#define POLLING_MIN_SLEEP_TX 400
-#define POLLING_MAX_SLEEP_TX 500
-#define SUSPEND_MIN_SLEEP_RX 1000
-#define SUSPEND_MAX_SLEEP_RX 1005
-/* 8K less 1 nominal MTU (1500 bytes) rounded to units of KB */
-#define IPA_MTU 1500
-#define IPA_GENERIC_AGGR_BYTE_LIMIT 6
-#define IPA_GENERIC_AGGR_TIME_LIMIT 500 /* 0.5msec */
-#define IPA_GENERIC_AGGR_PKT_LIMIT 0
-
-#define IPA_GSB_AGGR_BYTE_LIMIT 14
-#define IPA_GSB_RX_BUFF_BASE_SZ 16384
-
-#define IPA_GENERIC_RX_BUFF_BASE_SZ 8192
-#define IPA_REAL_GENERIC_RX_BUFF_SZ(X) (SKB_DATA_ALIGN(\
-		(X) + NET_SKB_PAD) +\
-		SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
-#define IPA_GENERIC_RX_BUFF_SZ(X) ((X) -\
-		(IPA_REAL_GENERIC_RX_BUFF_SZ(X) - (X)))
-#define IPA_GENERIC_RX_BUFF_LIMIT (\
-		IPA_REAL_GENERIC_RX_BUFF_SZ(\
-		IPA_GENERIC_RX_BUFF_BASE_SZ) -\
-		IPA_GENERIC_RX_BUFF_BASE_SZ)
-
-/* less 1 nominal MTU (1500 bytes) rounded to units of KB */
-#define IPA_ADJUST_AGGR_BYTE_LIMIT(X) (((X) - IPA_MTU)/1000)
-
-#define IPA_RX_BUFF_CLIENT_HEADROOM 256
-
-#define IPA_WLAN_RX_POOL_SZ 100
-#define IPA_WLAN_RX_POOL_SZ_LOW_WM 5
-#define IPA_WLAN_RX_BUFF_SZ 2048
-#define IPA_WLAN_COMM_RX_POOL_LOW 100
-#define IPA_WLAN_COMM_RX_POOL_HIGH 900
-
-#define IPA_ODU_RX_BUFF_SZ 2048
-#define IPA_ODU_RX_POOL_SZ 64
-
-#define IPA_ODL_RX_BUFF_SZ (16 * 1024)
-
-#define IPA_GSI_MAX_CH_LOW_WEIGHT 15
-#define IPA_GSI_EVT_RING_INT_MODT (16) /* 0.5ms under 32KHz clock */
-#define IPA_GSI_EVT_RING_INT_MODC (20)
-
-#define IPA_GSI_CH_20_WA_NUM_CH_TO_ALLOC 10
-/* The below virtual channel cannot be used by any entity */
-#define IPA_GSI_CH_20_WA_VIRT_CHAN 29
-
-#define IPA_DEFAULT_SYS_YELLOW_WM 32
-#define IPA_REPL_XFER_THRESH 20
-#define IPA_REPL_XFER_MAX 36
-
-#define IPA_TX_SEND_COMPL_NOP_DELAY_NS (2 * 1000 * 1000)
-
-#define IPA_APPS_BW_FOR_PM 700
-
-#define IPA_SEND_MAX_DESC (20)
-
-#define IPA_EOT_THRESH 32
-
-#define IPA_QMAP_ID_BYTE 0
-
-static struct sk_buff *ipa3_get_skb_ipa_rx(unsigned int len, gfp_t flags);
-static void ipa3_replenish_wlan_rx_cache(struct ipa3_sys_context *sys);
-static void ipa3_replenish_rx_cache(struct ipa3_sys_context *sys);
-static void ipa3_replenish_rx_work_func(struct work_struct *work);
-static void ipa3_fast_replenish_rx_cache(struct ipa3_sys_context *sys);
-static void ipa3_wq_handle_rx(struct work_struct *work);
-static void ipa3_wq_rx_common(struct ipa3_sys_context *sys,
-	struct gsi_chan_xfer_notify *notify);
-static void ipa3_wq_rx_napi_chain(struct ipa3_sys_context *sys,
-		struct gsi_chan_xfer_notify *notify, uint32_t num);
-static void ipa3_wlan_wq_rx_common(struct ipa3_sys_context *sys,
-				struct gsi_chan_xfer_notify *notify);
-static int ipa3_assign_policy(struct ipa_sys_connect_params *in,
-		struct ipa3_sys_context *sys);
-static void ipa3_cleanup_rx(struct ipa3_sys_context *sys);
-static void ipa3_wq_rx_avail(struct work_struct *work);
-static void ipa3_alloc_wlan_rx_common_cache(u32 size);
-static void ipa3_cleanup_wlan_rx_common_cache(void);
-static void ipa3_wq_repl_rx(struct work_struct *work);
-static void ipa3_dma_memcpy_notify(struct ipa3_sys_context *sys);
-static int ipa_gsi_setup_coal_def_channel(struct ipa_sys_connect_params *in,
-	struct ipa3_ep_context *ep, struct ipa3_ep_context *coal_ep);
-static int ipa_gsi_setup_channel(struct ipa_sys_connect_params *in,
-	struct ipa3_ep_context *ep);
-static int ipa_gsi_setup_event_ring(struct ipa3_ep_context *ep,
-	u32 ring_size, gfp_t mem_flag);
-static int ipa_gsi_setup_transfer_ring(struct ipa3_ep_context *ep,
-	u32 ring_size, struct ipa3_sys_context *user_data, gfp_t mem_flag);
-static int ipa3_teardown_coal_def_pipe(u32 clnt_hdl);
-static int ipa_populate_tag_field(struct ipa3_desc *desc,
-		struct ipa3_tx_pkt_wrapper *tx_pkt,
-		struct ipahal_imm_cmd_pyld **tag_pyld_ret);
-static int ipa_poll_gsi_pkt(struct ipa3_sys_context *sys,
-	struct gsi_chan_xfer_notify *notify);
-static int ipa_poll_gsi_n_pkt(struct ipa3_sys_context *sys,
-	struct gsi_chan_xfer_notify *notify, int expected_num,
-	int *actual_num);
-static unsigned long tag_to_pointer_wa(uint64_t tag);
-static uint64_t pointer_to_tag_wa(struct ipa3_tx_pkt_wrapper *tx_pkt);
-
-static u32 ipa_adjust_ra_buff_base_sz(u32 aggr_byte_limit);
-
-static void ipa3_wq_write_done_common(struct ipa3_sys_context *sys,
-				struct ipa3_tx_pkt_wrapper *tx_pkt)
-{
-	struct ipa3_tx_pkt_wrapper *next_pkt;
-	int i, cnt;
-
-	if (unlikely(tx_pkt == NULL)) {
-		IPAERR("tx_pkt is NULL\n");
-		return;
-	}
-
-	cnt = tx_pkt->cnt;
-	IPADBG_LOW("cnt: %d\n", cnt);
-	for (i = 0; i < cnt; i++) {
-		spin_lock_bh(&sys->spinlock);
-		if (unlikely(list_empty(&sys->head_desc_list))) {
-			spin_unlock_bh(&sys->spinlock);
-			return;
-		}
-		next_pkt = list_next_entry(tx_pkt, link);
-		list_del(&tx_pkt->link);
-		sys->len--;
-		spin_unlock_bh(&sys->spinlock);
-		if (!tx_pkt->no_unmap_dma) {
-			if (tx_pkt->type != IPA_DATA_DESC_SKB_PAGED) {
-				dma_unmap_single(ipa3_ctx->pdev,
-					tx_pkt->mem.phys_base,
-					tx_pkt->mem.size,
-					DMA_TO_DEVICE);
-			} else {
-				dma_unmap_page(ipa3_ctx->pdev,
-					tx_pkt->mem.phys_base,
-					tx_pkt->mem.size,
-					DMA_TO_DEVICE);
-			}
-		}
-		if (tx_pkt->callback)
-			tx_pkt->callback(tx_pkt->user1, tx_pkt->user2);
-
-		kmem_cache_free(ipa3_ctx->tx_pkt_wrapper_cache, tx_pkt);
-		tx_pkt = next_pkt;
-	}
-}
-
-static void ipa3_wq_write_done_status(int src_pipe,
-			struct ipa3_tx_pkt_wrapper *tx_pkt)
-{
-	struct ipa3_sys_context *sys;
-
-	WARN_ON(src_pipe >= ipa3_ctx->ipa_num_pipes);
-
-	if (!ipa3_ctx->ep[src_pipe].status.status_en)
-		return;
-
-	sys = ipa3_ctx->ep[src_pipe].sys;
-	if (!sys)
-		return;
-
-	ipa3_wq_write_done_common(sys, tx_pkt);
-}
-
-/**
- * ipa_write_done() - this function will be (eventually) called when a Tx
- * operation is complete
- * @data: user pointer point to the ipa3_sys_context
- *
- * Will be called in deferred context.
- * - invoke the callback supplied by the client who sent this command
- * - iterate over all packets and validate that
- *   the order for sent packet is the same as expected
- * - delete all the tx packet descriptors from the system
- *   pipe context (not needed anymore)
- */
-static void ipa3_tasklet_write_done(unsigned long data)
-{
-	struct ipa3_sys_context *sys;
-	struct ipa3_tx_pkt_wrapper *this_pkt;
-	bool xmit_done = false;
-
-	sys = (struct ipa3_sys_context *)data;
-	spin_lock_bh(&sys->spinlock);
-	while (atomic_add_unless(&sys->xmit_eot_cnt, -1, 0)) {
-		while (!list_empty(&sys->head_desc_list)) {
-			this_pkt = list_first_entry(&sys->head_desc_list,
-				struct ipa3_tx_pkt_wrapper, link);
-			xmit_done = this_pkt->xmit_done;
-			spin_unlock_bh(&sys->spinlock);
-			ipa3_wq_write_done_common(sys, this_pkt);
-			spin_lock_bh(&sys->spinlock);
-			if (xmit_done)
-				break;
-		}
-	}
-	spin_unlock_bh(&sys->spinlock);
-}
-
-
-static void ipa3_send_nop_desc(struct work_struct *work)
-{
-	struct ipa3_sys_context *sys = container_of(work,
-		struct ipa3_sys_context, work);
-	struct gsi_xfer_elem nop_xfer;
-	struct ipa3_tx_pkt_wrapper *tx_pkt;
-
-	IPADBG_LOW("gsi send NOP for ch: %lu\n", sys->ep->gsi_chan_hdl);
-	tx_pkt = kmem_cache_zalloc(ipa3_ctx->tx_pkt_wrapper_cache, GFP_KERNEL);
-	if (!tx_pkt) {
-		queue_work(sys->wq, &sys->work);
-		return;
-	}
-
-	INIT_LIST_HEAD(&tx_pkt->link);
-	tx_pkt->cnt = 1;
-	tx_pkt->no_unmap_dma = true;
-	tx_pkt->sys = sys;
-	spin_lock_bh(&sys->spinlock);
-	if (unlikely(!sys->nop_pending)) {
-		spin_unlock_bh(&sys->spinlock);
-		kmem_cache_free(ipa3_ctx->tx_pkt_wrapper_cache, tx_pkt);
-		return;
-	}
-	list_add_tail(&tx_pkt->link, &sys->head_desc_list);
-	sys->nop_pending = false;
-
-	memset(&nop_xfer, 0, sizeof(nop_xfer));
-	nop_xfer.type = GSI_XFER_ELEM_NOP;
-	nop_xfer.flags = GSI_XFER_FLAG_EOT;
-	nop_xfer.xfer_user_data = tx_pkt;
-	if (gsi_queue_xfer(sys->ep->gsi_chan_hdl, 1, &nop_xfer, true)) {
-		spin_unlock_bh(&sys->spinlock);
-		IPAERR("gsi_queue_xfer for ch:%lu failed\n",
-			sys->ep->gsi_chan_hdl);
-		queue_work(sys->wq, &sys->work);
-		return;
-	}
-	spin_unlock_bh(&sys->spinlock);
-
-	/* make sure TAG process is sent before clocks are gated */
-	ipa3_ctx->tag_process_before_gating = true;
-
-}
-
-
-/**
- * ipa3_send() - Send multiple descriptors in one HW transaction
- * @sys: system pipe context
- * @num_desc: number of packets
- * @desc: packets to send (may be immediate command or data)
- * @in_atomic:  whether caller is in atomic context
- *
- * This function is used for GPI connection.
- * - ipa3_tx_pkt_wrapper will be used for each ipa
- *   descriptor (allocated from wrappers cache)
- * - The wrapper struct will be configured for each ipa-desc payload and will
- *   contain information which will be later used by the user callbacks
- * - Each packet (command or data) that will be sent will also be saved in
- *   ipa3_sys_context for later check that all data was sent
- *
- * Return codes: 0: success, -EFAULT: failure
- */
-int ipa3_send(struct ipa3_sys_context *sys,
-		u32 num_desc,
-		struct ipa3_desc *desc,
-		bool in_atomic)
-{
-	struct ipa3_tx_pkt_wrapper *tx_pkt, *tx_pkt_first = NULL;
-	struct ipahal_imm_cmd_pyld *tag_pyld_ret = NULL;
-	struct ipa3_tx_pkt_wrapper *next_pkt;
-	struct gsi_xfer_elem gsi_xfer[IPA_SEND_MAX_DESC];
-	int i = 0;
-	int j;
-	int result;
-	u32 mem_flag = GFP_ATOMIC;
-	const struct ipa_gsi_ep_config *gsi_ep_cfg;
-	bool send_nop = false;
-	unsigned int max_desc;
-
-	if (unlikely(!in_atomic))
-		mem_flag = GFP_KERNEL;
-
-	gsi_ep_cfg = ipa3_get_gsi_ep_info(sys->ep->client);
-	if (unlikely(!gsi_ep_cfg)) {
-		IPAERR("failed to get gsi EP config for client=%d\n",
-			sys->ep->client);
-		return -EFAULT;
-	}
-	if (unlikely(num_desc > IPA_SEND_MAX_DESC)) {
-		IPAERR("max descriptors reached need=%d max=%d\n",
-			num_desc, IPA_SEND_MAX_DESC);
-		WARN_ON(1);
-		return -EPERM;
-	}
-
-	max_desc = gsi_ep_cfg->ipa_if_tlv;
-	if (gsi_ep_cfg->prefetch_mode == GSI_SMART_PRE_FETCH ||
-		gsi_ep_cfg->prefetch_mode == GSI_FREE_PRE_FETCH)
-		max_desc -= gsi_ep_cfg->prefetch_threshold;
-
-	if (unlikely(num_desc > max_desc)) {
-		IPAERR("Too many chained descriptors need=%d max=%d\n",
-			num_desc, max_desc);
-		WARN_ON(1);
-		return -EPERM;
-	}
-
-	/* initialize only the xfers we use */
-	memset(gsi_xfer, 0, sizeof(gsi_xfer[0]) * num_desc);
-
-	spin_lock_bh(&sys->spinlock);
-
-	for (i = 0; i < num_desc; i++) {
-		tx_pkt = kmem_cache_zalloc(ipa3_ctx->tx_pkt_wrapper_cache,
-					   GFP_ATOMIC);
-		if (!tx_pkt) {
-			IPAERR("failed to alloc tx wrapper\n");
-			result = -ENOMEM;
-			goto failure;
-		}
-		INIT_LIST_HEAD(&tx_pkt->link);
-
-		if (i == 0) {
-			tx_pkt_first = tx_pkt;
-			tx_pkt->cnt = num_desc;
-		}
-
-		/* populate tag field */
-		if (desc[i].is_tag_status) {
-			if (ipa_populate_tag_field(&desc[i], tx_pkt,
-				&tag_pyld_ret)) {
-				IPAERR("Failed to populate tag field\n");
-				result = -EFAULT;
-				goto failure_dma_map;
-			}
-		}
-
-		tx_pkt->type = desc[i].type;
-
-		if (desc[i].type != IPA_DATA_DESC_SKB_PAGED) {
-			tx_pkt->mem.base = desc[i].pyld;
-			tx_pkt->mem.size = desc[i].len;
-
-			if (!desc[i].dma_address_valid) {
-				tx_pkt->mem.phys_base =
-					dma_map_single(ipa3_ctx->pdev,
-					tx_pkt->mem.base,
-					tx_pkt->mem.size,
-					DMA_TO_DEVICE);
-			} else {
-				tx_pkt->mem.phys_base =
-					desc[i].dma_address;
-				tx_pkt->no_unmap_dma = true;
-			}
-		} else {
-			tx_pkt->mem.base = desc[i].frag;
-			tx_pkt->mem.size = desc[i].len;
-
-			if (!desc[i].dma_address_valid) {
-				tx_pkt->mem.phys_base =
-					skb_frag_dma_map(ipa3_ctx->pdev,
-					desc[i].frag,
-					0, tx_pkt->mem.size,
-					DMA_TO_DEVICE);
-			} else {
-				tx_pkt->mem.phys_base =
-					desc[i].dma_address;
-				tx_pkt->no_unmap_dma = true;
-			}
-		}
-		if (dma_mapping_error(ipa3_ctx->pdev, tx_pkt->mem.phys_base)) {
-			IPAERR("failed to do dma map.\n");
-			result = -EFAULT;
-			goto failure_dma_map;
-		}
-
-		tx_pkt->sys = sys;
-		tx_pkt->callback = desc[i].callback;
-		tx_pkt->user1 = desc[i].user1;
-		tx_pkt->user2 = desc[i].user2;
-		tx_pkt->xmit_done = false;
-
-		list_add_tail(&tx_pkt->link, &sys->head_desc_list);
-
-		gsi_xfer[i].addr = tx_pkt->mem.phys_base;
-
-		/*
-		 * Special treatment for immediate commands, where
-		 * the structure of the descriptor is different
-		 */
-		if (desc[i].type == IPA_IMM_CMD_DESC) {
-			gsi_xfer[i].len = desc[i].opcode;
-			gsi_xfer[i].type =
-				GSI_XFER_ELEM_IMME_CMD;
-		} else {
-			gsi_xfer[i].len = desc[i].len;
-			gsi_xfer[i].type =
-				GSI_XFER_ELEM_DATA;
-		}
-
-		if (i == (num_desc - 1)) {
-			if (!sys->use_comm_evt_ring ||
-			    (sys->pkt_sent % IPA_EOT_THRESH == 0)) {
-				gsi_xfer[i].flags |=
-					GSI_XFER_FLAG_EOT;
-				gsi_xfer[i].flags |=
-					GSI_XFER_FLAG_BEI;
-			} else {
-				send_nop = true;
-			}
-			gsi_xfer[i].xfer_user_data =
-				tx_pkt_first;
-		} else {
-			gsi_xfer[i].flags |=
-				GSI_XFER_FLAG_CHAIN;
-		}
-	}
-
-	IPADBG_LOW("ch:%lu queue xfer\n", sys->ep->gsi_chan_hdl);
-	result = gsi_queue_xfer(sys->ep->gsi_chan_hdl, num_desc,
-			gsi_xfer, true);
-	if (result != GSI_STATUS_SUCCESS) {
-		IPAERR_RL("GSI xfer failed.\n");
-		result = -EFAULT;
-		goto failure;
-	}
-
-	if (send_nop && !sys->nop_pending)
-		sys->nop_pending = true;
-	else
-		send_nop = false;
-
-	sys->pkt_sent++;
-	spin_unlock_bh(&sys->spinlock);
-
-	/* set the timer for sending the NOP descriptor */
-	if (send_nop) {
-		ktime_t time = ktime_set(0, IPA_TX_SEND_COMPL_NOP_DELAY_NS);
-
-		IPADBG_LOW("scheduling timer for ch %lu\n",
-			sys->ep->gsi_chan_hdl);
-		hrtimer_start(&sys->db_timer, time, HRTIMER_MODE_REL);
-	}
-
-	/* make sure TAG process is sent before clocks are gated */
-	ipa3_ctx->tag_process_before_gating = true;
-
-	return 0;
-
-failure_dma_map:
-		kmem_cache_free(ipa3_ctx->tx_pkt_wrapper_cache, tx_pkt);
-
-failure:
-	ipahal_destroy_imm_cmd(tag_pyld_ret);
-	tx_pkt = tx_pkt_first;
-	for (j = 0; j < i; j++) {
-		next_pkt = list_next_entry(tx_pkt, link);
-		list_del(&tx_pkt->link);
-
-		if (!tx_pkt->no_unmap_dma) {
-			if (desc[j].type != IPA_DATA_DESC_SKB_PAGED) {
-				dma_unmap_single(ipa3_ctx->pdev,
-					tx_pkt->mem.phys_base,
-					tx_pkt->mem.size, DMA_TO_DEVICE);
-			} else {
-				dma_unmap_page(ipa3_ctx->pdev,
-					tx_pkt->mem.phys_base,
-					tx_pkt->mem.size,
-					DMA_TO_DEVICE);
-			}
-		}
-		kmem_cache_free(ipa3_ctx->tx_pkt_wrapper_cache, tx_pkt);
-		tx_pkt = next_pkt;
-	}
-
-	spin_unlock_bh(&sys->spinlock);
-	return result;
-}
-
-/**
- * ipa3_send_one() - Send a single descriptor
- * @sys:	system pipe context
- * @desc:	descriptor to send
- * @in_atomic:  whether caller is in atomic context
- *
- * - Allocate tx_packet wrapper
- * - transfer data to the IPA
- * - after the transfer was done the SPS will
- *   notify the sending user via ipa_sps_irq_comp_tx()
- *
- * Return codes: 0: success, -EFAULT: failure
- */
-int ipa3_send_one(struct ipa3_sys_context *sys, struct ipa3_desc *desc,
-	bool in_atomic)
-{
-	return ipa3_send(sys, 1, desc, in_atomic);
-}
-
-/**
- * ipa3_transport_irq_cmd_ack - callback function which will be called by
- * the transport driver after an immediate command is complete.
- * @user1:	pointer to the descriptor of the transfer
- * @user2:
- *
- * Complete the immediate commands completion object, this will release the
- * thread which waits on this completion object (ipa3_send_cmd())
- */
-static void ipa3_transport_irq_cmd_ack(void *user1, int user2)
-{
-	struct ipa3_desc *desc = (struct ipa3_desc *)user1;
-
-	if (WARN(!desc, "desc is NULL"))
-		return;
-
-	IPADBG_LOW("got ack for cmd=%d\n", desc->opcode);
-	complete(&desc->xfer_done);
-}
-
-/**
- * ipa3_transport_irq_cmd_ack_free - callback function which will be
- * called by the transport driver after an immediate command is complete.
- * This function will also free the completion object once it is done.
- * @tag_comp: pointer to the completion object
- * @ignored: parameter not used
- *
- * Complete the immediate commands completion object, this will release the
- * thread which waits on this completion object (ipa3_send_cmd())
- */
-static void ipa3_transport_irq_cmd_ack_free(void *tag_comp, int ignored)
-{
-	struct ipa3_tag_completion *comp = tag_comp;
-
-	if (!comp) {
-		IPAERR("comp is NULL\n");
-		return;
-	}
-
-	complete(&comp->comp);
-	if (atomic_dec_return(&comp->cnt) == 0)
-		kfree(comp);
-}
-
-/**
- * ipa3_send_cmd - send immediate commands
- * @num_desc:	number of descriptors within the desc struct
- * @descr:	descriptor structure
- *
- * Function will block till command gets ACK from IPA HW, caller needs
- * to free any resources it allocated after function returns
- * The callback in ipa3_desc should not be set by the caller
- * for this function.
- */
-int ipa3_send_cmd(u16 num_desc, struct ipa3_desc *descr)
-{
-	struct ipa3_desc *desc;
-	int i, result = 0;
-	struct ipa3_sys_context *sys;
-	int ep_idx;
-
-	for (i = 0; i < num_desc; i++)
-		IPADBG("sending imm cmd %d\n", descr[i].opcode);
-
-	ep_idx = ipa3_get_ep_mapping(IPA_CLIENT_APPS_CMD_PROD);
-	if (-1 == ep_idx) {
-		IPAERR("Client %u is not mapped\n",
-			IPA_CLIENT_APPS_CMD_PROD);
-		return -EFAULT;
-	}
-
-	sys = ipa3_ctx->ep[ep_idx].sys;
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	if (num_desc == 1) {
-		init_completion(&descr->xfer_done);
-
-		if (descr->callback || descr->user1)
-			WARN_ON(1);
-
-		descr->callback = ipa3_transport_irq_cmd_ack;
-		descr->user1 = descr;
-		if (ipa3_send_one(sys, descr, true)) {
-			IPAERR("fail to send immediate command\n");
-			result = -EFAULT;
-			goto bail;
-		}
-		wait_for_completion(&descr->xfer_done);
-	} else {
-		desc = &descr[num_desc - 1];
-		init_completion(&desc->xfer_done);
-
-		if (desc->callback || desc->user1)
-			WARN_ON(1);
-
-		desc->callback = ipa3_transport_irq_cmd_ack;
-		desc->user1 = desc;
-		if (ipa3_send(sys, num_desc, descr, true)) {
-			IPAERR("fail to send multiple immediate command set\n");
-			result = -EFAULT;
-			goto bail;
-		}
-		wait_for_completion(&desc->xfer_done);
-	}
-
-bail:
-		IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-		return result;
-}
-
-/**
- * ipa3_send_cmd_timeout - send immediate commands with limited time
- *	waiting for ACK from IPA HW
- * @num_desc:	number of descriptors within the desc struct
- * @descr:	descriptor structure
- * @timeout:	millisecond to wait till get ACK from IPA HW
- *
- * Function will block till command gets ACK from IPA HW or timeout.
- * Caller needs to free any resources it allocated after function returns
- * The callback in ipa3_desc should not be set by the caller
- * for this function.
- */
-int ipa3_send_cmd_timeout(u16 num_desc, struct ipa3_desc *descr, u32 timeout)
-{
-	struct ipa3_desc *desc;
-	int i, result = 0;
-	struct ipa3_sys_context *sys;
-	int ep_idx;
-	int completed;
-	struct ipa3_tag_completion *comp;
-
-	for (i = 0; i < num_desc; i++)
-		IPADBG("sending imm cmd %d\n", descr[i].opcode);
-
-	ep_idx = ipa3_get_ep_mapping(IPA_CLIENT_APPS_CMD_PROD);
-	if (-1 == ep_idx) {
-		IPAERR("Client %u is not mapped\n",
-			IPA_CLIENT_APPS_CMD_PROD);
-		return -EFAULT;
-	}
-
-	comp = kzalloc(sizeof(*comp), GFP_ATOMIC);
-	if (!comp)
-		return -ENOMEM;
-
-	init_completion(&comp->comp);
-
-	/* completion needs to be released from both here and in ack callback */
-	atomic_set(&comp->cnt, 2);
-
-	sys = ipa3_ctx->ep[ep_idx].sys;
-
-	if (num_desc == 1) {
-		if (descr->callback || descr->user1)
-			WARN_ON(1);
-
-		descr->callback = ipa3_transport_irq_cmd_ack_free;
-		descr->user1 = comp;
-		if (ipa3_send_one(sys, descr, true)) {
-			IPAERR("fail to send immediate command\n");
-			kfree(comp);
-			result = -EFAULT;
-			goto bail;
-		}
-	} else {
-		desc = &descr[num_desc - 1];
-
-		if (desc->callback || desc->user1)
-			WARN_ON(1);
-
-		desc->callback = ipa3_transport_irq_cmd_ack_free;
-		desc->user1 = comp;
-		if (ipa3_send(sys, num_desc, descr, true)) {
-			IPAERR("fail to send multiple immediate command set\n");
-			kfree(comp);
-			result = -EFAULT;
-			goto bail;
-		}
-	}
-
-	completed = wait_for_completion_timeout(
-		&comp->comp, msecs_to_jiffies(timeout));
-	if (!completed)
-		IPADBG("timeout waiting for imm-cmd ACK\n");
-
-	if (atomic_dec_return(&comp->cnt) == 0)
-		kfree(comp);
-
-bail:
-	return result;
-}
-
-/**
- * ipa3_handle_rx_core() - The core functionality of packet reception. This
- * function is read from multiple code paths.
- *
- * All the packets on the Rx data path are received on the IPA_A5_LAN_WAN_IN
- * endpoint. The function runs as long as there are packets in the pipe.
- * For each packet:
- *  - Disconnect the packet from the system pipe linked list
- *  - Unmap the packets skb, make it non DMAable
- *  - Free the packet from the cache
- *  - Prepare a proper skb
- *  - Call the endpoints notify function, passing the skb in the parameters
- *  - Replenish the rx cache
- */
-static int ipa3_handle_rx_core(struct ipa3_sys_context *sys, bool process_all,
-		bool in_poll_state)
-{
-	int ret;
-	int cnt = 0;
-	struct gsi_chan_xfer_notify notify = { 0 };
-
-	while ((in_poll_state ? atomic_read(&sys->curr_polling_state) :
-		!atomic_read(&sys->curr_polling_state))) {
-		if (cnt && !process_all)
-			break;
-
-		ret = ipa_poll_gsi_pkt(sys, &notify);
-		if (ret)
-			break;
-
-		if (IPA_CLIENT_IS_MEMCPY_DMA_CONS(sys->ep->client))
-			ipa3_dma_memcpy_notify(sys);
-		else if (IPA_CLIENT_IS_WLAN_CONS(sys->ep->client))
-			ipa3_wlan_wq_rx_common(sys, &notify);
-		else
-			ipa3_wq_rx_common(sys, &notify);
-
-		++cnt;
-	}
-	return cnt;
-}
-
-/**
- * ipa3_rx_switch_to_intr_mode() - Operate the Rx data path in interrupt mode
- */
-static int ipa3_rx_switch_to_intr_mode(struct ipa3_sys_context *sys)
-{
-	int ret;
-
-	atomic_set(&sys->curr_polling_state, 0);
-	ipa3_dec_release_wakelock();
-	ret = gsi_config_channel_mode(sys->ep->gsi_chan_hdl,
-		GSI_CHAN_MODE_CALLBACK);
-	if ((ret != GSI_STATUS_SUCCESS) &&
-		!atomic_read(&sys->curr_polling_state)) {
-		if (ret == -GSI_STATUS_PENDING_IRQ) {
-			ipa3_inc_acquire_wakelock();
-			atomic_set(&sys->curr_polling_state, 1);
-		} else {
-			IPAERR("Failed to switch to intr mode.\n");
-		}
-	}
-
-	return ret;
-}
-
-/**
- * ipa3_handle_rx() - handle packet reception. This function is executed in the
- * context of a work queue.
- * @work: work struct needed by the work queue
- *
- * ipa3_handle_rx_core() is run in polling mode. After all packets has been
- * received, the driver switches back to interrupt mode.
- */
-static void ipa3_handle_rx(struct ipa3_sys_context *sys)
-{
-	int inactive_cycles;
-	int cnt;
-	int ret;
-
-	ipa_pm_activate_sync(sys->pm_hdl);
-start_poll:
-	inactive_cycles = 0;
-	do {
-		cnt = ipa3_handle_rx_core(sys, true, true);
-		if (cnt == 0)
-			inactive_cycles++;
-		else
-			inactive_cycles = 0;
-
-		trace_idle_sleep_enter3(sys->ep->client);
-		usleep_range(POLLING_MIN_SLEEP_RX, POLLING_MAX_SLEEP_RX);
-		trace_idle_sleep_exit3(sys->ep->client);
-
-		/*
-		 * if pipe is out of buffers there is no point polling for
-		 * completed descs; release the worker so delayed work can
-		 * run in a timely manner
-		 */
-		if (sys->len == 0)
-			break;
-
-	} while (inactive_cycles <= POLLING_INACTIVITY_RX);
-
-	trace_poll_to_intr3(sys->ep->client);
-	ret = ipa3_rx_switch_to_intr_mode(sys);
-	if (ret == -GSI_STATUS_PENDING_IRQ)
-		goto start_poll;
-
-	ipa_pm_deferred_deactivate(sys->pm_hdl);
-}
-
-static void ipa3_switch_to_intr_rx_work_func(struct work_struct *work)
-{
-	struct delayed_work *dwork;
-	struct ipa3_sys_context *sys;
-
-	dwork = container_of(work, struct delayed_work, work);
-	sys = container_of(dwork, struct ipa3_sys_context, switch_to_intr_work);
-
-	if (sys->napi_obj) {
-		/* interrupt mode is done in ipa3_rx_poll context */
-		ipa_assert();
-	} else
-		ipa3_handle_rx(sys);
-}
-
-enum hrtimer_restart ipa3_ring_doorbell_timer_fn(struct hrtimer *param)
-{
-	struct ipa3_sys_context *sys = container_of(param,
-		struct ipa3_sys_context, db_timer);
-
-	queue_work(sys->wq, &sys->work);
-	return HRTIMER_NORESTART;
-}
-
-static void ipa_pm_sys_pipe_cb(void *p, enum ipa_pm_cb_event event)
-{
-	struct ipa3_sys_context *sys = (struct ipa3_sys_context *)p;
-
-	switch (event) {
-	case IPA_PM_CLIENT_ACTIVATED:
-		/*
-		 * this event is ignored as the sync version of activation
-		 * will be used.
-		 */
-		break;
-	case IPA_PM_REQUEST_WAKEUP:
-		/*
-		 * pipe will be unsuspended as part of
-		 * enabling IPA clocks
-		 */
-		IPADBG("calling wakeup for client %d\n", sys->ep->client);
-		if (sys->ep->client == IPA_CLIENT_APPS_WAN_CONS) {
-			IPA_ACTIVE_CLIENTS_INC_SPECIAL("PIPE_SUSPEND_WAN");
-			usleep_range(SUSPEND_MIN_SLEEP_RX,
-				SUSPEND_MAX_SLEEP_RX);
-			IPA_ACTIVE_CLIENTS_DEC_SPECIAL("PIPE_SUSPEND_WAN");
-		} else if (sys->ep->client == IPA_CLIENT_APPS_LAN_CONS) {
-			IPA_ACTIVE_CLIENTS_INC_SPECIAL("PIPE_SUSPEND_LAN");
-			usleep_range(SUSPEND_MIN_SLEEP_RX,
-				SUSPEND_MAX_SLEEP_RX);
-			IPA_ACTIVE_CLIENTS_DEC_SPECIAL("PIPE_SUSPEND_LAN");
-		} else if (sys->ep->client == IPA_CLIENT_ODL_DPL_CONS) {
-			IPA_ACTIVE_CLIENTS_INC_SPECIAL("PIPE_SUSPEND_ODL");
-			usleep_range(SUSPEND_MIN_SLEEP_RX,
-				SUSPEND_MAX_SLEEP_RX);
-			IPA_ACTIVE_CLIENTS_DEC_SPECIAL("PIPE_SUSPEND_ODL");
-		} else
-			IPAERR("Unexpected event %d\n for client %d\n",
-				event, sys->ep->client);
-		break;
-	default:
-		IPAERR("Unexpected event %d\n for client %d\n",
-			event, sys->ep->client);
-		WARN_ON(1);
-		return;
-	}
-}
-
-/**
- * ipa3_setup_sys_pipe() - Setup an IPA GPI pipe and perform
- * IPA EP configuration
- * @sys_in:	[in] input needed to setup the pipe and configure EP
- * @clnt_hdl:	[out] client handle
- *
- *  - configure the end-point registers with the supplied
- *    parameters from the user.
- *  - Creates a GPI connection with IPA.
- *  - allocate descriptor FIFO
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl)
-{
-	struct ipa3_ep_context *ep;
-	int i, ipa_ep_idx, wan_handle;
-	int result = -EINVAL;
-	struct ipahal_reg_coal_qmap_cfg qmap_cfg;
-	struct ipahal_reg_coal_evict_lru evict_lru;
-	char buff[IPA_RESOURCE_NAME_MAX];
-	struct ipa_ep_cfg ep_cfg_copy;
-
-	if (sys_in == NULL || clnt_hdl == NULL) {
-		IPAERR("NULL args\n");
-		goto fail_gen;
-	}
-
-	if (sys_in->client >= IPA_CLIENT_MAX || sys_in->desc_fifo_sz == 0) {
-		IPAERR("bad parm client:%d fifo_sz:%d\n",
-			sys_in->client, sys_in->desc_fifo_sz);
-		goto fail_gen;
-	}
-
-	ipa_ep_idx = ipa3_get_ep_mapping(sys_in->client);
-	if (ipa_ep_idx == IPA_EP_NOT_ALLOCATED) {
-		IPAERR("Invalid client.\n");
-		goto fail_gen;
-	}
-
-	ep = &ipa3_ctx->ep[ipa_ep_idx];
-	if (ep->valid == 1) {
-		IPAERR("EP %d already allocated.\n", ipa_ep_idx);
-		goto fail_gen;
-	}
-
-	/* save the input config parameters */
-	if (sys_in->client == IPA_CLIENT_APPS_WAN_COAL_CONS)
-		ep_cfg_copy = sys_in->ipa_ep_cfg;
-
-	IPA_ACTIVE_CLIENTS_INC_EP(sys_in->client);
-	memset(ep, 0, offsetof(struct ipa3_ep_context, sys));
-
-	if (!ep->sys) {
-		struct ipa_pm_register_params pm_reg;
-
-		memset(&pm_reg, 0, sizeof(pm_reg));
-		ep->sys = kzalloc(sizeof(struct ipa3_sys_context), GFP_KERNEL);
-		if (!ep->sys) {
-			IPAERR("failed to sys ctx for client %d\n",
-					sys_in->client);
-			result = -ENOMEM;
-			goto fail_and_disable_clocks;
-		}
-
-		ep->sys->ep = ep;
-		snprintf(buff, IPA_RESOURCE_NAME_MAX, "ipawq%d",
-				sys_in->client);
-		ep->sys->wq = alloc_workqueue(buff,
-				WQ_MEM_RECLAIM | WQ_UNBOUND | WQ_SYSFS, 1);
-
-		if (!ep->sys->wq) {
-			IPAERR("failed to create wq for client %d\n",
-					sys_in->client);
-			result = -EFAULT;
-			goto fail_wq;
-		}
-
-		snprintf(buff, IPA_RESOURCE_NAME_MAX, "iparepwq%d",
-				sys_in->client);
-		ep->sys->repl_wq = alloc_workqueue(buff,
-				WQ_MEM_RECLAIM | WQ_UNBOUND | WQ_SYSFS, 1);
-		if (!ep->sys->repl_wq) {
-			IPAERR("failed to create rep wq for client %d\n",
-					sys_in->client);
-			result = -EFAULT;
-			goto fail_wq2;
-		}
-
-		INIT_LIST_HEAD(&ep->sys->head_desc_list);
-		INIT_LIST_HEAD(&ep->sys->rcycl_list);
-		spin_lock_init(&ep->sys->spinlock);
-		hrtimer_init(&ep->sys->db_timer, CLOCK_MONOTONIC,
-			HRTIMER_MODE_REL);
-		ep->sys->db_timer.function = ipa3_ring_doorbell_timer_fn;
-
-		/* create IPA PM resources for handling polling mode */
-		if (IPA_CLIENT_IS_CONS(sys_in->client)) {
-			pm_reg.name = ipa_clients_strings[sys_in->client];
-			pm_reg.callback = ipa_pm_sys_pipe_cb;
-			pm_reg.user_data = ep->sys;
-			pm_reg.group = IPA_PM_GROUP_APPS;
-			result = ipa_pm_register(&pm_reg, &ep->sys->pm_hdl);
-			if (result) {
-				IPAERR("failed to create IPA PM client %d\n",
-					result);
-				goto fail_pm;
-			}
-
-			if (IPA_CLIENT_IS_APPS_CONS(sys_in->client)) {
-				result = ipa_pm_associate_ipa_cons_to_client(
-					ep->sys->pm_hdl, sys_in->client);
-				if (result) {
-					IPAERR("failed to associate\n");
-					goto fail_gen2;
-				}
-			}
-
-			result = ipa_pm_set_throughput(ep->sys->pm_hdl,
-				IPA_APPS_BW_FOR_PM);
-			if (result) {
-				IPAERR("failed to set profile IPA PM client\n");
-				goto fail_gen2;
-			}
-		}
-	} else {
-		memset(ep->sys, 0, offsetof(struct ipa3_sys_context, ep));
-	}
-
-	atomic_set(&ep->sys->xmit_eot_cnt, 0);
-	tasklet_init(&ep->sys->tasklet, ipa3_tasklet_write_done,
-			(unsigned long) ep->sys);
-	ep->skip_ep_cfg = sys_in->skip_ep_cfg;
-	if (ipa3_assign_policy(sys_in, ep->sys)) {
-		IPAERR("failed to sys ctx for client %d\n", sys_in->client);
-		result = -ENOMEM;
-		goto fail_gen2;
-	}
-
-	ep->valid = 1;
-	ep->client = sys_in->client;
-	ep->client_notify = sys_in->notify;
-	ep->sys->napi_obj = sys_in->napi_obj;
-	ep->priv = sys_in->priv;
-	ep->keep_ipa_awake = sys_in->keep_ipa_awake;
-	atomic_set(&ep->avail_fifo_desc,
-		((sys_in->desc_fifo_sz / IPA_FIFO_ELEMENT_SIZE) - 1));
-
-	if (ep->status.status_en && IPA_CLIENT_IS_CONS(ep->client) &&
-	    ep->sys->status_stat == NULL) {
-		ep->sys->status_stat =
-			kzalloc(sizeof(struct ipa3_status_stats), GFP_KERNEL);
-		if (!ep->sys->status_stat)
-			goto fail_gen2;
-	}
-
-	if (!ep->skip_ep_cfg) {
-		if (ipa3_cfg_ep(ipa_ep_idx, &sys_in->ipa_ep_cfg)) {
-			IPAERR("fail to configure EP.\n");
-			goto fail_gen2;
-		}
-		if (ipa3_cfg_ep_status(ipa_ep_idx, &ep->status)) {
-			IPAERR("fail to configure status of EP.\n");
-			goto fail_gen2;
-		}
-		IPADBG("ep %d configuration successful\n", ipa_ep_idx);
-	} else {
-		IPADBG("skipping ep %d configuration\n", ipa_ep_idx);
-	}
-
-	result = ipa_gsi_setup_channel(sys_in, ep);
-	if (result) {
-		IPAERR("Failed to setup GSI channel\n");
-		goto fail_gen2;
-	}
-
-	*clnt_hdl = ipa_ep_idx;
-
-	if (ep->sys->repl_hdlr == ipa3_fast_replenish_rx_cache) {
-		ep->sys->repl = kzalloc(sizeof(*ep->sys->repl), GFP_KERNEL);
-		if (!ep->sys->repl) {
-			IPAERR("failed to alloc repl for client %d\n",
-					sys_in->client);
-			result = -ENOMEM;
-			goto fail_gen2;
-		}
-		atomic_set(&ep->sys->repl->pending, 0);
-		ep->sys->repl->capacity = ep->sys->rx_pool_sz + 1;
-
-		ep->sys->repl->cache = kcalloc(ep->sys->repl->capacity,
-				sizeof(void *), GFP_KERNEL);
-		if (!ep->sys->repl->cache) {
-			IPAERR("ep=%d fail to alloc repl cache\n", ipa_ep_idx);
-			ep->sys->repl_hdlr = ipa3_replenish_rx_cache;
-			ep->sys->repl->capacity = 0;
-		} else {
-			atomic_set(&ep->sys->repl->head_idx, 0);
-			atomic_set(&ep->sys->repl->tail_idx, 0);
-			ipa3_wq_repl_rx(&ep->sys->repl_work);
-		}
-	}
-
-	if (IPA_CLIENT_IS_CONS(sys_in->client)) {
-		ipa3_replenish_rx_cache(ep->sys);
-		for (i = 0; i < GSI_VEID_MAX; i++)
-			INIT_LIST_HEAD(&ep->sys->pending_pkts[i]);
-	}
-
-	if (IPA_CLIENT_IS_WLAN_CONS(sys_in->client)) {
-		ipa3_alloc_wlan_rx_common_cache(IPA_WLAN_COMM_RX_POOL_LOW);
-		atomic_inc(&ipa3_ctx->wc_memb.active_clnt_cnt);
-	}
-
-	ipa3_ctx->skip_ep_cfg_shadow[ipa_ep_idx] = ep->skip_ep_cfg;
-	if (!ep->skip_ep_cfg && IPA_CLIENT_IS_PROD(sys_in->client)) {
-		if (ipa3_ctx->modem_cfg_emb_pipe_flt &&
-			sys_in->client == IPA_CLIENT_APPS_WAN_PROD)
-			IPADBG("modem cfg emb pipe flt\n");
-		else
-			ipa3_install_dflt_flt_rules(ipa_ep_idx);
-	}
-
-	result = ipa3_enable_data_path(ipa_ep_idx);
-	if (result) {
-		IPAERR("enable data path failed res=%d ep=%d.\n", result,
-			ipa_ep_idx);
-		goto fail_repl;
-	}
-
-	result = gsi_start_channel(ep->gsi_chan_hdl);
-	if (result != GSI_STATUS_SUCCESS) {
-		IPAERR("gsi_start_channel failed res=%d ep=%d.\n", result,
-			ipa_ep_idx);
-		goto fail_gen3;
-	}
-
-	if (!ep->keep_ipa_awake)
-		IPA_ACTIVE_CLIENTS_DEC_EP(sys_in->client);
-
-	IPADBG("client %d (ep: %d) connected sys=%pK\n", sys_in->client,
-			ipa_ep_idx, ep->sys);
-
-	/* configure the registers and setup the default pipe */
-	if (sys_in->client == IPA_CLIENT_APPS_WAN_COAL_CONS) {
-		evict_lru.coal_vp_lru_thrshld = 0;
-		evict_lru.coal_eviction_en = true;
-		ipahal_write_reg_fields(IPA_COAL_EVICT_LRU, &evict_lru);
-
-		qmap_cfg.mux_id_byte_sel = IPA_QMAP_ID_BYTE;
-		ipahal_write_reg_fields(IPA_COAL_QMAP_CFG, &qmap_cfg);
-
-		sys_in->client = IPA_CLIENT_APPS_WAN_CONS;
-		sys_in->ipa_ep_cfg = ep_cfg_copy;
-		result = ipa3_setup_sys_pipe(sys_in, &wan_handle);
-		if (result) {
-			IPAERR("failed to setup default coalescing pipe\n");
-			goto fail_repl;
-		}
-	}
-
-	return 0;
-
-fail_gen3:
-	ipa3_disable_data_path(ipa_ep_idx);
-fail_repl:
-	ep->sys->repl_hdlr = ipa3_replenish_rx_cache;
-	ep->sys->repl->capacity = 0;
-	kfree(ep->sys->repl);
-fail_gen2:
-	ipa_pm_deregister(ep->sys->pm_hdl);
-fail_pm:
-	destroy_workqueue(ep->sys->repl_wq);
-fail_wq2:
-	destroy_workqueue(ep->sys->wq);
-fail_wq:
-	kfree(ep->sys);
-	memset(&ipa3_ctx->ep[ipa_ep_idx], 0, sizeof(struct ipa3_ep_context));
-fail_and_disable_clocks:
-	IPA_ACTIVE_CLIENTS_DEC_EP(sys_in->client);
-fail_gen:
-	return result;
-}
-
-/**
- * ipa3_teardown_sys_pipe() - Teardown the GPI pipe and cleanup IPA EP
- * @clnt_hdl:	[in] the handle obtained from ipa3_setup_sys_pipe
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa3_teardown_sys_pipe(u32 clnt_hdl)
-{
-	struct ipa3_ep_context *ep;
-	int empty;
-	int result;
-	int i;
-
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-	    ipa3_ctx->ep[clnt_hdl].valid == 0) {
-		IPAERR("bad parm.\n");
-		return -EINVAL;
-	}
-
-	ep = &ipa3_ctx->ep[clnt_hdl];
-
-	if (!ep->keep_ipa_awake)
-		IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	ipa3_disable_data_path(clnt_hdl);
-
-	if (IPA_CLIENT_IS_PROD(ep->client)) {
-		do {
-			spin_lock_bh(&ep->sys->spinlock);
-			empty = list_empty(&ep->sys->head_desc_list);
-			spin_unlock_bh(&ep->sys->spinlock);
-			if (!empty)
-				usleep_range(95, 105);
-			else
-				break;
-		} while (1);
-	}
-
-	/* channel stop might fail on timeout if IPA is busy */
-	for (i = 0; i < IPA_GSI_CHANNEL_STOP_MAX_RETRY; i++) {
-		result = ipa3_stop_gsi_channel(clnt_hdl);
-		if (result == GSI_STATUS_SUCCESS)
-			break;
-
-		if (result != -GSI_STATUS_AGAIN &&
-			result != -GSI_STATUS_TIMED_OUT)
-			break;
-	}
-
-	if (result != GSI_STATUS_SUCCESS) {
-		IPAERR("GSI stop chan err: %d.\n", result);
-		ipa_assert();
-		return result;
-	}
-
-	if (ep->sys->napi_obj) {
-		do {
-			usleep_range(95, 105);
-		} while (atomic_read(&ep->sys->curr_polling_state));
-	}
-
-	if (IPA_CLIENT_IS_CONS(ep->client))
-		cancel_delayed_work_sync(&ep->sys->replenish_rx_work);
-	flush_workqueue(ep->sys->wq);
-
-	/* tear down the default pipe before we reset the channel*/
-	if (ep->client == IPA_CLIENT_APPS_WAN_COAL_CONS) {
-		i = ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_CONS);
-
-		if (i == IPA_EP_NOT_ALLOCATED) {
-			IPAERR("failed to get idx");
-			return i;
-		}
-
-		result = ipa3_teardown_coal_def_pipe(i);
-		if (result) {
-			IPAERR("failed to teardown default coal pipe\n");
-			return result;
-		}
-	}
-
-	result = ipa3_reset_gsi_channel(clnt_hdl);
-	if (result != GSI_STATUS_SUCCESS) {
-		IPAERR("Failed to reset chan: %d.\n", result);
-		ipa_assert();
-		return result;
-	}
-	dma_free_coherent(ipa3_ctx->pdev,
-		ep->gsi_mem_info.chan_ring_len,
-		ep->gsi_mem_info.chan_ring_base_vaddr,
-		ep->gsi_mem_info.chan_ring_base_addr);
-	result = gsi_dealloc_channel(ep->gsi_chan_hdl);
-	if (result != GSI_STATUS_SUCCESS) {
-		IPAERR("Failed to dealloc chan: %d.\n", result);
-		ipa_assert();
-		return result;
-	}
-
-	/* free event ring only when it is present */
-	if (ep->sys->use_comm_evt_ring) {
-		ipa3_ctx->gsi_evt_comm_ring_rem +=
-			ep->gsi_mem_info.chan_ring_len;
-	} else if (ep->gsi_evt_ring_hdl != ~0) {
-		result = gsi_reset_evt_ring(ep->gsi_evt_ring_hdl);
-		if (WARN(result != GSI_STATUS_SUCCESS, "reset evt %d", result))
-			return result;
-
-		dma_free_coherent(ipa3_ctx->pdev,
-			ep->gsi_mem_info.evt_ring_len,
-			ep->gsi_mem_info.evt_ring_base_vaddr,
-			ep->gsi_mem_info.evt_ring_base_addr);
-		result = gsi_dealloc_evt_ring(ep->gsi_evt_ring_hdl);
-		if (WARN(result != GSI_STATUS_SUCCESS, "deall evt %d", result))
-			return result;
-	}
-	if (ep->sys->repl_wq)
-		flush_workqueue(ep->sys->repl_wq);
-	if (IPA_CLIENT_IS_CONS(ep->client))
-		ipa3_cleanup_rx(ep->sys);
-
-	if (!ep->skip_ep_cfg && IPA_CLIENT_IS_PROD(ep->client)) {
-		if (ipa3_ctx->modem_cfg_emb_pipe_flt &&
-			ep->client == IPA_CLIENT_APPS_WAN_PROD)
-			IPADBG("modem cfg emb pipe flt\n");
-		else
-			ipa3_delete_dflt_flt_rules(clnt_hdl);
-	}
-
-	if (IPA_CLIENT_IS_WLAN_CONS(ep->client))
-		atomic_dec(&ipa3_ctx->wc_memb.active_clnt_cnt);
-
-	memset(&ep->wstats, 0, sizeof(struct ipa3_wlan_stats));
-
-	if (!atomic_read(&ipa3_ctx->wc_memb.active_clnt_cnt))
-		ipa3_cleanup_wlan_rx_common_cache();
-
-	ep->valid = 0;
-
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	IPADBG("client (ep: %d) disconnected\n", clnt_hdl);
-
-	return 0;
-}
-
-/**
- * ipa3_teardown_coal_def_pipe() - Teardown the APPS_WAN_COAL_CONS
- *				   default GPI pipe and cleanup IPA EP
- *				   called after the coalesced pipe is destroyed.
- * @clnt_hdl:	[in] the handle obtained from ipa3_setup_sys_pipe
- *
- * Returns:	0 on success, negative on failure
- */
-static int ipa3_teardown_coal_def_pipe(u32 clnt_hdl)
-{
-	struct ipa3_ep_context *ep;
-	int result;
-	int i;
-
-	ep = &ipa3_ctx->ep[clnt_hdl];
-
-	ipa3_disable_data_path(clnt_hdl);
-
-	/* channel stop might fail on timeout if IPA is busy */
-	for (i = 0; i < IPA_GSI_CHANNEL_STOP_MAX_RETRY; i++) {
-		result = ipa3_stop_gsi_channel(clnt_hdl);
-		if (result == GSI_STATUS_SUCCESS)
-			break;
-
-		if (result != -GSI_STATUS_AGAIN &&
-		    result != -GSI_STATUS_TIMED_OUT)
-			break;
-	}
-
-	if (result != GSI_STATUS_SUCCESS) {
-		IPAERR("GSI stop chan err: %d.\n", result);
-		ipa_assert();
-		return result;
-	}
-	result = ipa3_reset_gsi_channel(clnt_hdl);
-	if (result != GSI_STATUS_SUCCESS) {
-		IPAERR("Failed to reset chan: %d.\n", result);
-		ipa_assert();
-		return result;
-	}
-	dma_free_coherent(ipa3_ctx->pdev,
-		ep->gsi_mem_info.chan_ring_len,
-		ep->gsi_mem_info.chan_ring_base_vaddr,
-		ep->gsi_mem_info.chan_ring_base_addr);
-	result = gsi_dealloc_channel(ep->gsi_chan_hdl);
-	if (result != GSI_STATUS_SUCCESS) {
-		IPAERR("Failed to dealloc chan: %d.\n", result);
-		ipa_assert();
-		return result;
-	}
-
-	if (IPA_CLIENT_IS_CONS(ep->client))
-		cancel_delayed_work_sync(&ep->sys->replenish_rx_work);
-
-	flush_workqueue(ep->sys->wq);
-
-	if (ep->sys->repl_wq)
-		flush_workqueue(ep->sys->repl_wq);
-	if (IPA_CLIENT_IS_CONS(ep->client))
-		ipa3_cleanup_rx(ep->sys);
-
-	ep->valid = 0;
-
-	IPADBG("client (ep: %d) disconnected\n", clnt_hdl);
-
-	return 0;
-}
-
-/**
- * ipa3_tx_comp_usr_notify_release() - Callback function which will call the
- * user supplied callback function to release the skb, or release it on
- * its own if no callback function was supplied.
- * @user1
- * @user2
- *
- * This notified callback is for the destination client.
- */
-static void ipa3_tx_comp_usr_notify_release(void *user1, int user2)
-{
-	struct sk_buff *skb = (struct sk_buff *)user1;
-	int ep_idx = user2;
-
-	IPADBG_LOW("skb=%pK ep=%d\n", skb, ep_idx);
-
-	IPA_STATS_INC_CNT(ipa3_ctx->stats.tx_pkts_compl);
-
-	if (ipa3_ctx->ep[ep_idx].client_notify)
-		ipa3_ctx->ep[ep_idx].client_notify(ipa3_ctx->ep[ep_idx].priv,
-				IPA_WRITE_DONE, (unsigned long)skb);
-	else
-		dev_kfree_skb_any(skb);
-}
-
-void ipa3_tx_cmd_comp(void *user1, int user2)
-{
-	ipahal_destroy_imm_cmd(user1);
-}
-
-/**
- * ipa3_tx_dp() - Data-path tx handler
- * @dst:	[in] which IPA destination to route tx packets to
- * @skb:	[in] the packet to send
- * @metadata:	[in] TX packet meta-data
- *
- * Data-path tx handler, this is used for both SW data-path which by-passes most
- * IPA HW blocks AND the regular HW data-path for WLAN AMPDU traffic only. If
- * dst is a "valid" CONS type, then SW data-path is used. If dst is the
- * WLAN_AMPDU PROD type, then HW data-path for WLAN AMPDU is used. Anything else
- * is an error. For errors, client needs to free the skb as needed. For success,
- * IPA driver will later invoke client callback if one was supplied. That
- * callback should free the skb. If no callback supplied, IPA driver will free
- * the skb internally
- *
- * The function will use two descriptors for this send command
- * (for A5_WLAN_AMPDU_PROD only one desciprtor will be sent),
- * the first descriptor will be used to inform the IPA hardware that
- * apps need to push data into the IPA (IP_PACKET_INIT immediate command).
- * Once this send was done from transport point-of-view the IPA driver will
- * get notified by the supplied callback.
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa3_tx_dp(enum ipa_client_type dst, struct sk_buff *skb,
-		struct ipa_tx_meta *meta)
-{
-	struct ipa3_desc *desc;
-	struct ipa3_desc _desc[3];
-	int dst_ep_idx;
-	struct ipahal_imm_cmd_pyld *cmd_pyld = NULL;
-	struct ipa3_sys_context *sys;
-	int src_ep_idx;
-	int num_frags, f;
-	const struct ipa_gsi_ep_config *gsi_ep;
-	int data_idx;
-	unsigned int max_desc;
-
-	if (unlikely(!ipa3_ctx)) {
-		IPAERR("IPA3 driver was not initialized\n");
-		return -EINVAL;
-	}
-
-	if (skb->len == 0) {
-		IPAERR("packet size is 0\n");
-		return -EINVAL;
-	}
-
-	/*
-	 * USB_CONS: PKT_INIT ep_idx = dst pipe
-	 * Q6_CONS: PKT_INIT ep_idx = sender pipe
-	 * A5_LAN_WAN_PROD: HW path ep_idx = sender pipe
-	 *
-	 * LAN TX: all PKT_INIT
-	 * WAN TX: PKT_INIT (cmd) + HW (data)
-	 *
-	 */
-	if (IPA_CLIENT_IS_CONS(dst)) {
-		src_ep_idx = ipa3_get_ep_mapping(IPA_CLIENT_APPS_LAN_PROD);
-		if (-1 == src_ep_idx) {
-			IPAERR("Client %u is not mapped\n",
-				IPA_CLIENT_APPS_LAN_PROD);
-			goto fail_gen;
-		}
-		dst_ep_idx = ipa3_get_ep_mapping(dst);
-	} else {
-		src_ep_idx = ipa3_get_ep_mapping(dst);
-		if (-1 == src_ep_idx) {
-			IPAERR("Client %u is not mapped\n", dst);
-			goto fail_gen;
-		}
-		if (meta && meta->pkt_init_dst_ep_valid)
-			dst_ep_idx = meta->pkt_init_dst_ep;
-		else
-			dst_ep_idx = -1;
-	}
-
-	sys = ipa3_ctx->ep[src_ep_idx].sys;
-
-	if (!sys || !sys->ep->valid) {
-		IPAERR_RL("pipe %d not valid\n", src_ep_idx);
-		goto fail_pipe_not_valid;
-	}
-
-	num_frags = skb_shinfo(skb)->nr_frags;
-	/*
-	 * make sure TLV FIFO supports the needed frags.
-	 * 2 descriptors are needed for IP_PACKET_INIT and TAG_STATUS.
-	 * 1 descriptor needed for the linear portion of skb.
-	 */
-	gsi_ep = ipa3_get_gsi_ep_info(ipa3_ctx->ep[src_ep_idx].client);
-	if (unlikely(gsi_ep == NULL)) {
-		IPAERR("failed to get EP %d GSI info\n", src_ep_idx);
-		goto fail_gen;
-	}
-	max_desc =  gsi_ep->ipa_if_tlv;
-	if (gsi_ep->prefetch_mode == GSI_SMART_PRE_FETCH ||
-		gsi_ep->prefetch_mode == GSI_FREE_PRE_FETCH)
-		max_desc -= gsi_ep->prefetch_threshold;
-	if (num_frags + 3 > max_desc) {
-		if (skb_linearize(skb)) {
-			IPAERR("Failed to linear skb with %d frags\n",
-				num_frags);
-			goto fail_gen;
-		}
-		num_frags = 0;
-	}
-	if (num_frags) {
-		/* 1 desc for tag to resolve status out-of-order issue;
-		 * 1 desc is needed for the linear portion of skb;
-		 * 1 desc may be needed for the PACKET_INIT;
-		 * 1 desc for each frag
-		 */
-		desc = kzalloc(sizeof(*desc) * (num_frags + 3), GFP_ATOMIC);
-		if (!desc) {
-			IPAERR("failed to alloc desc array\n");
-			goto fail_gen;
-		}
-	} else {
-		memset(_desc, 0, 3 * sizeof(struct ipa3_desc));
-		desc = &_desc[0];
-	}
-
-	if (dst_ep_idx != -1) {
-		int skb_idx;
-
-		/* SW data path */
-		data_idx = 0;
-		if (sys->policy == IPA_POLICY_NOINTR_MODE) {
-			/*
-			 * For non-interrupt mode channel (where there is no
-			 * event ring) TAG STATUS are used for completion
-			 * notification. IPA will generate a status packet with
-			 * tag info as a result of the TAG STATUS command.
-			 */
-			desc[data_idx].is_tag_status = true;
-			data_idx++;
-		}
-		desc[data_idx].opcode = ipa3_ctx->pkt_init_imm_opcode;
-		desc[data_idx].dma_address_valid = true;
-		desc[data_idx].dma_address = ipa3_ctx->pkt_init_imm[dst_ep_idx];
-		desc[data_idx].type = IPA_IMM_CMD_DESC;
-		desc[data_idx].callback = NULL;
-		data_idx++;
-		desc[data_idx].pyld = skb->data;
-		desc[data_idx].len = skb_headlen(skb);
-		desc[data_idx].type = IPA_DATA_DESC_SKB;
-		desc[data_idx].callback = ipa3_tx_comp_usr_notify_release;
-		desc[data_idx].user1 = skb;
-		desc[data_idx].user2 = (meta && meta->pkt_init_dst_ep_valid &&
-				meta->pkt_init_dst_ep_remote) ?
-				src_ep_idx :
-				dst_ep_idx;
-		if (meta && meta->dma_address_valid) {
-			desc[data_idx].dma_address_valid = true;
-			desc[data_idx].dma_address = meta->dma_address;
-		}
-
-		skb_idx = data_idx;
-		data_idx++;
-
-		for (f = 0; f < num_frags; f++) {
-			desc[data_idx + f].frag = &skb_shinfo(skb)->frags[f];
-			desc[data_idx + f].type = IPA_DATA_DESC_SKB_PAGED;
-			desc[data_idx + f].len =
-				skb_frag_size(desc[data_idx + f].frag);
-		}
-		/* don't free skb till frag mappings are released */
-		if (num_frags) {
-			desc[data_idx + f - 1].callback =
-				desc[skb_idx].callback;
-			desc[data_idx + f - 1].user1 = desc[skb_idx].user1;
-			desc[data_idx + f - 1].user2 = desc[skb_idx].user2;
-			desc[skb_idx].callback = NULL;
-		}
-
-		if (ipa3_send(sys, num_frags + data_idx, desc, true)) {
-			IPAERR_RL("fail to send skb %pK num_frags %u SWP\n",
-				skb, num_frags);
-			goto fail_send;
-		}
-		IPA_STATS_INC_CNT(ipa3_ctx->stats.tx_sw_pkts);
-	} else {
-		/* HW data path */
-		data_idx = 0;
-		if (sys->policy == IPA_POLICY_NOINTR_MODE) {
-			/*
-			 * For non-interrupt mode channel (where there is no
-			 * event ring) TAG STATUS are used for completion
-			 * notification. IPA will generate a status packet with
-			 * tag info as a result of the TAG STATUS command.
-			 */
-			desc[data_idx].is_tag_status = true;
-			data_idx++;
-		}
-		desc[data_idx].pyld = skb->data;
-		desc[data_idx].len = skb_headlen(skb);
-		desc[data_idx].type = IPA_DATA_DESC_SKB;
-		desc[data_idx].callback = ipa3_tx_comp_usr_notify_release;
-		desc[data_idx].user1 = skb;
-		desc[data_idx].user2 = src_ep_idx;
-
-		if (meta && meta->dma_address_valid) {
-			desc[data_idx].dma_address_valid = true;
-			desc[data_idx].dma_address = meta->dma_address;
-		}
-		if (num_frags == 0) {
-			if (ipa3_send(sys, data_idx + 1, desc, true)) {
-				IPAERR("fail to send skb %pK HWP\n", skb);
-				goto fail_mem;
-			}
-		} else {
-			for (f = 0; f < num_frags; f++) {
-				desc[data_idx+f+1].frag =
-					&skb_shinfo(skb)->frags[f];
-				desc[data_idx+f+1].type =
-					IPA_DATA_DESC_SKB_PAGED;
-				desc[data_idx+f+1].len =
-					skb_frag_size(desc[data_idx+f+1].frag);
-			}
-			/* don't free skb till frag mappings are released */
-			desc[data_idx+f].callback = desc[data_idx].callback;
-			desc[data_idx+f].user1 = desc[data_idx].user1;
-			desc[data_idx+f].user2 = desc[data_idx].user2;
-			desc[data_idx].callback = NULL;
-
-			if (ipa3_send(sys, num_frags + data_idx + 1,
-				desc, true)) {
-				IPAERR("fail to send skb %pK num_frags %u\n",
-					skb, num_frags);
-				goto fail_mem;
-			}
-		}
-		IPA_STATS_INC_CNT(ipa3_ctx->stats.tx_hw_pkts);
-	}
-
-	if (num_frags) {
-		kfree(desc);
-		IPA_STATS_INC_CNT(ipa3_ctx->stats.tx_non_linear);
-	}
-	return 0;
-
-fail_send:
-	ipahal_destroy_imm_cmd(cmd_pyld);
-fail_mem:
-	if (num_frags)
-		kfree(desc);
-fail_gen:
-	return -EFAULT;
-fail_pipe_not_valid:
-	return -EPIPE;
-}
-
-static void ipa3_wq_handle_rx(struct work_struct *work)
-{
-	struct ipa3_sys_context *sys;
-
-	sys = container_of(work, struct ipa3_sys_context, work);
-
-	if (sys->napi_obj) {
-		ipa_pm_activate_sync(sys->pm_hdl);
-		napi_schedule(sys->napi_obj);
-	} else
-		ipa3_handle_rx(sys);
-}
-
-static void ipa3_wq_repl_rx(struct work_struct *work)
-{
-	struct ipa3_sys_context *sys;
-	void *ptr;
-	struct ipa3_rx_pkt_wrapper *rx_pkt;
-	gfp_t flag = GFP_KERNEL;
-	u32 next;
-	u32 curr;
-
-	sys = container_of(work, struct ipa3_sys_context, repl_work);
-	atomic_set(&sys->repl->pending, 0);
-	curr = atomic_read(&sys->repl->tail_idx);
-
-begin:
-	while (1) {
-		next = (curr + 1) % sys->repl->capacity;
-		if (next == atomic_read(&sys->repl->head_idx))
-			goto fail_kmem_cache_alloc;
-
-		rx_pkt = kmem_cache_zalloc(ipa3_ctx->rx_pkt_wrapper_cache,
-					   flag);
-		if (!rx_pkt)
-			goto fail_kmem_cache_alloc;
-
-		INIT_WORK(&rx_pkt->work, ipa3_wq_rx_avail);
-		rx_pkt->sys = sys;
-
-		rx_pkt->data.skb = sys->get_skb(sys->rx_buff_sz, flag);
-		if (rx_pkt->data.skb == NULL)
-			goto fail_skb_alloc;
-
-		ptr = skb_put(rx_pkt->data.skb, sys->rx_buff_sz);
-		rx_pkt->data.dma_addr = dma_map_single(ipa3_ctx->pdev, ptr,
-						     sys->rx_buff_sz,
-						     DMA_FROM_DEVICE);
-		if (dma_mapping_error(ipa3_ctx->pdev, rx_pkt->data.dma_addr)) {
-			pr_err_ratelimited("%s dma map fail %pK for %pK sys=%pK\n",
-			       __func__, (void *)rx_pkt->data.dma_addr,
-			       ptr, sys);
-			goto fail_dma_mapping;
-		}
-
-		sys->repl->cache[curr] = rx_pkt;
-		curr = next;
-		/* ensure write is done before setting tail index */
-		mb();
-		atomic_set(&sys->repl->tail_idx, next);
-	}
-
-	return;
-
-fail_dma_mapping:
-	sys->free_skb(rx_pkt->data.skb);
-fail_skb_alloc:
-	kmem_cache_free(ipa3_ctx->rx_pkt_wrapper_cache, rx_pkt);
-fail_kmem_cache_alloc:
-	if (atomic_read(&sys->repl->tail_idx) ==
-			atomic_read(&sys->repl->head_idx)) {
-		if (sys->ep->client == IPA_CLIENT_APPS_WAN_CONS ||
-			sys->ep->client == IPA_CLIENT_APPS_WAN_COAL_CONS)
-			IPA_STATS_INC_CNT(ipa3_ctx->stats.wan_repl_rx_empty);
-		else if (sys->ep->client == IPA_CLIENT_APPS_LAN_CONS)
-			IPA_STATS_INC_CNT(ipa3_ctx->stats.lan_repl_rx_empty);
-		pr_err_ratelimited("%s sys=%pK repl ring empty\n",
-				__func__, sys);
-		goto begin;
-	}
-}
-
-static void ipa3_replenish_wlan_rx_cache(struct ipa3_sys_context *sys)
-{
-	struct ipa3_rx_pkt_wrapper *rx_pkt = NULL;
-	struct ipa3_rx_pkt_wrapper *tmp;
-	int ret;
-	struct gsi_xfer_elem gsi_xfer_elem_one;
-	u32 rx_len_cached = 0;
-
-	IPADBG_LOW("\n");
-
-	spin_lock_bh(&ipa3_ctx->wc_memb.wlan_spinlock);
-	rx_len_cached = sys->len;
-
-	if (rx_len_cached < sys->rx_pool_sz) {
-		list_for_each_entry_safe(rx_pkt, tmp,
-			&ipa3_ctx->wc_memb.wlan_comm_desc_list, link) {
-			list_del(&rx_pkt->link);
-
-			if (ipa3_ctx->wc_memb.wlan_comm_free_cnt > 0)
-				ipa3_ctx->wc_memb.wlan_comm_free_cnt--;
-
-			rx_pkt->len = 0;
-			rx_pkt->sys = sys;
-
-			memset(&gsi_xfer_elem_one, 0,
-				sizeof(gsi_xfer_elem_one));
-			gsi_xfer_elem_one.addr = rx_pkt->data.dma_addr;
-			gsi_xfer_elem_one.len = IPA_WLAN_RX_BUFF_SZ;
-			gsi_xfer_elem_one.flags |= GSI_XFER_FLAG_EOT;
-			gsi_xfer_elem_one.flags |= GSI_XFER_FLAG_EOB;
-			gsi_xfer_elem_one.type = GSI_XFER_ELEM_DATA;
-			gsi_xfer_elem_one.xfer_user_data = rx_pkt;
-
-			ret = gsi_queue_xfer(sys->ep->gsi_chan_hdl, 1,
-				&gsi_xfer_elem_one, true);
-
-			if (ret) {
-				IPAERR("failed to provide buffer: %d\n", ret);
-				goto fail_provide_rx_buffer;
-			}
-
-			rx_len_cached = ++sys->len;
-
-			if (rx_len_cached >= sys->rx_pool_sz) {
-				spin_unlock_bh(
-					&ipa3_ctx->wc_memb.wlan_spinlock);
-				return;
-			}
-		}
-	}
-	spin_unlock_bh(&ipa3_ctx->wc_memb.wlan_spinlock);
-
-	if (rx_len_cached < sys->rx_pool_sz &&
-			ipa3_ctx->wc_memb.wlan_comm_total_cnt <
-			 IPA_WLAN_COMM_RX_POOL_HIGH) {
-		ipa3_replenish_rx_cache(sys);
-		ipa3_ctx->wc_memb.wlan_comm_total_cnt +=
-			(sys->rx_pool_sz - rx_len_cached);
-	}
-
-	return;
-
-fail_provide_rx_buffer:
-	list_del(&rx_pkt->link);
-	spin_unlock_bh(&ipa3_ctx->wc_memb.wlan_spinlock);
-}
-
-static void ipa3_cleanup_wlan_rx_common_cache(void)
-{
-	struct ipa3_rx_pkt_wrapper *rx_pkt;
-	struct ipa3_rx_pkt_wrapper *tmp;
-
-	spin_lock_bh(&ipa3_ctx->wc_memb.wlan_spinlock);
-
-	list_for_each_entry_safe(rx_pkt, tmp,
-		&ipa3_ctx->wc_memb.wlan_comm_desc_list, link) {
-		list_del(&rx_pkt->link);
-		dma_unmap_single(ipa3_ctx->pdev, rx_pkt->data.dma_addr,
-				IPA_WLAN_RX_BUFF_SZ, DMA_FROM_DEVICE);
-		dev_kfree_skb_any(rx_pkt->data.skb);
-		kmem_cache_free(ipa3_ctx->rx_pkt_wrapper_cache, rx_pkt);
-		ipa3_ctx->wc_memb.wlan_comm_free_cnt--;
-		ipa3_ctx->wc_memb.wlan_comm_total_cnt--;
-	}
-	ipa3_ctx->wc_memb.total_tx_pkts_freed = 0;
-
-	if (ipa3_ctx->wc_memb.wlan_comm_free_cnt != 0)
-		IPAERR("wlan comm buff free cnt: %d\n",
-			ipa3_ctx->wc_memb.wlan_comm_free_cnt);
-
-	if (ipa3_ctx->wc_memb.wlan_comm_total_cnt != 0)
-		IPAERR("wlan comm buff total cnt: %d\n",
-			ipa3_ctx->wc_memb.wlan_comm_total_cnt);
-
-	spin_unlock_bh(&ipa3_ctx->wc_memb.wlan_spinlock);
-
-}
-
-static void ipa3_alloc_wlan_rx_common_cache(u32 size)
-{
-	void *ptr;
-	struct ipa3_rx_pkt_wrapper *rx_pkt;
-	int rx_len_cached = 0;
-	gfp_t flag = GFP_NOWAIT | __GFP_NOWARN;
-
-	rx_len_cached = ipa3_ctx->wc_memb.wlan_comm_total_cnt;
-	while (rx_len_cached < size) {
-		rx_pkt = kmem_cache_zalloc(ipa3_ctx->rx_pkt_wrapper_cache,
-					   flag);
-		if (!rx_pkt)
-			goto fail_kmem_cache_alloc;
-
-		INIT_LIST_HEAD(&rx_pkt->link);
-		INIT_WORK(&rx_pkt->work, ipa3_wq_rx_avail);
-
-		rx_pkt->data.skb =
-			ipa3_get_skb_ipa_rx(IPA_WLAN_RX_BUFF_SZ,
-						flag);
-		if (rx_pkt->data.skb == NULL) {
-			IPAERR("failed to alloc skb\n");
-			goto fail_skb_alloc;
-		}
-		ptr = skb_put(rx_pkt->data.skb, IPA_WLAN_RX_BUFF_SZ);
-		rx_pkt->data.dma_addr = dma_map_single(ipa3_ctx->pdev, ptr,
-				IPA_WLAN_RX_BUFF_SZ, DMA_FROM_DEVICE);
-		if (dma_mapping_error(ipa3_ctx->pdev, rx_pkt->data.dma_addr)) {
-			IPAERR("dma_map_single failure %pK for %pK\n",
-			       (void *)rx_pkt->data.dma_addr, ptr);
-			goto fail_dma_mapping;
-		}
-
-		spin_lock_bh(&ipa3_ctx->wc_memb.wlan_spinlock);
-		list_add_tail(&rx_pkt->link,
-			&ipa3_ctx->wc_memb.wlan_comm_desc_list);
-		rx_len_cached = ++ipa3_ctx->wc_memb.wlan_comm_total_cnt;
-
-		ipa3_ctx->wc_memb.wlan_comm_free_cnt++;
-		spin_unlock_bh(&ipa3_ctx->wc_memb.wlan_spinlock);
-
-	}
-
-	return;
-
-fail_dma_mapping:
-	dev_kfree_skb_any(rx_pkt->data.skb);
-fail_skb_alloc:
-	kmem_cache_free(ipa3_ctx->rx_pkt_wrapper_cache, rx_pkt);
-fail_kmem_cache_alloc:
-	return;
-}
-
-
-/**
- * ipa3_replenish_rx_cache() - Replenish the Rx packets cache.
- *
- * The function allocates buffers in the rx_pkt_wrapper_cache cache until there
- * are IPA_RX_POOL_CEIL buffers in the cache.
- *   - Allocate a buffer in the cache
- *   - Initialized the packets link
- *   - Initialize the packets work struct
- *   - Allocate the packets socket buffer (skb)
- *   - Fill the packets skb with data
- *   - Make the packet DMAable
- *   - Add the packet to the system pipe linked list
- */
-static void ipa3_replenish_rx_cache(struct ipa3_sys_context *sys)
-{
-	void *ptr;
-	struct ipa3_rx_pkt_wrapper *rx_pkt;
-	int ret;
-	int idx = 0;
-	int rx_len_cached = 0;
-	struct gsi_xfer_elem gsi_xfer_elem_array[IPA_REPL_XFER_MAX];
-	gfp_t flag = GFP_NOWAIT | __GFP_NOWARN;
-
-	rx_len_cached = sys->len;
-
-	/* start replenish only when buffers go lower than the threshold */
-	if (sys->rx_pool_sz - sys->len < IPA_REPL_XFER_THRESH)
-		return;
-
-
-	while (rx_len_cached < sys->rx_pool_sz) {
-		rx_pkt = kmem_cache_zalloc(ipa3_ctx->rx_pkt_wrapper_cache,
-					   flag);
-		if (!rx_pkt)
-			goto fail_kmem_cache_alloc;
-
-		INIT_WORK(&rx_pkt->work, ipa3_wq_rx_avail);
-		rx_pkt->sys = sys;
-
-		rx_pkt->data.skb = sys->get_skb(sys->rx_buff_sz, flag);
-		if (rx_pkt->data.skb == NULL) {
-			IPAERR("failed to alloc skb\n");
-			goto fail_skb_alloc;
-		}
-		ptr = skb_put(rx_pkt->data.skb, sys->rx_buff_sz);
-		rx_pkt->data.dma_addr = dma_map_single(ipa3_ctx->pdev, ptr,
-						     sys->rx_buff_sz,
-						     DMA_FROM_DEVICE);
-		if (dma_mapping_error(ipa3_ctx->pdev, rx_pkt->data.dma_addr)) {
-			IPAERR("dma_map_single failure %pK for %pK\n",
-			       (void *)rx_pkt->data.dma_addr, ptr);
-			goto fail_dma_mapping;
-		}
-
-		gsi_xfer_elem_array[idx].addr = rx_pkt->data.dma_addr;
-		gsi_xfer_elem_array[idx].len = sys->rx_buff_sz;
-		gsi_xfer_elem_array[idx].flags = GSI_XFER_FLAG_EOT;
-		gsi_xfer_elem_array[idx].flags |= GSI_XFER_FLAG_EOB;
-		gsi_xfer_elem_array[idx].flags |= GSI_XFER_FLAG_BEI;
-		gsi_xfer_elem_array[idx].type = GSI_XFER_ELEM_DATA;
-		gsi_xfer_elem_array[idx].xfer_user_data = rx_pkt;
-		idx++;
-		rx_len_cached++;
-		/*
-		 * gsi_xfer_elem_buffer has a size of IPA_REPL_XFER_MAX.
-		 * If this size is reached we need to queue the xfers.
-		 */
-		if (idx == IPA_REPL_XFER_MAX) {
-			ret = gsi_queue_xfer(sys->ep->gsi_chan_hdl, idx,
-				gsi_xfer_elem_array, false);
-			if (ret != GSI_STATUS_SUCCESS) {
-				/* we don't expect this will happen */
-				IPAERR("failed to provide buffer: %d\n", ret);
-				WARN_ON(1);
-				break;
-			}
-			idx = 0;
-		}
-	}
-	goto done;
-
-fail_dma_mapping:
-	sys->free_skb(rx_pkt->data.skb);
-fail_skb_alloc:
-	kmem_cache_free(ipa3_ctx->rx_pkt_wrapper_cache, rx_pkt);
-fail_kmem_cache_alloc:
-	if (rx_len_cached == 0)
-		queue_delayed_work(sys->wq, &sys->replenish_rx_work,
-				msecs_to_jiffies(1));
-done:
-	/* only ring doorbell once here */
-	ret = gsi_queue_xfer(sys->ep->gsi_chan_hdl, idx,
-		gsi_xfer_elem_array, true);
-	if (ret == GSI_STATUS_SUCCESS) {
-		sys->len = rx_len_cached;
-	} else {
-		/* we don't expect this will happen */
-		IPAERR("failed to provide buffer: %d\n", ret);
-		WARN_ON(1);
-	}
-}
-
-static void ipa3_replenish_rx_cache_recycle(struct ipa3_sys_context *sys)
-{
-	void *ptr;
-	struct ipa3_rx_pkt_wrapper *rx_pkt;
-	int ret;
-	int idx = 0;
-	int rx_len_cached = 0;
-	struct gsi_xfer_elem gsi_xfer_elem_array[IPA_REPL_XFER_MAX];
-	gfp_t flag = GFP_NOWAIT | __GFP_NOWARN;
-
-	/* start replenish only when buffers go lower than the threshold */
-	if (sys->rx_pool_sz - sys->len < IPA_REPL_XFER_THRESH)
-		return;
-
-	rx_len_cached = sys->len;
-
-	while (rx_len_cached < sys->rx_pool_sz) {
-		if (list_empty(&sys->rcycl_list)) {
-			rx_pkt = kmem_cache_zalloc(
-				ipa3_ctx->rx_pkt_wrapper_cache, flag);
-			if (!rx_pkt)
-				goto fail_kmem_cache_alloc;
-
-			INIT_WORK(&rx_pkt->work, ipa3_wq_rx_avail);
-			rx_pkt->sys = sys;
-
-			rx_pkt->data.skb = sys->get_skb(sys->rx_buff_sz, flag);
-			if (rx_pkt->data.skb == NULL) {
-				IPAERR("failed to alloc skb\n");
-				kmem_cache_free(ipa3_ctx->rx_pkt_wrapper_cache,
-					rx_pkt);
-				goto fail_kmem_cache_alloc;
-			}
-			ptr = skb_put(rx_pkt->data.skb, sys->rx_buff_sz);
-			rx_pkt->data.dma_addr = dma_map_single(ipa3_ctx->pdev,
-				ptr, sys->rx_buff_sz, DMA_FROM_DEVICE);
-			if (dma_mapping_error(ipa3_ctx->pdev,
-				rx_pkt->data.dma_addr)) {
-				IPAERR("dma_map_single failure %pK for %pK\n",
-					(void *)rx_pkt->data.dma_addr, ptr);
-				goto fail_dma_mapping;
-			}
-		} else {
-			spin_lock_bh(&sys->spinlock);
-			rx_pkt = list_first_entry(&sys->rcycl_list,
-				struct ipa3_rx_pkt_wrapper, link);
-			list_del(&rx_pkt->link);
-			spin_unlock_bh(&sys->spinlock);
-			ptr = skb_put(rx_pkt->data.skb, sys->rx_buff_sz);
-			rx_pkt->data.dma_addr = dma_map_single(ipa3_ctx->pdev,
-				ptr, sys->rx_buff_sz, DMA_FROM_DEVICE);
-			if (dma_mapping_error(ipa3_ctx->pdev,
-				rx_pkt->data.dma_addr)) {
-				IPAERR("dma_map_single failure %pK for %pK\n",
-					(void *)rx_pkt->data.dma_addr, ptr);
-				goto fail_dma_mapping;
-			}
-		}
-
-		gsi_xfer_elem_array[idx].addr = rx_pkt->data.dma_addr;
-		gsi_xfer_elem_array[idx].len = sys->rx_buff_sz;
-		gsi_xfer_elem_array[idx].flags = GSI_XFER_FLAG_EOT;
-		gsi_xfer_elem_array[idx].flags |= GSI_XFER_FLAG_EOB;
-		gsi_xfer_elem_array[idx].flags |= GSI_XFER_FLAG_BEI;
-		gsi_xfer_elem_array[idx].type = GSI_XFER_ELEM_DATA;
-		gsi_xfer_elem_array[idx].xfer_user_data = rx_pkt;
-		idx++;
-		rx_len_cached++;
-		/*
-		 * gsi_xfer_elem_buffer has a size of IPA_REPL_XFER_MAX.
-		 * If this size is reached we need to queue the xfers.
-		 */
-		if (idx == IPA_REPL_XFER_MAX) {
-			ret = gsi_queue_xfer(sys->ep->gsi_chan_hdl, idx,
-				gsi_xfer_elem_array, false);
-			if (ret != GSI_STATUS_SUCCESS) {
-				/* we don't expect this will happen */
-				IPAERR("failed to provide buffer: %d\n", ret);
-				WARN_ON(1);
-				break;
-			}
-			idx = 0;
-		}
-	}
-	goto done;
-fail_dma_mapping:
-	spin_lock_bh(&sys->spinlock);
-	list_add_tail(&rx_pkt->link, &sys->rcycl_list);
-	INIT_LIST_HEAD(&rx_pkt->link);
-	spin_unlock_bh(&sys->spinlock);
-fail_kmem_cache_alloc:
-	if (rx_len_cached == 0)
-		queue_delayed_work(sys->wq, &sys->replenish_rx_work,
-		msecs_to_jiffies(1));
-done:
-	/* only ring doorbell once here */
-	ret = gsi_queue_xfer(sys->ep->gsi_chan_hdl, idx,
-		gsi_xfer_elem_array, true);
-	if (ret == GSI_STATUS_SUCCESS) {
-		sys->len = rx_len_cached;
-	} else {
-		/* we don't expect this will happen */
-		IPAERR("failed to provide buffer: %d\n", ret);
-		WARN_ON(1);
-	}
-}
-
-static inline void __trigger_repl_work(struct ipa3_sys_context *sys)
-{
-	int tail, head, avail;
-
-	if (atomic_read(&sys->repl->pending))
-		return;
-
-	tail = atomic_read(&sys->repl->tail_idx);
-	head = atomic_read(&sys->repl->head_idx);
-	avail = (tail - head) % sys->repl->capacity;
-
-	if (avail < sys->repl->capacity / 4) {
-		atomic_set(&sys->repl->pending, 1);
-		queue_work(sys->repl_wq, &sys->repl_work);
-	}
-}
-
-static void ipa3_fast_replenish_rx_cache(struct ipa3_sys_context *sys)
-{
-	struct ipa3_rx_pkt_wrapper *rx_pkt;
-	int ret;
-	int rx_len_cached = 0;
-	struct gsi_xfer_elem gsi_xfer_elem_array[IPA_REPL_XFER_MAX];
-	u32 curr;
-	int idx = 0;
-
-	/* start replenish only when buffers go lower than the threshold */
-	if (sys->rx_pool_sz - sys->len < IPA_REPL_XFER_THRESH)
-		return;
-
-	spin_lock_bh(&sys->spinlock);
-	rx_len_cached = sys->len;
-	curr = atomic_read(&sys->repl->head_idx);
-
-	while (rx_len_cached < sys->rx_pool_sz) {
-		if (curr == atomic_read(&sys->repl->tail_idx))
-			break;
-		rx_pkt = sys->repl->cache[curr];
-		gsi_xfer_elem_array[idx].addr = rx_pkt->data.dma_addr;
-		gsi_xfer_elem_array[idx].len = sys->rx_buff_sz;
-		gsi_xfer_elem_array[idx].flags = GSI_XFER_FLAG_EOT;
-		gsi_xfer_elem_array[idx].flags |= GSI_XFER_FLAG_EOB;
-		gsi_xfer_elem_array[idx].flags |= GSI_XFER_FLAG_BEI;
-		gsi_xfer_elem_array[idx].type = GSI_XFER_ELEM_DATA;
-		gsi_xfer_elem_array[idx].xfer_user_data = rx_pkt;
-		rx_len_cached++;
-		curr = (++curr == sys->repl->capacity) ? 0 : curr;
-		idx++;
-		/*
-		 * gsi_xfer_elem_buffer has a size of IPA_REPL_XFER_THRESH.
-		 * If this size is reached we need to queue the xfers.
-		 */
-		if (idx == IPA_REPL_XFER_MAX) {
-			ret = gsi_queue_xfer(sys->ep->gsi_chan_hdl, idx,
-				gsi_xfer_elem_array, false);
-			if (ret != GSI_STATUS_SUCCESS) {
-				/* we don't expect this will happen */
-				IPAERR("failed to provide buffer: %d\n", ret);
-				WARN_ON(1);
-				break;
-			}
-			idx = 0;
-		}
-	}
-	/* only ring doorbell once here */
-	ret = gsi_queue_xfer(sys->ep->gsi_chan_hdl, idx,
-			gsi_xfer_elem_array, true);
-	if (ret == GSI_STATUS_SUCCESS) {
-		/* ensure write is done before setting head index */
-		mb();
-		atomic_set(&sys->repl->head_idx, curr);
-		sys->len = rx_len_cached;
-	} else {
-		/* we don't expect this will happen */
-		IPAERR("failed to provide buffer: %d\n", ret);
-		WARN_ON(1);
-	}
-
-	spin_unlock_bh(&sys->spinlock);
-
-	__trigger_repl_work(sys);
-
-	if (rx_len_cached <= IPA_DEFAULT_SYS_YELLOW_WM) {
-		if (sys->ep->client == IPA_CLIENT_APPS_WAN_CONS ||
-			sys->ep->client == IPA_CLIENT_APPS_WAN_COAL_CONS)
-			IPA_STATS_INC_CNT(ipa3_ctx->stats.wan_rx_empty);
-		else if (sys->ep->client == IPA_CLIENT_APPS_LAN_CONS)
-			IPA_STATS_INC_CNT(ipa3_ctx->stats.lan_rx_empty);
-		else
-			WARN_ON(1);
-		queue_delayed_work(sys->wq, &sys->replenish_rx_work,
-				msecs_to_jiffies(1));
-	}
-}
-
-static void ipa3_replenish_rx_work_func(struct work_struct *work)
-{
-	struct delayed_work *dwork;
-	struct ipa3_sys_context *sys;
-
-	dwork = container_of(work, struct delayed_work, work);
-	sys = container_of(dwork, struct ipa3_sys_context, replenish_rx_work);
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-	sys->repl_hdlr(sys);
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-}
-
-/**
- * free_rx_pkt() - function to free the skb and rx_pkt_wrapper
- *
- * @chan_user_data: ipa_sys_context used for skb size and skb_free func
- * @xfer_uder_data: rx_pkt wrapper to be freed
- *
- */
-static void free_rx_pkt(void *chan_user_data, void *xfer_user_data)
-{
-
-	struct ipa3_rx_pkt_wrapper *rx_pkt = (struct ipa3_rx_pkt_wrapper *)
-		xfer_user_data;
-	struct ipa3_sys_context *sys = (struct ipa3_sys_context *)
-		chan_user_data;
-
-	dma_unmap_single(ipa3_ctx->pdev, rx_pkt->data.dma_addr,
-		sys->rx_buff_sz, DMA_FROM_DEVICE);
-	sys->free_skb(rx_pkt->data.skb);
-	kmem_cache_free(ipa3_ctx->rx_pkt_wrapper_cache, rx_pkt);
-}
-
-/**
- * ipa3_cleanup_rx() - release RX queue resources
- *
- */
-static void ipa3_cleanup_rx(struct ipa3_sys_context *sys)
-{
-	struct ipa3_rx_pkt_wrapper *rx_pkt;
-	struct ipa3_rx_pkt_wrapper *r;
-	u32 head;
-	u32 tail;
-
-	/*
-	 * buffers not consumed by gsi are cleaned up using cleanup callback
-	 * provided to gsi
-	 */
-
-	spin_lock_bh(&sys->spinlock);
-	list_for_each_entry_safe(rx_pkt, r,
-				 &sys->rcycl_list, link) {
-		list_del(&rx_pkt->link);
-		dma_unmap_single(ipa3_ctx->pdev, rx_pkt->data.dma_addr,
-			sys->rx_buff_sz, DMA_FROM_DEVICE);
-		sys->free_skb(rx_pkt->data.skb);
-		kmem_cache_free(ipa3_ctx->rx_pkt_wrapper_cache, rx_pkt);
-	}
-	spin_unlock_bh(&sys->spinlock);
-
-	if (sys->repl) {
-		head = atomic_read(&sys->repl->head_idx);
-		tail = atomic_read(&sys->repl->tail_idx);
-		while (head != tail) {
-			rx_pkt = sys->repl->cache[head];
-			dma_unmap_single(ipa3_ctx->pdev, rx_pkt->data.dma_addr,
-					sys->rx_buff_sz, DMA_FROM_DEVICE);
-			sys->free_skb(rx_pkt->data.skb);
-			kmem_cache_free(ipa3_ctx->rx_pkt_wrapper_cache, rx_pkt);
-			head = (head + 1) % sys->repl->capacity;
-		}
-		kfree(sys->repl->cache);
-		kfree(sys->repl);
-	}
-}
-
-static struct sk_buff *ipa3_skb_copy_for_client(struct sk_buff *skb, int len)
-{
-	struct sk_buff *skb2 = NULL;
-
-	skb2 = __dev_alloc_skb(len + IPA_RX_BUFF_CLIENT_HEADROOM, GFP_KERNEL);
-	if (likely(skb2)) {
-		/* Set the data pointer */
-		skb_reserve(skb2, IPA_RX_BUFF_CLIENT_HEADROOM);
-		memcpy(skb2->data, skb->data, len);
-		skb2->len = len;
-		skb_set_tail_pointer(skb2, len);
-	}
-
-	return skb2;
-}
-
-static int ipa3_lan_rx_pyld_hdlr(struct sk_buff *skb,
-		struct ipa3_sys_context *sys)
-{
-	int rc = 0;
-	struct ipahal_pkt_status status;
-	u32 pkt_status_sz;
-	struct sk_buff *skb2;
-	int pad_len_byte;
-	int len;
-	unsigned char *buf;
-	int src_pipe;
-	unsigned int used = *(unsigned int *)skb->cb;
-	unsigned int used_align = ALIGN(used, 32);
-	unsigned long unused = IPA_GENERIC_RX_BUFF_BASE_SZ - used;
-	struct ipa3_tx_pkt_wrapper *tx_pkt = NULL;
-	unsigned long ptr;
-
-	IPA_DUMP_BUFF(skb->data, 0, skb->len);
-
-	if (skb->len == 0) {
-		IPAERR("ZLT packet arrived to AP\n");
-		goto out;
-	}
-
-	if (sys->len_partial) {
-		IPADBG_LOW("len_partial %d\n", sys->len_partial);
-		buf = skb_push(skb, sys->len_partial);
-		memcpy(buf, sys->prev_skb->data, sys->len_partial);
-		sys->len_partial = 0;
-		sys->free_skb(sys->prev_skb);
-		sys->prev_skb = NULL;
-		goto begin;
-	}
-
-	/* this pipe has TX comp (status only) + mux-ed LAN RX data
-	 * (status+data)
-	 */
-	if (sys->len_rem) {
-		IPADBG_LOW("rem %d skb %d pad %d\n", sys->len_rem, skb->len,
-				sys->len_pad);
-		if (sys->len_rem <= skb->len) {
-			if (sys->prev_skb) {
-				skb2 = skb_copy_expand(sys->prev_skb, 0,
-						sys->len_rem, GFP_KERNEL);
-				if (likely(skb2)) {
-					memcpy(skb_put(skb2, sys->len_rem),
-						skb->data, sys->len_rem);
-					skb_trim(skb2,
-						skb2->len - sys->len_pad);
-					skb2->truesize = skb2->len +
-						sizeof(struct sk_buff);
-					if (sys->drop_packet)
-						dev_kfree_skb_any(skb2);
-					else
-						sys->ep->client_notify(
-							sys->ep->priv,
-							IPA_RECEIVE,
-							(unsigned long)(skb2));
-				} else {
-					IPAERR("copy expand failed\n");
-				}
-				dev_kfree_skb_any(sys->prev_skb);
-			}
-			skb_pull(skb, sys->len_rem);
-			sys->prev_skb = NULL;
-			sys->len_rem = 0;
-			sys->len_pad = 0;
-		} else {
-			if (sys->prev_skb) {
-				skb2 = skb_copy_expand(sys->prev_skb, 0,
-					skb->len, GFP_KERNEL);
-				if (likely(skb2)) {
-					memcpy(skb_put(skb2, skb->len),
-						skb->data, skb->len);
-				} else {
-					IPAERR("copy expand failed\n");
-				}
-				dev_kfree_skb_any(sys->prev_skb);
-				sys->prev_skb = skb2;
-			}
-			sys->len_rem -= skb->len;
-			goto out;
-		}
-	}
-
-begin:
-	pkt_status_sz = ipahal_pkt_status_get_size();
-	while (skb->len) {
-		sys->drop_packet = false;
-		IPADBG_LOW("LEN_REM %d\n", skb->len);
-
-		if (skb->len < pkt_status_sz) {
-			WARN_ON(sys->prev_skb != NULL);
-			IPADBG_LOW("status straddles buffer\n");
-			sys->prev_skb = skb_copy(skb, GFP_KERNEL);
-			sys->len_partial = skb->len;
-			goto out;
-		}
-
-		ipahal_pkt_status_parse(skb->data, &status);
-		IPADBG_LOW("STATUS opcode=%d src=%d dst=%d len=%d\n",
-				status.status_opcode, status.endp_src_idx,
-				status.endp_dest_idx, status.pkt_len);
-		if (sys->status_stat) {
-			sys->status_stat->status[sys->status_stat->curr] =
-				status;
-			sys->status_stat->curr++;
-			if (sys->status_stat->curr == IPA_MAX_STATUS_STAT_NUM)
-				sys->status_stat->curr = 0;
-		}
-
-		if ((status.status_opcode !=
-			IPAHAL_PKT_STATUS_OPCODE_DROPPED_PACKET) &&
-			(status.status_opcode !=
-			IPAHAL_PKT_STATUS_OPCODE_PACKET) &&
-			(status.status_opcode !=
-			IPAHAL_PKT_STATUS_OPCODE_SUSPENDED_PACKET) &&
-			(status.status_opcode !=
-			IPAHAL_PKT_STATUS_OPCODE_PACKET_2ND_PASS)) {
-			IPAERR("unsupported opcode(%d)\n",
-				status.status_opcode);
-			skb_pull(skb, pkt_status_sz);
-			continue;
-		}
-		IPA_STATS_EXCP_CNT(status.exception,
-				ipa3_ctx->stats.rx_excp_pkts);
-		if (status.endp_dest_idx >= ipa3_ctx->ipa_num_pipes ||
-			status.endp_src_idx >= ipa3_ctx->ipa_num_pipes) {
-			IPAERR("status fields invalid\n");
-			IPAERR("STATUS opcode=%d src=%d dst=%d len=%d\n",
-				status.status_opcode, status.endp_src_idx,
-				status.endp_dest_idx, status.pkt_len);
-			WARN_ON(1);
-			/* HW gave an unexpected status */
-			ipa_assert();
-		}
-		if (IPAHAL_PKT_STATUS_MASK_FLAG_VAL(
-			IPAHAL_PKT_STATUS_MASK_TAG_VALID_SHFT, &status)) {
-			struct ipa3_tag_completion *comp;
-
-			IPADBG_LOW("TAG packet arrived\n");
-			if (status.tag_info == IPA_COOKIE) {
-				skb_pull(skb, pkt_status_sz);
-				if (skb->len < sizeof(comp)) {
-					IPAERR("TAG arrived without packet\n");
-					goto out;
-				}
-				memcpy(&comp, skb->data, sizeof(comp));
-				skb_pull(skb, sizeof(comp));
-				complete(&comp->comp);
-				if (atomic_dec_return(&comp->cnt) == 0)
-					kfree(comp);
-				continue;
-			} else {
-				ptr = tag_to_pointer_wa(status.tag_info);
-				tx_pkt = (struct ipa3_tx_pkt_wrapper *)ptr;
-				IPADBG_LOW("tx_pkt recv = %pK\n", tx_pkt);
-			}
-		}
-		if (status.pkt_len == 0) {
-			IPADBG_LOW("Skip aggr close status\n");
-			skb_pull(skb, pkt_status_sz);
-			IPA_STATS_INC_CNT(ipa3_ctx->stats.aggr_close);
-			IPA_STATS_DEC_CNT(ipa3_ctx->stats.rx_excp_pkts
-				[IPAHAL_PKT_STATUS_EXCEPTION_NONE]);
-			continue;
-		}
-
-		if (status.endp_dest_idx == (sys->ep - ipa3_ctx->ep)) {
-			/* RX data */
-			src_pipe = status.endp_src_idx;
-
-			/*
-			 * A packet which is received back to the AP after
-			 * there was no route match.
-			 */
-			if (status.exception ==
-				IPAHAL_PKT_STATUS_EXCEPTION_NONE &&
-				ipahal_is_rule_miss_id(status.rt_rule_id))
-				sys->drop_packet = true;
-
-			if (skb->len == pkt_status_sz &&
-				status.exception ==
-				IPAHAL_PKT_STATUS_EXCEPTION_NONE) {
-				WARN_ON(sys->prev_skb != NULL);
-				IPADBG_LOW("Ins header in next buffer\n");
-				sys->prev_skb = skb_copy(skb, GFP_KERNEL);
-				sys->len_partial = skb->len;
-				goto out;
-			}
-
-			pad_len_byte = ((status.pkt_len + 3) & ~3) -
-					status.pkt_len;
-
-			len = status.pkt_len + pad_len_byte;
-			IPADBG_LOW("pad %d pkt_len %d len %d\n", pad_len_byte,
-					status.pkt_len, len);
-
-			if (status.exception ==
-					IPAHAL_PKT_STATUS_EXCEPTION_DEAGGR) {
-				IPADBG_LOW(
-					"Dropping packet on DeAggr Exception\n");
-				sys->drop_packet = true;
-			}
-
-			skb2 = ipa3_skb_copy_for_client(skb,
-				min(status.pkt_len + pkt_status_sz, skb->len));
-			if (likely(skb2)) {
-				if (skb->len < len + pkt_status_sz) {
-					IPADBG_LOW("SPL skb len %d len %d\n",
-							skb->len, len);
-					sys->prev_skb = skb2;
-					sys->len_rem = len - skb->len +
-						pkt_status_sz;
-					sys->len_pad = pad_len_byte;
-					skb_pull(skb, skb->len);
-				} else {
-					skb_trim(skb2, status.pkt_len +
-							pkt_status_sz);
-					IPADBG_LOW("rx avail for %d\n",
-							status.endp_dest_idx);
-					if (sys->drop_packet) {
-						dev_kfree_skb_any(skb2);
-					} else if (status.pkt_len >
-						   IPA_GENERIC_AGGR_BYTE_LIMIT *
-						   1024) {
-						IPAERR("packet size invalid\n");
-						IPAERR("STATUS opcode=%d\n",
-							status.status_opcode);
-						IPAERR("src=%d dst=%d len=%d\n",
-							status.endp_src_idx,
-							status.endp_dest_idx,
-							status.pkt_len);
-						/* Unexpected HW status */
-						ipa_assert();
-					} else {
-						skb2->truesize = skb2->len +
-						sizeof(struct sk_buff) +
-						(ALIGN(len +
-						pkt_status_sz, 32) *
-						unused / used_align);
-						sys->ep->client_notify(
-							sys->ep->priv,
-							IPA_RECEIVE,
-							(unsigned long)(skb2));
-					}
-					skb_pull(skb, len + pkt_status_sz);
-				}
-			} else {
-				IPAERR("fail to alloc skb\n");
-				if (skb->len < len) {
-					sys->prev_skb = NULL;
-					sys->len_rem = len - skb->len +
-						pkt_status_sz;
-					sys->len_pad = pad_len_byte;
-					skb_pull(skb, skb->len);
-				} else {
-					skb_pull(skb, len + pkt_status_sz);
-				}
-			}
-			/* TX comp */
-			ipa3_wq_write_done_status(src_pipe, tx_pkt);
-			IPADBG_LOW("tx comp imp for %d\n", src_pipe);
-		} else {
-			/* TX comp */
-			ipa3_wq_write_done_status(status.endp_src_idx, tx_pkt);
-			IPADBG_LOW("tx comp exp for %d\n",
-				status.endp_src_idx);
-			skb_pull(skb, pkt_status_sz);
-			IPA_STATS_INC_CNT(ipa3_ctx->stats.stat_compl);
-			IPA_STATS_DEC_CNT(ipa3_ctx->stats.rx_excp_pkts
-				[IPAHAL_PKT_STATUS_EXCEPTION_NONE]);
-		}
-		tx_pkt = NULL;
-	}
-
-out:
-	ipa3_skb_recycle(skb);
-	return rc;
-}
-
-static struct sk_buff *ipa3_join_prev_skb(struct sk_buff *prev_skb,
-		struct sk_buff *skb, unsigned int len)
-{
-	struct sk_buff *skb2;
-
-	skb2 = skb_copy_expand(prev_skb, 0,
-			len, GFP_KERNEL);
-	if (likely(skb2)) {
-		memcpy(skb_put(skb2, len),
-			skb->data, len);
-	} else {
-		IPAERR("copy expand failed\n");
-		skb2 = NULL;
-	}
-	dev_kfree_skb_any(prev_skb);
-
-	return skb2;
-}
-
-static void ipa3_wan_rx_handle_splt_pyld(struct sk_buff *skb,
-		struct ipa3_sys_context *sys)
-{
-	struct sk_buff *skb2;
-
-	IPADBG_LOW("rem %d skb %d\n", sys->len_rem, skb->len);
-	if (sys->len_rem <= skb->len) {
-		if (sys->prev_skb) {
-			skb2 = ipa3_join_prev_skb(sys->prev_skb, skb,
-					sys->len_rem);
-			if (likely(skb2)) {
-				IPADBG_LOW(
-					"removing Status element from skb and sending to WAN client");
-				skb_pull(skb2, ipahal_pkt_status_get_size());
-				skb2->truesize = skb2->len +
-					sizeof(struct sk_buff);
-				sys->ep->client_notify(sys->ep->priv,
-					IPA_RECEIVE,
-					(unsigned long)(skb2));
-			}
-		}
-		skb_pull(skb, sys->len_rem);
-		sys->prev_skb = NULL;
-		sys->len_rem = 0;
-	} else {
-		if (sys->prev_skb) {
-			skb2 = ipa3_join_prev_skb(sys->prev_skb, skb,
-					skb->len);
-			sys->prev_skb = skb2;
-		}
-		sys->len_rem -= skb->len;
-		skb_pull(skb, skb->len);
-	}
-}
-
-static int ipa3_wan_rx_pyld_hdlr(struct sk_buff *skb,
-		struct ipa3_sys_context *sys)
-{
-	struct ipahal_pkt_status status;
-	unsigned char *skb_data;
-	u32 pkt_status_sz;
-	struct sk_buff *skb2;
-	u16 pkt_len_with_pad;
-	u32 qmap_hdr;
-	int checksum_trailer_exists;
-	int frame_len;
-	int ep_idx;
-	unsigned int used = *(unsigned int *)skb->cb;
-	unsigned int used_align = ALIGN(used, 32);
-	unsigned long unused = IPA_GENERIC_RX_BUFF_BASE_SZ - used;
-
-	IPA_DUMP_BUFF(skb->data, 0, skb->len);
-	if (skb->len == 0) {
-		IPAERR("ZLT\n");
-		goto bail;
-	}
-
-	if (ipa3_ctx->ipa_client_apps_wan_cons_agg_gro) {
-		sys->ep->client_notify(sys->ep->priv,
-			IPA_RECEIVE, (unsigned long)(skb));
-		return 0;
-	}
-	if (sys->repl_hdlr == ipa3_replenish_rx_cache_recycle) {
-		IPAERR("Recycle should enable only with GRO Aggr\n");
-		ipa_assert();
-	}
-
-	/*
-	 * payload splits across 2 buff or more,
-	 * take the start of the payload from prev_skb
-	 */
-	if (sys->len_rem)
-		ipa3_wan_rx_handle_splt_pyld(skb, sys);
-
-	pkt_status_sz = ipahal_pkt_status_get_size();
-	while (skb->len) {
-		IPADBG_LOW("LEN_REM %d\n", skb->len);
-		if (skb->len < pkt_status_sz) {
-			IPAERR("status straddles buffer\n");
-			WARN_ON(1);
-			goto bail;
-		}
-		ipahal_pkt_status_parse(skb->data, &status);
-		skb_data = skb->data;
-		IPADBG_LOW("STATUS opcode=%d src=%d dst=%d len=%d\n",
-				status.status_opcode, status.endp_src_idx,
-				status.endp_dest_idx, status.pkt_len);
-
-		if (sys->status_stat) {
-			sys->status_stat->status[sys->status_stat->curr] =
-				status;
-			sys->status_stat->curr++;
-			if (sys->status_stat->curr == IPA_MAX_STATUS_STAT_NUM)
-				sys->status_stat->curr = 0;
-		}
-
-		if ((status.status_opcode !=
-			IPAHAL_PKT_STATUS_OPCODE_DROPPED_PACKET) &&
-			(status.status_opcode !=
-			IPAHAL_PKT_STATUS_OPCODE_PACKET) &&
-			(status.status_opcode !=
-			IPAHAL_PKT_STATUS_OPCODE_PACKET_2ND_PASS)) {
-			IPAERR("unsupported opcode(%d)\n",
-				status.status_opcode);
-			skb_pull(skb, pkt_status_sz);
-			continue;
-		}
-
-		IPA_STATS_INC_CNT(ipa3_ctx->stats.rx_pkts);
-		if (status.endp_dest_idx >= ipa3_ctx->ipa_num_pipes ||
-			status.endp_src_idx >= ipa3_ctx->ipa_num_pipes) {
-			IPAERR("status fields invalid\n");
-			WARN_ON(1);
-			goto bail;
-		}
-		if (status.pkt_len == 0) {
-			IPADBG_LOW("Skip aggr close status\n");
-			skb_pull(skb, pkt_status_sz);
-			IPA_STATS_DEC_CNT(ipa3_ctx->stats.rx_pkts);
-			IPA_STATS_INC_CNT(ipa3_ctx->stats.wan_aggr_close);
-			continue;
-		}
-		ep_idx = ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_CONS);
-		if (status.endp_dest_idx != ep_idx) {
-			IPAERR("expected endp_dest_idx %d received %d\n",
-					ep_idx, status.endp_dest_idx);
-			WARN_ON(1);
-			goto bail;
-		}
-		/* RX data */
-		if (skb->len == pkt_status_sz) {
-			IPAERR("Ins header in next buffer\n");
-			WARN_ON(1);
-			goto bail;
-		}
-		qmap_hdr = *(u32 *)(skb_data + pkt_status_sz);
-		/*
-		 * Take the pkt_len_with_pad from the last 2 bytes of the QMAP
-		 * header
-		 */
-
-		/*QMAP is BE: convert the pkt_len field from BE to LE*/
-		pkt_len_with_pad = ntohs((qmap_hdr>>16) & 0xffff);
-		IPADBG_LOW("pkt_len with pad %d\n", pkt_len_with_pad);
-		/*get the CHECKSUM_PROCESS bit*/
-		checksum_trailer_exists = IPAHAL_PKT_STATUS_MASK_FLAG_VAL(
-			IPAHAL_PKT_STATUS_MASK_CKSUM_PROCESS_SHFT, &status);
-		IPADBG_LOW("checksum_trailer_exists %d\n",
-				checksum_trailer_exists);
-
-		frame_len = pkt_status_sz + IPA_QMAP_HEADER_LENGTH +
-			    pkt_len_with_pad;
-		if (checksum_trailer_exists)
-			frame_len += IPA_DL_CHECKSUM_LENGTH;
-		IPADBG_LOW("frame_len %d\n", frame_len);
-
-		skb2 = skb_clone(skb, GFP_KERNEL);
-		if (likely(skb2)) {
-			/*
-			 * the len of actual data is smaller than expected
-			 * payload split across 2 buff
-			 */
-			if (skb->len < frame_len) {
-				IPADBG_LOW("SPL skb len %d len %d\n",
-						skb->len, frame_len);
-				sys->prev_skb = skb2;
-				sys->len_rem = frame_len - skb->len;
-				skb_pull(skb, skb->len);
-			} else {
-				skb_trim(skb2, frame_len);
-				IPADBG_LOW("rx avail for %d\n",
-						status.endp_dest_idx);
-				IPADBG_LOW(
-					"removing Status element from skb and sending to WAN client");
-				skb_pull(skb2, pkt_status_sz);
-				skb2->truesize = skb2->len +
-					sizeof(struct sk_buff) +
-					(ALIGN(frame_len, 32) *
-					 unused / used_align);
-				sys->ep->client_notify(sys->ep->priv,
-					IPA_RECEIVE, (unsigned long)(skb2));
-				skb_pull(skb, frame_len);
-			}
-		} else {
-			IPAERR("fail to clone\n");
-			if (skb->len < frame_len) {
-				sys->prev_skb = NULL;
-				sys->len_rem = frame_len - skb->len;
-				skb_pull(skb, skb->len);
-			} else {
-				skb_pull(skb, frame_len);
-			}
-		}
-	}
-bail:
-	sys->free_skb(skb);
-	return 0;
-}
-
-static struct sk_buff *ipa3_get_skb_ipa_rx(unsigned int len, gfp_t flags)
-{
-	return __dev_alloc_skb(len, flags);
-}
-
-static void ipa3_free_skb_rx(struct sk_buff *skb)
-{
-	dev_kfree_skb_any(skb);
-}
-
-void ipa3_lan_rx_cb(void *priv, enum ipa_dp_evt_type evt, unsigned long data)
-{
-	struct sk_buff *rx_skb = (struct sk_buff *)data;
-	struct ipahal_pkt_status status;
-	struct ipa3_ep_context *ep;
-	unsigned int src_pipe;
-	u32 metadata;
-	u8 ucp;
-
-	ipahal_pkt_status_parse(rx_skb->data, &status);
-	src_pipe = status.endp_src_idx;
-	metadata = status.metadata;
-	ucp = status.ucp;
-	ep = &ipa3_ctx->ep[src_pipe];
-	if (unlikely(src_pipe >= ipa3_ctx->ipa_num_pipes ||
-		!ep->valid ||
-		!ep->client_notify)) {
-		IPAERR_RL("drop pipe=%d ep_valid=%d client_notify=%pK\n",
-		  src_pipe, ep->valid, ep->client_notify);
-		dev_kfree_skb_any(rx_skb);
-		return;
-	}
-	if (status.exception == IPAHAL_PKT_STATUS_EXCEPTION_NONE)
-		skb_pull(rx_skb, ipahal_pkt_status_get_size() +
-				IPA_LAN_RX_HEADER_LENGTH);
-	else
-		skb_pull(rx_skb, ipahal_pkt_status_get_size());
-
-	/* Metadata Info
-	 *  ------------------------------------------
-	 *  |   3     |   2     |    1        |  0   |
-	 *  | fw_desc | vdev_id | qmap mux id | Resv |
-	 *  ------------------------------------------
-	 */
-	*(u16 *)rx_skb->cb = ((metadata >> 16) & 0xFFFF);
-	*(u8 *)(rx_skb->cb + 4) = ucp;
-	IPADBG_LOW("meta_data: 0x%x cb: 0x%x\n",
-			metadata, *(u32 *)rx_skb->cb);
-	IPADBG_LOW("ucp: %d\n", *(u8 *)(rx_skb->cb + 4));
-
-	ep->client_notify(ep->priv, IPA_RECEIVE, (unsigned long)(rx_skb));
-}
-
-static void ipa3_recycle_rx_wrapper(struct ipa3_rx_pkt_wrapper *rx_pkt)
-{
-	rx_pkt->data.dma_addr = 0;
-	/* skb recycle was moved to pyld_hdlr */
-	INIT_LIST_HEAD(&rx_pkt->link);
-	spin_lock_bh(&rx_pkt->sys->spinlock);
-	list_add_tail(&rx_pkt->link, &rx_pkt->sys->rcycl_list);
-	spin_unlock_bh(&rx_pkt->sys->spinlock);
-}
-
-/**
- * handle_skb_completion()- Handle event completion EOB or EOT and prep the skb
- *
- * if eob: Set skb values, put rx_pkt at the end of the list and return NULL
- *
- * if eot: Set skb values, put skb at the end of the list. Then update the
- * length and chain the skbs together while also freeing and unmapping the
- * corresponding rx pkt. Once finished return the head_skb to be sent up the
- * network stack.
- */
-static struct sk_buff *handle_skb_completion(struct gsi_chan_xfer_notify
-		*notify, bool update_truesize)
-{
-	struct ipa3_rx_pkt_wrapper *rx_pkt, *tmp;
-	struct sk_buff *rx_skb, *next_skb = NULL;
-	struct list_head *head;
-	struct ipa3_sys_context *sys;
-
-	sys = (struct ipa3_sys_context *) notify->chan_user_data;
-	rx_pkt = (struct ipa3_rx_pkt_wrapper *) notify->xfer_user_data;
-
-	spin_lock_bh(&rx_pkt->sys->spinlock);
-	rx_pkt->sys->len--;
-	spin_unlock_bh(&rx_pkt->sys->spinlock);
-
-	if (notify->bytes_xfered)
-		rx_pkt->len = notify->bytes_xfered;
-
-	rx_skb = rx_pkt->data.skb;
-	skb_set_tail_pointer(rx_skb, rx_pkt->len);
-	rx_skb->len = rx_pkt->len;
-
-	if (update_truesize) {
-		*(unsigned int *)rx_skb->cb = rx_skb->len;
-		rx_skb->truesize = rx_pkt->len + sizeof(struct sk_buff);
-	}
-
-	if (notify->veid >= GSI_VEID_MAX) {
-		WARN_ON(1);
-		return NULL;
-	}
-
-	/*Assesrt when WAN consumer channel receive EOB event*/
-	if (notify->evt_id == GSI_CHAN_EVT_EOB &&
-		sys->ep->client == IPA_CLIENT_APPS_WAN_CONS) {
-		IPAERR("EOB event received on WAN consumer channel\n");
-		ipa_assert();
-	}
-
-	head = &rx_pkt->sys->pending_pkts[notify->veid];
-
-	INIT_LIST_HEAD(&rx_pkt->link);
-	list_add_tail(&rx_pkt->link, head);
-
-	/* Check added for handling LAN consumer packet without EOT flag */
-	if (notify->evt_id == GSI_CHAN_EVT_EOT ||
-		sys->ep->client == IPA_CLIENT_APPS_LAN_CONS) {
-	/* go over the list backward to save computations on updating length */
-		list_for_each_entry_safe_reverse(rx_pkt, tmp, head, link) {
-			rx_skb = rx_pkt->data.skb;
-
-			list_del(&rx_pkt->link);
-			dma_unmap_single(ipa3_ctx->pdev, rx_pkt->data.dma_addr,
-				sys->rx_buff_sz, DMA_FROM_DEVICE);
-			sys->free_rx_wrapper(rx_pkt);
-
-			if (next_skb) {
-				skb_shinfo(rx_skb)->frag_list = next_skb;
-				rx_skb->len += next_skb->len;
-				rx_skb->data_len += next_skb->len;
-			}
-			next_skb = rx_skb;
-		}
-	} else {
-		return NULL;
-	}
-	return rx_skb;
-}
-
-static void ipa3_wq_rx_common(struct ipa3_sys_context *sys,
-	struct gsi_chan_xfer_notify *notify)
-{
-	struct sk_buff *rx_skb;
-	struct ipa3_sys_context *coal_sys;
-	int ipa_ep_idx;
-
-	if (!notify) {
-		IPAERR_RL("gsi_chan_xfer_notify is null\n");
-		return;
-	}
-	rx_skb = handle_skb_completion(notify, true);
-
-	if (rx_skb) {
-		sys->pyld_hdlr(rx_skb, sys);
-
-		/* For coalescing, we have 2 transfer rings to replenish */
-		if (sys->ep->client == IPA_CLIENT_APPS_WAN_COAL_CONS) {
-			ipa_ep_idx = ipa3_get_ep_mapping(
-					IPA_CLIENT_APPS_WAN_CONS);
-
-			if (ipa_ep_idx == IPA_EP_NOT_ALLOCATED) {
-				IPAERR("Invalid client.\n");
-				return;
-			}
-
-			coal_sys = ipa3_ctx->ep[ipa_ep_idx].sys;
-			coal_sys->repl_hdlr(coal_sys);
-		}
-
-		sys->repl_hdlr(sys);
-	}
-}
-
-static void ipa3_wq_rx_napi_chain(struct ipa3_sys_context *sys,
-		struct gsi_chan_xfer_notify *notify, uint32_t num)
-{
-	struct ipa3_sys_context *wan_def_sys;
-	int i, ipa_ep_idx;
-	struct sk_buff *rx_skb, *first_skb = NULL, *prev_skb = NULL;
-
-	/* non-coalescing case (SKB chaining enabled) */
-	if (sys->ep->client != IPA_CLIENT_APPS_WAN_COAL_CONS) {
-		for (i = 0; i < num; i++) {
-			rx_skb = handle_skb_completion(&notify[i], false);
-
-			/* this is always true for EOTs */
-			if (rx_skb) {
-				if (!first_skb)
-					first_skb = rx_skb;
-
-				if (prev_skb)
-					skb_shinfo(prev_skb)->frag_list =
-						rx_skb;
-
-				prev_skb = rx_skb;
-			}
-		}
-		if (prev_skb) {
-			skb_shinfo(prev_skb)->frag_list = NULL;
-			sys->pyld_hdlr(first_skb, sys);
-		}
-
-	/* TODO: add chaining for coal case */
-	} else {
-		for (i = 0; i < num; i++) {
-			rx_skb = handle_skb_completion(&notify[i], false);
-
-			if (rx_skb) {
-				sys->pyld_hdlr(rx_skb, sys);
-
-			/*
-			 * For coalescing, we have 2 transfer rings to replenish
-			 */
-				ipa_ep_idx = ipa3_get_ep_mapping(
-						IPA_CLIENT_APPS_WAN_CONS);
-
-				if (ipa_ep_idx == IPA_EP_NOT_ALLOCATED) {
-					IPAERR("Invalid client.\n");
-					return;
-				}
-				wan_def_sys = ipa3_ctx->ep[ipa_ep_idx].sys;
-				wan_def_sys->repl_hdlr(wan_def_sys);
-			}
-		}
-	}
-}
-
-static void ipa3_wlan_wq_rx_common(struct ipa3_sys_context *sys,
-	struct gsi_chan_xfer_notify *notify)
-{
-	struct ipa3_rx_pkt_wrapper *rx_pkt_expected;
-	struct sk_buff *rx_skb;
-
-	rx_pkt_expected = (struct ipa3_rx_pkt_wrapper *) notify->xfer_user_data;
-
-	sys->len--;
-
-	if (notify->bytes_xfered)
-		rx_pkt_expected->len = notify->bytes_xfered;
-
-	rx_skb = rx_pkt_expected->data.skb;
-	skb_set_tail_pointer(rx_skb, rx_pkt_expected->len);
-	rx_skb->len = rx_pkt_expected->len;
-	rx_skb->truesize = rx_pkt_expected->len + sizeof(struct sk_buff);
-	sys->ep->wstats.tx_pkts_rcvd++;
-	if (sys->len <= IPA_WLAN_RX_POOL_SZ_LOW_WM) {
-		ipa3_free_skb(&rx_pkt_expected->data);
-		sys->ep->wstats.tx_pkts_dropped++;
-	} else {
-		sys->ep->wstats.tx_pkts_sent++;
-		sys->ep->client_notify(sys->ep->priv, IPA_RECEIVE,
-				(unsigned long)(&rx_pkt_expected->data));
-	}
-	ipa3_replenish_wlan_rx_cache(sys);
-}
-
-static void ipa3_dma_memcpy_notify(struct ipa3_sys_context *sys)
-{
-	IPADBG_LOW("ENTER.\n");
-	if (unlikely(list_empty(&sys->head_desc_list))) {
-		IPAERR("descriptor list is empty!\n");
-		WARN_ON(1);
-		return;
-	}
-	sys->ep->client_notify(sys->ep->priv, IPA_RECEIVE, 0);
-	IPADBG_LOW("EXIT\n");
-}
-
-static void ipa3_wq_rx_avail(struct work_struct *work)
-{
-	struct ipa3_rx_pkt_wrapper *rx_pkt;
-	struct ipa3_sys_context *sys;
-
-	rx_pkt = container_of(work, struct ipa3_rx_pkt_wrapper, work);
-	WARN(unlikely(rx_pkt == NULL), "rx pkt is null");
-	sys = rx_pkt->sys;
-	ipa3_wq_rx_common(sys, 0);
-}
-
-static int ipa3_odu_rx_pyld_hdlr(struct sk_buff *rx_skb,
-	struct ipa3_sys_context *sys)
-{
-	if (sys->ep->client_notify) {
-		sys->ep->client_notify(sys->ep->priv, IPA_RECEIVE,
-			(unsigned long)(rx_skb));
-	} else {
-		dev_kfree_skb_any(rx_skb);
-		WARN(1, "client notify is null");
-	}
-
-	return 0;
-}
-
-static int ipa3_odl_dpl_rx_pyld_hdlr(struct sk_buff *rx_skb,
-	struct ipa3_sys_context *sys)
-{
-	if (WARN(!sys->ep->client_notify, "sys->ep->client_notify is NULL\n"))
-		dev_kfree_skb_any(rx_skb);
-	else
-		sys->ep->client_notify(sys->ep->priv, IPA_RECEIVE,
-			(unsigned long)(rx_skb));
-
-	return 0;
-}
-static void ipa3_free_rx_wrapper(struct ipa3_rx_pkt_wrapper *rk_pkt)
-{
-	kmem_cache_free(ipa3_ctx->rx_pkt_wrapper_cache, rk_pkt);
-}
-
-static void ipa3_set_aggr_limit(struct ipa_sys_connect_params *in,
-		struct ipa3_sys_context *sys)
-{
-	u32 *aggr_byte_limit = &in->ipa_ep_cfg.aggr.aggr_byte_limit;
-	u32 adjusted_sz = ipa_adjust_ra_buff_base_sz(*aggr_byte_limit);
-
-	IPADBG("get close-by %u\n", adjusted_sz);
-	IPADBG("set rx_buff_sz %lu\n", (unsigned long)
-		IPA_GENERIC_RX_BUFF_SZ(adjusted_sz));
-
-	/* disable ipa_status */
-	sys->ep->status.status_en = false;
-	sys->rx_buff_sz = IPA_GENERIC_RX_BUFF_SZ(adjusted_sz);
-
-	if (in->client == IPA_CLIENT_APPS_WAN_COAL_CONS)
-		in->ipa_ep_cfg.aggr.aggr_hard_byte_limit_en = 1;
-
-	*aggr_byte_limit = sys->rx_buff_sz < *aggr_byte_limit ?
-		IPA_ADJUST_AGGR_BYTE_LIMIT(sys->rx_buff_sz) :
-		IPA_ADJUST_AGGR_BYTE_LIMIT(*aggr_byte_limit);
-
-	IPADBG("set aggr_limit %lu\n", (unsigned long) *aggr_byte_limit);
-}
-
-static int ipa3_assign_policy(struct ipa_sys_connect_params *in,
-		struct ipa3_sys_context *sys)
-{
-	bool apps_wan_cons_agg_gro_flag;
-	unsigned long aggr_byte_limit;
-
-	if (in->client == IPA_CLIENT_APPS_CMD_PROD) {
-		sys->policy = IPA_POLICY_INTR_MODE;
-		sys->use_comm_evt_ring = false;
-		return 0;
-	}
-
-	if (in->client == IPA_CLIENT_APPS_WAN_PROD) {
-		sys->policy = IPA_POLICY_INTR_MODE;
-		sys->use_comm_evt_ring = true;
-		INIT_WORK(&sys->work, ipa3_send_nop_desc);
-
-		/*
-		 * enable source notification status for exception packets
-		 * (i.e. QMAP commands) to be routed to modem.
-		 */
-		sys->ep->status.status_en = true;
-		sys->ep->status.status_ep =
-			ipa3_get_ep_mapping(IPA_CLIENT_Q6_WAN_CONS);
-		return 0;
-	}
-
-	if (IPA_CLIENT_IS_MEMCPY_DMA_PROD(in->client)) {
-		sys->policy = IPA_POLICY_NOINTR_MODE;
-		return 0;
-	}
-
-	apps_wan_cons_agg_gro_flag =
-		ipa3_ctx->ipa_client_apps_wan_cons_agg_gro;
-	aggr_byte_limit = in->ipa_ep_cfg.aggr.aggr_byte_limit;
-
-	if (IPA_CLIENT_IS_PROD(in->client)) {
-		if (sys->ep->skip_ep_cfg) {
-			sys->policy = IPA_POLICY_INTR_POLL_MODE;
-			sys->use_comm_evt_ring = true;
-			atomic_set(&sys->curr_polling_state, 0);
-		} else {
-			sys->policy = IPA_POLICY_INTR_MODE;
-			sys->use_comm_evt_ring = true;
-			INIT_WORK(&sys->work, ipa3_send_nop_desc);
-		}
-	} else {
-		if (in->client == IPA_CLIENT_APPS_LAN_CONS ||
-		    in->client == IPA_CLIENT_APPS_WAN_CONS ||
-		    in->client == IPA_CLIENT_APPS_WAN_COAL_CONS) {
-			sys->ep->status.status_en = true;
-			sys->policy = IPA_POLICY_INTR_POLL_MODE;
-			INIT_WORK(&sys->work, ipa3_wq_handle_rx);
-			INIT_DELAYED_WORK(&sys->switch_to_intr_work,
-				ipa3_switch_to_intr_rx_work_func);
-			INIT_DELAYED_WORK(&sys->replenish_rx_work,
-					ipa3_replenish_rx_work_func);
-			INIT_WORK(&sys->repl_work, ipa3_wq_repl_rx);
-			atomic_set(&sys->curr_polling_state, 0);
-			sys->rx_buff_sz = IPA_GENERIC_RX_BUFF_SZ(
-				IPA_GENERIC_RX_BUFF_BASE_SZ);
-			sys->get_skb = ipa3_get_skb_ipa_rx;
-			sys->free_skb = ipa3_free_skb_rx;
-			in->ipa_ep_cfg.aggr.aggr_en = IPA_ENABLE_AGGR;
-			if (in->client == IPA_CLIENT_APPS_WAN_COAL_CONS)
-				in->ipa_ep_cfg.aggr.aggr = IPA_COALESCE;
-			else
-				in->ipa_ep_cfg.aggr.aggr = IPA_GENERIC;
-			in->ipa_ep_cfg.aggr.aggr_time_limit =
-				IPA_GENERIC_AGGR_TIME_LIMIT;
-			if (in->client == IPA_CLIENT_APPS_LAN_CONS) {
-				sys->pyld_hdlr = ipa3_lan_rx_pyld_hdlr;
-				sys->repl_hdlr =
-					ipa3_replenish_rx_cache_recycle;
-				sys->free_rx_wrapper =
-					ipa3_recycle_rx_wrapper;
-				sys->rx_pool_sz =
-					ipa3_ctx->lan_rx_ring_size;
-				in->ipa_ep_cfg.aggr.aggr_byte_limit =
-				IPA_GENERIC_AGGR_BYTE_LIMIT;
-				in->ipa_ep_cfg.aggr.aggr_pkt_limit =
-				IPA_GENERIC_AGGR_PKT_LIMIT;
-			} else if (in->client == IPA_CLIENT_APPS_WAN_CONS ||
-				in->client == IPA_CLIENT_APPS_WAN_COAL_CONS) {
-				sys->pyld_hdlr = ipa3_wan_rx_pyld_hdlr;
-				sys->free_rx_wrapper = ipa3_free_rx_wrapper;
-				sys->rx_pool_sz = ipa3_ctx->wan_rx_ring_size;
-				if (nr_cpu_ids > 1) {
-					sys->repl_hdlr =
-					   ipa3_fast_replenish_rx_cache;
-				} else {
-					sys->repl_hdlr =
-					   ipa3_replenish_rx_cache;
-				}
-				if (in->napi_obj && in->recycle_enabled)
-					sys->repl_hdlr =
-					 ipa3_replenish_rx_cache_recycle;
-				in->ipa_ep_cfg.aggr.aggr_sw_eof_active
-					= true;
-				if (apps_wan_cons_agg_gro_flag) {
-					ipa3_set_aggr_limit(in, sys);
-				} else {
-					in->ipa_ep_cfg.aggr.aggr_byte_limit =
-					IPA_GENERIC_AGGR_BYTE_LIMIT;
-					in->ipa_ep_cfg.aggr.aggr_pkt_limit =
-					IPA_GENERIC_AGGR_PKT_LIMIT;
-				}
-			}
-		} else if (IPA_CLIENT_IS_WLAN_CONS(in->client)) {
-			IPADBG("assigning policy to client:%d",
-				in->client);
-
-			sys->policy = IPA_POLICY_INTR_POLL_MODE;
-			INIT_WORK(&sys->work, ipa3_wq_handle_rx);
-			INIT_DELAYED_WORK(&sys->switch_to_intr_work,
-				ipa3_switch_to_intr_rx_work_func);
-			INIT_DELAYED_WORK(&sys->replenish_rx_work,
-				ipa3_replenish_rx_work_func);
-			atomic_set(&sys->curr_polling_state, 0);
-			sys->rx_buff_sz = IPA_WLAN_RX_BUFF_SZ;
-			sys->rx_pool_sz = in->desc_fifo_sz /
-				IPA_FIFO_ELEMENT_SIZE - 1;
-			if (sys->rx_pool_sz > IPA_WLAN_RX_POOL_SZ)
-				sys->rx_pool_sz = IPA_WLAN_RX_POOL_SZ;
-			sys->pyld_hdlr = NULL;
-			sys->repl_hdlr = ipa3_replenish_wlan_rx_cache;
-			sys->get_skb = ipa3_get_skb_ipa_rx;
-			sys->free_skb = ipa3_free_skb_rx;
-			sys->free_rx_wrapper = ipa3_free_rx_wrapper;
-			in->ipa_ep_cfg.aggr.aggr_en = IPA_BYPASS_AGGR;
-		} else if (IPA_CLIENT_IS_ODU_CONS(in->client)) {
-			IPADBG("assigning policy to client:%d",
-				in->client);
-
-			sys->policy = IPA_POLICY_INTR_POLL_MODE;
-			INIT_WORK(&sys->work, ipa3_wq_handle_rx);
-			INIT_DELAYED_WORK(&sys->switch_to_intr_work,
-				ipa3_switch_to_intr_rx_work_func);
-			INIT_DELAYED_WORK(&sys->replenish_rx_work,
-				ipa3_replenish_rx_work_func);
-			atomic_set(&sys->curr_polling_state, 0);
-			sys->rx_pool_sz = in->desc_fifo_sz /
-				IPA_FIFO_ELEMENT_SIZE - 1;
-			if (sys->rx_pool_sz > IPA_ODU_RX_POOL_SZ)
-				sys->rx_pool_sz = IPA_ODU_RX_POOL_SZ;
-			sys->pyld_hdlr = ipa3_odu_rx_pyld_hdlr;
-			sys->get_skb = ipa3_get_skb_ipa_rx;
-			sys->free_skb = ipa3_free_skb_rx;
-			/* recycle skb for GSB use case */
-			if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0) {
-				sys->free_rx_wrapper =
-					ipa3_free_rx_wrapper;
-				sys->repl_hdlr =
-					ipa3_replenish_rx_cache;
-				/* Overwrite buffer size & aggr limit for GSB */
-				sys->rx_buff_sz = IPA_GENERIC_RX_BUFF_SZ(
-					IPA_GSB_RX_BUFF_BASE_SZ);
-				in->ipa_ep_cfg.aggr.aggr_byte_limit =
-					IPA_GSB_AGGR_BYTE_LIMIT;
-			} else {
-				sys->free_rx_wrapper =
-					ipa3_free_rx_wrapper;
-				sys->repl_hdlr = ipa3_replenish_rx_cache;
-				sys->rx_buff_sz = IPA_ODU_RX_BUFF_SZ;
-			}
-		} else if (in->client ==
-				IPA_CLIENT_MEMCPY_DMA_ASYNC_CONS) {
-			IPADBG("assigning policy to client:%d",
-				in->client);
-
-			sys->policy = IPA_POLICY_INTR_POLL_MODE;
-			INIT_WORK(&sys->work, ipa3_wq_handle_rx);
-			INIT_DELAYED_WORK(&sys->switch_to_intr_work,
-				ipa3_switch_to_intr_rx_work_func);
-		} else if (in->client ==
-				IPA_CLIENT_MEMCPY_DMA_SYNC_CONS) {
-			IPADBG("assigning policy to client:%d",
-				in->client);
-
-			sys->policy = IPA_POLICY_NOINTR_MODE;
-		}  else if (in->client == IPA_CLIENT_ODL_DPL_CONS) {
-			IPADBG("assigning policy to ODL client:%d\n",
-				in->client);
-			sys->ep->status.status_en = true;
-			sys->policy = IPA_POLICY_INTR_POLL_MODE;
-			INIT_WORK(&sys->work, ipa3_wq_handle_rx);
-			INIT_DELAYED_WORK(&sys->switch_to_intr_work,
-				ipa3_switch_to_intr_rx_work_func);
-			INIT_DELAYED_WORK(&sys->replenish_rx_work,
-				ipa3_replenish_rx_work_func);
-			atomic_set(&sys->curr_polling_state, 0);
-			sys->rx_buff_sz =
-				IPA_GENERIC_RX_BUFF_SZ(IPA_ODL_RX_BUFF_SZ);
-			sys->pyld_hdlr = ipa3_odl_dpl_rx_pyld_hdlr;
-			sys->get_skb = ipa3_get_skb_ipa_rx;
-			sys->free_skb = ipa3_free_skb_rx;
-			sys->free_rx_wrapper = ipa3_recycle_rx_wrapper;
-			sys->repl_hdlr = ipa3_replenish_rx_cache_recycle;
-			sys->rx_pool_sz = in->desc_fifo_sz /
-					IPA_FIFO_ELEMENT_SIZE - 1;
-		} else {
-			WARN(1, "Need to install a RX pipe hdlr\n");
-			return -EINVAL;
-		}
-	}
-
-	return 0;
-}
-
-/**
- * ipa3_tx_client_rx_notify_release() - Callback function
- * which will call the user supplied callback function to
- * release the skb, or release it on its own if no callback
- * function was supplied
- *
- * @user1: [in] - Data Descriptor
- * @user2: [in] - endpoint idx
- *
- * This notified callback is for the destination client
- * This function is supplied in ipa3_tx_dp_mul
- */
-static void ipa3_tx_client_rx_notify_release(void *user1, int user2)
-{
-	struct ipa_tx_data_desc *dd = (struct ipa_tx_data_desc *)user1;
-	int ep_idx = user2;
-
-	IPADBG_LOW("Received data desc anchor:%pK\n", dd);
-
-	atomic_inc(&ipa3_ctx->ep[ep_idx].avail_fifo_desc);
-	ipa3_ctx->ep[ep_idx].wstats.rx_pkts_status_rcvd++;
-
-  /* wlan host driver waits till tx complete before unload */
-	IPADBG_LOW("ep=%d fifo_desc_free_count=%d\n",
-		ep_idx, atomic_read(&ipa3_ctx->ep[ep_idx].avail_fifo_desc));
-	IPADBG_LOW("calling client notify callback with priv:%pK\n",
-		ipa3_ctx->ep[ep_idx].priv);
-
-	if (ipa3_ctx->ep[ep_idx].client_notify) {
-		ipa3_ctx->ep[ep_idx].client_notify(ipa3_ctx->ep[ep_idx].priv,
-				IPA_WRITE_DONE, (unsigned long)user1);
-		ipa3_ctx->ep[ep_idx].wstats.rx_hd_reply++;
-	}
-}
-/**
- * ipa3_tx_client_rx_pkt_status() - Callback function
- * which will call the user supplied callback function to
- * increase the available fifo descriptor
- *
- * @user1: [in] - Data Descriptor
- * @user2: [in] - endpoint idx
- *
- * This notified callback is for the destination client
- * This function is supplied in ipa3_tx_dp_mul
- */
-static void ipa3_tx_client_rx_pkt_status(void *user1, int user2)
-{
-	int ep_idx = user2;
-
-	atomic_inc(&ipa3_ctx->ep[ep_idx].avail_fifo_desc);
-	ipa3_ctx->ep[ep_idx].wstats.rx_pkts_status_rcvd++;
-}
-
-
-/**
- * ipa3_tx_dp_mul() - Data-path tx handler for multiple packets
- * @src: [in] - Client that is sending data
- * @ipa_tx_data_desc:	[in] data descriptors from wlan
- *
- * this is used for to transfer data descriptors that received
- * from WLAN1_PROD pipe to IPA HW
- *
- * The function will send data descriptors from WLAN1_PROD (one
- * at a time). Will set EOT flag for last descriptor Once this send was done
- * from transport point-of-view the IPA driver will get notified by the
- * supplied callback - ipa_gsi_irq_tx_notify_cb()
- *
- * ipa_gsi_irq_tx_notify_cb will call to the user supplied callback
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa3_tx_dp_mul(enum ipa_client_type src,
-			struct ipa_tx_data_desc *data_desc)
-{
-	/* The second byte in wlan header holds qmap id */
-#define IPA_WLAN_HDR_QMAP_ID_OFFSET 1
-	struct ipa_tx_data_desc *entry;
-	struct ipa3_sys_context *sys;
-	struct ipa3_desc desc[2];
-	u32 num_desc, cnt;
-	int ep_idx;
-
-	IPADBG_LOW("Received data desc anchor:%pK\n", data_desc);
-
-	spin_lock_bh(&ipa3_ctx->wc_memb.ipa_tx_mul_spinlock);
-
-	ep_idx = ipa3_get_ep_mapping(src);
-	if (unlikely(ep_idx == -1)) {
-		IPAERR("dest EP does not exist.\n");
-		goto fail_send;
-	}
-	IPADBG_LOW("ep idx:%d\n", ep_idx);
-	sys = ipa3_ctx->ep[ep_idx].sys;
-
-	if (unlikely(ipa3_ctx->ep[ep_idx].valid == 0)) {
-		IPAERR("dest EP not valid.\n");
-		goto fail_send;
-	}
-	sys->ep->wstats.rx_hd_rcvd++;
-
-	/* Calculate the number of descriptors */
-	num_desc = 0;
-	list_for_each_entry(entry, &data_desc->link, link) {
-		num_desc++;
-	}
-	IPADBG_LOW("Number of Data Descriptors:%d", num_desc);
-
-	if (atomic_read(&sys->ep->avail_fifo_desc) < num_desc) {
-		IPAERR("Insufficient data descriptors available\n");
-		goto fail_send;
-	}
-
-	/* Assign callback only for last data descriptor */
-	cnt = 0;
-	list_for_each_entry(entry, &data_desc->link, link) {
-		memset(desc, 0, 2 * sizeof(struct ipa3_desc));
-
-		IPADBG_LOW("Parsing data desc :%d\n", cnt);
-		cnt++;
-		((u8 *)entry->pyld_buffer)[IPA_WLAN_HDR_QMAP_ID_OFFSET] =
-			(u8)sys->ep->cfg.meta.qmap_id;
-
-		/* the tag field will be populated in ipa3_send() function */
-		desc[0].is_tag_status = true;
-		desc[1].pyld = entry->pyld_buffer;
-		desc[1].len = entry->pyld_len;
-		desc[1].type = IPA_DATA_DESC_SKB;
-		desc[1].user1 = data_desc;
-		desc[1].user2 = ep_idx;
-		IPADBG_LOW("priv:%pK pyld_buf:0x%pK pyld_len:%d\n",
-			entry->priv, desc[1].pyld, desc[1].len);
-
-		/* In case of last descriptor populate callback */
-		if (cnt == num_desc) {
-			IPADBG_LOW("data desc:%pK\n", data_desc);
-			desc[1].callback = ipa3_tx_client_rx_notify_release;
-		} else {
-			desc[1].callback = ipa3_tx_client_rx_pkt_status;
-		}
-
-		IPADBG_LOW("calling ipa3_send()\n");
-		if (ipa3_send(sys, 2, desc, true)) {
-			IPAERR("fail to send skb\n");
-			sys->ep->wstats.rx_pkt_leak += (cnt-1);
-			sys->ep->wstats.rx_dp_fail++;
-			goto fail_send;
-		}
-
-		if (atomic_read(&sys->ep->avail_fifo_desc) >= 0)
-			atomic_dec(&sys->ep->avail_fifo_desc);
-
-		sys->ep->wstats.rx_pkts_rcvd++;
-		IPADBG_LOW("ep=%d fifo desc=%d\n",
-			ep_idx, atomic_read(&sys->ep->avail_fifo_desc));
-	}
-
-	sys->ep->wstats.rx_hd_processed++;
-	spin_unlock_bh(&ipa3_ctx->wc_memb.ipa_tx_mul_spinlock);
-	return 0;
-
-fail_send:
-	spin_unlock_bh(&ipa3_ctx->wc_memb.ipa_tx_mul_spinlock);
-	return -EFAULT;
-
-}
-
-void ipa3_free_skb(struct ipa_rx_data *data)
-{
-	struct ipa3_rx_pkt_wrapper *rx_pkt;
-
-	spin_lock_bh(&ipa3_ctx->wc_memb.wlan_spinlock);
-
-	ipa3_ctx->wc_memb.total_tx_pkts_freed++;
-	rx_pkt = container_of(data, struct ipa3_rx_pkt_wrapper, data);
-
-	ipa3_skb_recycle(rx_pkt->data.skb);
-	(void)skb_put(rx_pkt->data.skb, IPA_WLAN_RX_BUFF_SZ);
-
-	list_add_tail(&rx_pkt->link,
-		&ipa3_ctx->wc_memb.wlan_comm_desc_list);
-	ipa3_ctx->wc_memb.wlan_comm_free_cnt++;
-
-	spin_unlock_bh(&ipa3_ctx->wc_memb.wlan_spinlock);
-}
-
-/* Functions added to support kernel tests */
-
-int ipa3_sys_setup(struct ipa_sys_connect_params *sys_in,
-			unsigned long *ipa_transport_hdl,
-			u32 *ipa_pipe_num, u32 *clnt_hdl, bool en_status)
-{
-	struct ipa3_ep_context *ep;
-	int ipa_ep_idx;
-	int result = -EINVAL;
-
-	if (sys_in == NULL || clnt_hdl == NULL) {
-		IPAERR("NULL args\n");
-		goto fail_gen;
-	}
-
-	if (ipa_transport_hdl == NULL || ipa_pipe_num == NULL) {
-		IPAERR("NULL args\n");
-		goto fail_gen;
-	}
-	if (sys_in->client >= IPA_CLIENT_MAX) {
-		IPAERR("bad parm client:%d\n", sys_in->client);
-		goto fail_gen;
-	}
-
-	ipa_ep_idx = ipa3_get_ep_mapping(sys_in->client);
-	if (ipa_ep_idx == -1) {
-		IPAERR("Invalid client :%d\n", sys_in->client);
-		goto fail_gen;
-	}
-
-	ep = &ipa3_ctx->ep[ipa_ep_idx];
-	IPA_ACTIVE_CLIENTS_INC_EP(sys_in->client);
-
-	if (ep->valid == 1) {
-		if (sys_in->client != IPA_CLIENT_APPS_WAN_PROD) {
-			IPAERR("EP %d already allocated\n", ipa_ep_idx);
-			goto fail_and_disable_clocks;
-		} else {
-			if (ipa3_cfg_ep_hdr(ipa_ep_idx,
-						&sys_in->ipa_ep_cfg.hdr)) {
-				IPAERR("fail to configure hdr prop of EP %d\n",
-						ipa_ep_idx);
-				result = -EFAULT;
-				goto fail_and_disable_clocks;
-			}
-			if (ipa3_cfg_ep_hdr_ext(ipa_ep_idx,
-						&sys_in->ipa_ep_cfg.hdr_ext)) {
-				IPAERR("fail config hdr_ext prop of EP %d\n",
-						ipa_ep_idx);
-				result = -EFAULT;
-				goto fail_and_disable_clocks;
-			}
-			if (ipa3_cfg_ep_cfg(ipa_ep_idx,
-						&sys_in->ipa_ep_cfg.cfg)) {
-				IPAERR("fail to configure cfg prop of EP %d\n",
-						ipa_ep_idx);
-				result = -EFAULT;
-				goto fail_and_disable_clocks;
-			}
-			IPAERR("client %d (ep: %d) overlay ok sys=%pK\n",
-					sys_in->client, ipa_ep_idx, ep->sys);
-			ep->client_notify = sys_in->notify;
-			ep->priv = sys_in->priv;
-			*clnt_hdl = ipa_ep_idx;
-			if (!ep->keep_ipa_awake)
-				IPA_ACTIVE_CLIENTS_DEC_EP(sys_in->client);
-
-			return 0;
-		}
-	}
-
-	memset(ep, 0, offsetof(struct ipa3_ep_context, sys));
-
-	ep->valid = 1;
-	ep->client = sys_in->client;
-	ep->client_notify = sys_in->notify;
-	ep->priv = sys_in->priv;
-	ep->keep_ipa_awake = true;
-	if (en_status) {
-		ep->status.status_en = true;
-		ep->status.status_ep = ipa_ep_idx;
-	}
-
-	result = ipa3_enable_data_path(ipa_ep_idx);
-	if (result) {
-		IPAERR("enable data path failed res=%d clnt=%d.\n",
-				 result, ipa_ep_idx);
-		goto fail_gen2;
-	}
-
-	if (!ep->skip_ep_cfg) {
-		if (ipa3_cfg_ep(ipa_ep_idx, &sys_in->ipa_ep_cfg)) {
-			IPAERR("fail to configure EP.\n");
-			goto fail_gen2;
-		}
-		if (ipa3_cfg_ep_status(ipa_ep_idx, &ep->status)) {
-			IPAERR("fail to configure status of EP.\n");
-			goto fail_gen2;
-		}
-		IPADBG("ep configuration successful\n");
-	} else {
-		IPADBG("skipping ep configuration\n");
-	}
-
-	*clnt_hdl = ipa_ep_idx;
-
-	*ipa_pipe_num = ipa_ep_idx;
-	*ipa_transport_hdl = ipa3_ctx->gsi_dev_hdl;
-
-	if (!ep->keep_ipa_awake)
-		IPA_ACTIVE_CLIENTS_DEC_EP(sys_in->client);
-
-	ipa3_ctx->skip_ep_cfg_shadow[ipa_ep_idx] = ep->skip_ep_cfg;
-	IPADBG("client %d (ep: %d) connected sys=%pK\n", sys_in->client,
-			ipa_ep_idx, ep->sys);
-
-	return 0;
-
-fail_gen2:
-fail_and_disable_clocks:
-	IPA_ACTIVE_CLIENTS_DEC_EP(sys_in->client);
-fail_gen:
-	return result;
-}
-
-int ipa3_sys_teardown(u32 clnt_hdl)
-{
-	struct ipa3_ep_context *ep;
-
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-	    ipa3_ctx->ep[clnt_hdl].valid == 0) {
-		IPAERR("bad parm(Either endpoint or client hdl invalid)\n");
-		return -EINVAL;
-	}
-
-	ep = &ipa3_ctx->ep[clnt_hdl];
-
-	if (!ep->keep_ipa_awake)
-		IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	ipa3_disable_data_path(clnt_hdl);
-	ep->valid = 0;
-
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	IPADBG("client (ep: %d) disconnected\n", clnt_hdl);
-
-	return 0;
-}
-
-int ipa3_sys_update_gsi_hdls(u32 clnt_hdl, unsigned long gsi_ch_hdl,
-	unsigned long gsi_ev_hdl)
-{
-	struct ipa3_ep_context *ep;
-
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-		ipa3_ctx->ep[clnt_hdl].valid == 0) {
-		IPAERR("bad parm(Either endpoint or client hdl invalid)\n");
-		return -EINVAL;
-	}
-
-	ep = &ipa3_ctx->ep[clnt_hdl];
-
-	ep->gsi_chan_hdl = gsi_ch_hdl;
-	ep->gsi_evt_ring_hdl = gsi_ev_hdl;
-
-	return 0;
-}
-
-static void ipa_gsi_evt_ring_err_cb(struct gsi_evt_err_notify *notify)
-{
-	switch (notify->evt_id) {
-	case GSI_EVT_OUT_OF_BUFFERS_ERR:
-		IPAERR("Got GSI_EVT_OUT_OF_BUFFERS_ERR\n");
-		break;
-	case GSI_EVT_OUT_OF_RESOURCES_ERR:
-		IPAERR("Got GSI_EVT_OUT_OF_RESOURCES_ERR\n");
-		break;
-	case GSI_EVT_UNSUPPORTED_INTER_EE_OP_ERR:
-		IPAERR("Got GSI_EVT_UNSUPPORTED_INTER_EE_OP_ERR\n");
-		break;
-	case GSI_EVT_EVT_RING_EMPTY_ERR:
-		IPAERR("Got GSI_EVT_EVT_RING_EMPTY_ERR\n");
-		break;
-	default:
-		IPAERR("Unexpected err evt: %d\n", notify->evt_id);
-	}
-}
-
-static void ipa_gsi_chan_err_cb(struct gsi_chan_err_notify *notify)
-{
-	switch (notify->evt_id) {
-	case GSI_CHAN_INVALID_TRE_ERR:
-		IPAERR("Got GSI_CHAN_INVALID_TRE_ERR\n");
-		break;
-	case GSI_CHAN_NON_ALLOCATED_EVT_ACCESS_ERR:
-		IPAERR("Got GSI_CHAN_NON_ALLOCATED_EVT_ACCESS_ERR\n");
-		break;
-	case GSI_CHAN_OUT_OF_BUFFERS_ERR:
-		IPAERR("Got GSI_CHAN_OUT_OF_BUFFERS_ERR\n");
-		break;
-	case GSI_CHAN_OUT_OF_RESOURCES_ERR:
-		IPAERR("Got GSI_CHAN_OUT_OF_RESOURCES_ERR\n");
-		break;
-	case GSI_CHAN_UNSUPPORTED_INTER_EE_OP_ERR:
-		IPAERR("Got GSI_CHAN_UNSUPPORTED_INTER_EE_OP_ERR\n");
-		break;
-	case GSI_CHAN_HWO_1_ERR:
-		IPAERR("Got GSI_CHAN_HWO_1_ERR\n");
-		break;
-	default:
-		IPAERR("Unexpected err evt: %d\n", notify->evt_id);
-	}
-}
-
-static void ipa_gsi_irq_tx_notify_cb(struct gsi_chan_xfer_notify *notify)
-{
-	struct ipa3_tx_pkt_wrapper *tx_pkt;
-
-	IPADBG_LOW("event %d notified\n", notify->evt_id);
-
-	switch (notify->evt_id) {
-	case GSI_CHAN_EVT_EOT:
-		atomic_set(&ipa3_ctx->transport_pm.eot_activity, 1);
-		tx_pkt = notify->xfer_user_data;
-		tx_pkt->xmit_done = true;
-		atomic_inc(&tx_pkt->sys->xmit_eot_cnt);
-		tasklet_schedule(&tx_pkt->sys->tasklet);
-		break;
-	default:
-		IPAERR("received unexpected event id %d\n", notify->evt_id);
-	}
-}
-
-void __ipa_gsi_irq_rx_scedule_poll(struct ipa3_sys_context *sys)
-{
-	bool clk_off;
-
-	atomic_set(&sys->curr_polling_state, 1);
-	ipa3_inc_acquire_wakelock();
-
-	/*
-	 * pm deactivate is done in wq context
-	 * or after NAPI poll
-	 */
-
-	clk_off = ipa_pm_activate(sys->pm_hdl);
-	if (!clk_off && sys->napi_obj) {
-		napi_schedule(sys->napi_obj);
-		return;
-	}
-	queue_work(sys->wq, &sys->work);
-	return;
-
-}
-
-static void ipa_gsi_irq_rx_notify_cb(struct gsi_chan_xfer_notify *notify)
-{
-	struct ipa3_sys_context *sys;
-
-	if (!notify) {
-		IPAERR("gsi notify is NULL.\n");
-		return;
-	}
-	IPADBG_LOW("event %d notified\n", notify->evt_id);
-
-	sys = (struct ipa3_sys_context *)notify->chan_user_data;
-
-
-	sys->ep->xfer_notify_valid = true;
-	sys->ep->xfer_notify = *notify;
-
-	switch (notify->evt_id) {
-	case GSI_CHAN_EVT_EOT:
-	case GSI_CHAN_EVT_EOB:
-		atomic_set(&ipa3_ctx->transport_pm.eot_activity, 1);
-		if (!atomic_read(&sys->curr_polling_state)) {
-			/* put the gsi channel into polling mode */
-			gsi_config_channel_mode(sys->ep->gsi_chan_hdl,
-				GSI_CHAN_MODE_POLL);
-			__ipa_gsi_irq_rx_scedule_poll(sys);
-		}
-		break;
-	default:
-		IPAERR("received unexpected event id %d\n", notify->evt_id);
-	}
-}
-
-static void ipa_dma_gsi_irq_rx_notify_cb(struct gsi_chan_xfer_notify *notify)
-{
-	struct ipa3_sys_context *sys;
-
-	if (!notify) {
-		IPAERR("gsi notify is NULL.\n");
-		return;
-	}
-	IPADBG_LOW("event %d notified\n", notify->evt_id);
-
-	sys = (struct ipa3_sys_context *)notify->chan_user_data;
-	if (sys->ep->client == IPA_CLIENT_MEMCPY_DMA_SYNC_CONS) {
-		IPAERR("IRQ_RX Callback was called for DMA_SYNC_CONS.\n");
-		return;
-	}
-
-	sys->ep->xfer_notify_valid = true;
-	sys->ep->xfer_notify = *notify;
-
-	switch (notify->evt_id) {
-	case GSI_CHAN_EVT_EOT:
-		if (!atomic_read(&sys->curr_polling_state)) {
-			/* put the gsi channel into polling mode */
-			gsi_config_channel_mode(sys->ep->gsi_chan_hdl,
-				GSI_CHAN_MODE_POLL);
-			ipa3_inc_acquire_wakelock();
-			atomic_set(&sys->curr_polling_state, 1);
-			queue_work(sys->wq, &sys->work);
-		}
-		break;
-	default:
-		IPAERR("received unexpected event id %d\n", notify->evt_id);
-	}
-}
-
-int ipa3_alloc_common_event_ring(void)
-{
-	struct gsi_evt_ring_props gsi_evt_ring_props;
-	dma_addr_t evt_dma_addr;
-	int result;
-
-	memset(&gsi_evt_ring_props, 0, sizeof(gsi_evt_ring_props));
-	gsi_evt_ring_props.intf = GSI_EVT_CHTYPE_GPI_EV;
-	gsi_evt_ring_props.intr = GSI_INTR_IRQ;
-	gsi_evt_ring_props.re_size = GSI_EVT_RING_RE_SIZE_16B;
-
-	gsi_evt_ring_props.ring_len = IPA_COMMON_EVENT_RING_SIZE;
-
-	gsi_evt_ring_props.ring_base_vaddr =
-		dma_alloc_coherent(ipa3_ctx->pdev,
-		gsi_evt_ring_props.ring_len, &evt_dma_addr, GFP_KERNEL);
-	if (!gsi_evt_ring_props.ring_base_vaddr) {
-		IPAERR("fail to dma alloc %u bytes\n",
-			gsi_evt_ring_props.ring_len);
-		return -ENOMEM;
-	}
-	gsi_evt_ring_props.ring_base_addr = evt_dma_addr;
-	gsi_evt_ring_props.int_modt = 0;
-	gsi_evt_ring_props.int_modc = 1; /* moderation comes from channel*/
-	gsi_evt_ring_props.rp_update_addr = 0;
-	gsi_evt_ring_props.exclusive = false;
-	gsi_evt_ring_props.err_cb = ipa_gsi_evt_ring_err_cb;
-	gsi_evt_ring_props.user_data = NULL;
-
-	result = gsi_alloc_evt_ring(&gsi_evt_ring_props,
-		ipa3_ctx->gsi_dev_hdl, &ipa3_ctx->gsi_evt_comm_hdl);
-	if (result) {
-		IPAERR("gsi_alloc_evt_ring failed %d\n", result);
-		return result;
-	}
-	ipa3_ctx->gsi_evt_comm_ring_rem = IPA_COMMON_EVENT_RING_SIZE;
-
-	return 0;
-}
-
-static int ipa_gsi_setup_channel(struct ipa_sys_connect_params *in,
-	struct ipa3_ep_context *ep)
-{
-	u32 ring_size;
-	int result;
-	gfp_t mem_flag = GFP_KERNEL;
-	u32 coale_ep_idx;
-
-	if (in->client == IPA_CLIENT_APPS_WAN_CONS ||
-		in->client == IPA_CLIENT_APPS_WAN_COAL_CONS ||
-		in->client == IPA_CLIENT_APPS_WAN_PROD)
-		mem_flag = GFP_ATOMIC;
-
-	if (!ep) {
-		IPAERR("EP context is empty\n");
-		return -EINVAL;
-	}
-	coale_ep_idx = ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS);
-	/*
-	 * GSI ring length is calculated based on the desc_fifo_sz
-	 * which was meant to define the BAM desc fifo. GSI descriptors
-	 * are 16B as opposed to 8B for BAM.
-	 */
-	ring_size = 2 * in->desc_fifo_sz;
-	ep->gsi_evt_ring_hdl = ~0;
-	if (ep->sys->use_comm_evt_ring) {
-		if (ipa3_ctx->gsi_evt_comm_ring_rem < ring_size) {
-			IPAERR("not enough space in common event ring\n");
-			IPAERR("available: %d needed: %d\n",
-				ipa3_ctx->gsi_evt_comm_ring_rem,
-				ring_size);
-			WARN_ON(1);
-			return -EFAULT;
-		}
-		ipa3_ctx->gsi_evt_comm_ring_rem -= (ring_size);
-		ep->gsi_evt_ring_hdl = ipa3_ctx->gsi_evt_comm_hdl;
-	} else if (in->client == IPA_CLIENT_APPS_WAN_CONS &&
-			coale_ep_idx != IPA_EP_NOT_ALLOCATED &&
-			ipa3_ctx->ep[coale_ep_idx].valid == 1) {
-		IPADBG("Wan consumer pipe configured\n");
-		result = ipa_gsi_setup_coal_def_channel(in, ep,
-					&ipa3_ctx->ep[coale_ep_idx]);
-		if (result) {
-			IPAERR("Failed to setup default coal GSI channel\n");
-			goto fail_setup_event_ring;
-		}
-		return result;
-	} else if (ep->sys->policy != IPA_POLICY_NOINTR_MODE ||
-			IPA_CLIENT_IS_CONS(ep->client)) {
-		result = ipa_gsi_setup_event_ring(ep, ring_size, mem_flag);
-		if (result)
-			goto fail_setup_event_ring;
-	}
-	result = ipa_gsi_setup_transfer_ring(ep, ring_size,
-		ep->sys, mem_flag);
-	if (result)
-		goto fail_setup_transfer_ring;
-
-	if (ep->client == IPA_CLIENT_MEMCPY_DMA_SYNC_CONS)
-		gsi_config_channel_mode(ep->gsi_chan_hdl,
-				GSI_CHAN_MODE_POLL);
-	return 0;
-
-fail_setup_transfer_ring:
-	if (ep->gsi_mem_info.evt_ring_base_vaddr)
-		dma_free_coherent(ipa3_ctx->pdev, ep->gsi_mem_info.evt_ring_len,
-			ep->gsi_mem_info.evt_ring_base_vaddr,
-			ep->gsi_mem_info.evt_ring_base_addr);
-fail_setup_event_ring:
-	IPAERR("Return with err: %d\n", result);
-	return result;
-}
-
-static int ipa_gsi_setup_event_ring(struct ipa3_ep_context *ep,
-	u32 ring_size, gfp_t mem_flag)
-{
-	struct gsi_evt_ring_props gsi_evt_ring_props;
-	dma_addr_t evt_dma_addr;
-	int result;
-
-	evt_dma_addr = 0;
-	memset(&gsi_evt_ring_props, 0, sizeof(gsi_evt_ring_props));
-	gsi_evt_ring_props.intf = GSI_EVT_CHTYPE_GPI_EV;
-	gsi_evt_ring_props.intr = GSI_INTR_IRQ;
-	gsi_evt_ring_props.re_size = GSI_EVT_RING_RE_SIZE_16B;
-	gsi_evt_ring_props.ring_len = ring_size;
-	gsi_evt_ring_props.ring_base_vaddr =
-		dma_alloc_coherent(ipa3_ctx->pdev, gsi_evt_ring_props.ring_len,
-		&evt_dma_addr, mem_flag);
-	if (!gsi_evt_ring_props.ring_base_vaddr) {
-		IPAERR("fail to dma alloc %u bytes\n",
-			gsi_evt_ring_props.ring_len);
-		return -ENOMEM;
-	}
-	gsi_evt_ring_props.ring_base_addr = evt_dma_addr;
-
-	/* copy mem info */
-	ep->gsi_mem_info.evt_ring_len = gsi_evt_ring_props.ring_len;
-	ep->gsi_mem_info.evt_ring_base_addr =
-		gsi_evt_ring_props.ring_base_addr;
-	ep->gsi_mem_info.evt_ring_base_vaddr =
-		gsi_evt_ring_props.ring_base_vaddr;
-
-	if (ep->sys->napi_obj) {
-		gsi_evt_ring_props.int_modt = IPA_GSI_EVT_RING_INT_MODT;
-		gsi_evt_ring_props.int_modc = IPA_GSI_EVT_RING_INT_MODC;
-	} else {
-		gsi_evt_ring_props.int_modt = IPA_GSI_EVT_RING_INT_MODT;
-		gsi_evt_ring_props.int_modc = 1;
-	}
-
-	IPADBG("client=%d moderation threshold cycles=%u cnt=%u\n",
-		ep->client,
-		gsi_evt_ring_props.int_modt,
-		gsi_evt_ring_props.int_modc);
-	gsi_evt_ring_props.rp_update_addr = 0;
-	gsi_evt_ring_props.exclusive = true;
-	gsi_evt_ring_props.err_cb = ipa_gsi_evt_ring_err_cb;
-	gsi_evt_ring_props.user_data = NULL;
-
-	result = gsi_alloc_evt_ring(&gsi_evt_ring_props,
-		ipa3_ctx->gsi_dev_hdl, &ep->gsi_evt_ring_hdl);
-	if (result != GSI_STATUS_SUCCESS)
-		goto fail_alloc_evt_ring;
-
-	return 0;
-
-fail_alloc_evt_ring:
-	if (ep->gsi_mem_info.evt_ring_base_vaddr)
-		dma_free_coherent(ipa3_ctx->pdev, ep->gsi_mem_info.evt_ring_len,
-			ep->gsi_mem_info.evt_ring_base_vaddr,
-			ep->gsi_mem_info.evt_ring_base_addr);
-	IPAERR("Return with err: %d\n", result);
-	return result;
-}
-
-static int ipa_gsi_setup_transfer_ring(struct ipa3_ep_context *ep,
-	u32 ring_size, struct ipa3_sys_context *user_data, gfp_t mem_flag)
-{
-	dma_addr_t dma_addr;
-	union __packed gsi_channel_scratch ch_scratch;
-	struct gsi_chan_props gsi_channel_props;
-	const struct ipa_gsi_ep_config *gsi_ep_info;
-	int result;
-
-	memset(&gsi_channel_props, 0, sizeof(gsi_channel_props));
-	if (ep->client == IPA_CLIENT_APPS_WAN_COAL_CONS)
-		gsi_channel_props.prot = GSI_CHAN_PROT_GCI;
-	else
-		gsi_channel_props.prot = GSI_CHAN_PROT_GPI;
-	if (IPA_CLIENT_IS_PROD(ep->client)) {
-		gsi_channel_props.dir = GSI_CHAN_DIR_TO_GSI;
-	} else {
-		gsi_channel_props.dir = GSI_CHAN_DIR_FROM_GSI;
-		gsi_channel_props.max_re_expected = ep->sys->rx_pool_sz;
-	}
-
-	gsi_ep_info = ipa3_get_gsi_ep_info(ep->client);
-	if (!gsi_ep_info) {
-		IPAERR("Failed getting GSI EP info for client=%d\n",
-		       ep->client);
-		result = -EINVAL;
-		goto fail_get_gsi_ep_info;
-	} else {
-		gsi_channel_props.ch_id = gsi_ep_info->ipa_gsi_chan_num;
-	}
-
-	gsi_channel_props.evt_ring_hdl = ep->gsi_evt_ring_hdl;
-	gsi_channel_props.re_size = GSI_CHAN_RE_SIZE_16B;
-	gsi_channel_props.ring_len = ring_size;
-
-	gsi_channel_props.ring_base_vaddr =
-		dma_alloc_coherent(ipa3_ctx->pdev, gsi_channel_props.ring_len,
-			&dma_addr, mem_flag);
-	if (!gsi_channel_props.ring_base_vaddr) {
-		IPAERR("fail to dma alloc %u bytes\n",
-			gsi_channel_props.ring_len);
-		result = -ENOMEM;
-		goto fail_alloc_channel_ring;
-	}
-	gsi_channel_props.ring_base_addr = dma_addr;
-
-	/* copy mem info */
-	ep->gsi_mem_info.chan_ring_len = gsi_channel_props.ring_len;
-	ep->gsi_mem_info.chan_ring_base_addr =
-		gsi_channel_props.ring_base_addr;
-	ep->gsi_mem_info.chan_ring_base_vaddr =
-		gsi_channel_props.ring_base_vaddr;
-
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0)
-		gsi_channel_props.use_db_eng = GSI_CHAN_DIRECT_MODE;
-	else
-		gsi_channel_props.use_db_eng = GSI_CHAN_DB_MODE;
-	gsi_channel_props.max_prefetch = GSI_ONE_PREFETCH_SEG;
-	if (ep->client == IPA_CLIENT_APPS_CMD_PROD)
-		gsi_channel_props.low_weight = IPA_GSI_MAX_CH_LOW_WEIGHT;
-	else
-		gsi_channel_props.low_weight = 1;
-	gsi_channel_props.prefetch_mode = gsi_ep_info->prefetch_mode;
-	gsi_channel_props.empty_lvl_threshold = gsi_ep_info->prefetch_threshold;
-	gsi_channel_props.chan_user_data = user_data;
-	gsi_channel_props.err_cb = ipa_gsi_chan_err_cb;
-	if (IPA_CLIENT_IS_PROD(ep->client))
-		gsi_channel_props.xfer_cb = ipa_gsi_irq_tx_notify_cb;
-	else
-		gsi_channel_props.xfer_cb = ipa_gsi_irq_rx_notify_cb;
-	if (IPA_CLIENT_IS_MEMCPY_DMA_CONS(ep->client))
-		gsi_channel_props.xfer_cb = ipa_dma_gsi_irq_rx_notify_cb;
-
-	if (IPA_CLIENT_IS_CONS(ep->client))
-		gsi_channel_props.cleanup_cb = free_rx_pkt;
-
-	result = gsi_alloc_channel(&gsi_channel_props, ipa3_ctx->gsi_dev_hdl,
-		&ep->gsi_chan_hdl);
-	if (result != GSI_STATUS_SUCCESS) {
-		IPAERR("Failed to alloc GSI chan.\n");
-		goto fail_alloc_channel;
-	}
-
-	memset(&ch_scratch, 0, sizeof(ch_scratch));
-	/*
-	 * Update scratch for MCS smart prefetch:
-	 * Starting IPA4.5, smart prefetch implemented by H/W.
-	 * At IPA 4.0/4.1/4.2, we do not use MCS smart prefetch
-	 *  so keep the fields zero.
-	 */
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_0) {
-		ch_scratch.gpi.max_outstanding_tre =
-			gsi_ep_info->ipa_if_tlv * GSI_CHAN_RE_SIZE_16B;
-		ch_scratch.gpi.outstanding_threshold =
-			2 * GSI_CHAN_RE_SIZE_16B;
-	}
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_5)
-		ch_scratch.gpi.dl_nlo_channel = 0;
-	result = gsi_write_channel_scratch(ep->gsi_chan_hdl, ch_scratch);
-	if (result != GSI_STATUS_SUCCESS) {
-		IPAERR("failed to write scratch %d\n", result);
-		goto fail_write_channel_scratch;
-	}
-	return 0;
-
-fail_write_channel_scratch:
-	if (gsi_dealloc_channel(ep->gsi_chan_hdl)
-		!= GSI_STATUS_SUCCESS) {
-		IPAERR("Failed to dealloc GSI chan.\n");
-		WARN_ON(1);
-	}
-fail_alloc_channel:
-	dma_free_coherent(ipa3_ctx->pdev, ep->gsi_mem_info.chan_ring_len,
-			ep->gsi_mem_info.chan_ring_base_vaddr,
-			ep->gsi_mem_info.chan_ring_base_addr);
-fail_alloc_channel_ring:
-fail_get_gsi_ep_info:
-	if (ep->gsi_evt_ring_hdl != ~0) {
-		gsi_dealloc_evt_ring(ep->gsi_evt_ring_hdl);
-		ep->gsi_evt_ring_hdl = ~0;
-	}
-	return result;
-}
-
-static int ipa_gsi_setup_coal_def_channel(struct ipa_sys_connect_params *in,
-	struct ipa3_ep_context *ep, struct ipa3_ep_context *coal_ep)
-{
-	u32 ring_size;
-	int result;
-
-	ring_size = 2 * in->desc_fifo_sz;
-
-	/* copy event ring handle */
-	ep->gsi_evt_ring_hdl = coal_ep->gsi_evt_ring_hdl;
-
-	result = ipa_gsi_setup_transfer_ring(ep, ring_size,
-		coal_ep->sys, GFP_ATOMIC);
-	if (result) {
-		if (ep->gsi_mem_info.evt_ring_base_vaddr)
-			dma_free_coherent(ipa3_ctx->pdev,
-					ep->gsi_mem_info.chan_ring_len,
-					ep->gsi_mem_info.chan_ring_base_vaddr,
-					ep->gsi_mem_info.chan_ring_base_addr);
-		IPAERR("Destroying WAN_COAL_CONS evt_ring");
-		if (ep->gsi_evt_ring_hdl != ~0) {
-			gsi_dealloc_evt_ring(ep->gsi_evt_ring_hdl);
-			ep->gsi_evt_ring_hdl = ~0;
-		}
-		IPAERR("Return with err: %d\n", result);
-		return result;
-	}
-	return 0;
-}
-
-static int ipa_populate_tag_field(struct ipa3_desc *desc,
-		struct ipa3_tx_pkt_wrapper *tx_pkt,
-		struct ipahal_imm_cmd_pyld **tag_pyld_ret)
-{
-	struct ipahal_imm_cmd_pyld *tag_pyld;
-	struct ipahal_imm_cmd_ip_packet_tag_status tag_cmd = {0};
-
-	/* populate tag field only if it is NULL */
-	if (desc->pyld == NULL) {
-		tag_cmd.tag = pointer_to_tag_wa(tx_pkt);
-		tag_pyld = ipahal_construct_imm_cmd(
-			IPA_IMM_CMD_IP_PACKET_TAG_STATUS, &tag_cmd, true);
-		if (unlikely(!tag_pyld)) {
-			IPAERR("Failed to construct ip_packet_tag_status\n");
-			return -EFAULT;
-		}
-		/*
-		 * This is for 32-bit pointer, will need special
-		 * handling if 64-bit pointer is used
-		 */
-		IPADBG_LOW("tx_pkt sent in tag: 0x%pK\n", tx_pkt);
-		desc->pyld = tag_pyld->data;
-		desc->opcode = tag_pyld->opcode;
-		desc->len = tag_pyld->len;
-		desc->user1 = tag_pyld;
-		desc->type = IPA_IMM_CMD_DESC;
-		desc->callback = ipa3_tag_destroy_imm;
-
-		*tag_pyld_ret = tag_pyld;
-	}
-	return 0;
-}
-
-static int ipa_poll_gsi_pkt(struct ipa3_sys_context *sys,
-		struct gsi_chan_xfer_notify *notify)
-{
-	int unused_var;
-
-	return ipa_poll_gsi_n_pkt(sys, notify, 1, &unused_var);
-}
-
-
-static int ipa_poll_gsi_n_pkt(struct ipa3_sys_context *sys,
-		struct gsi_chan_xfer_notify *notify,
-		int expected_num, int *actual_num)
-{
-	int ret;
-	int idx = 0;
-	int poll_num = 0;
-
-	if (!actual_num || expected_num <= 0 ||
-		expected_num > IPA_WAN_NAPI_MAX_FRAMES) {
-		IPAERR("bad params actual_num=%pK expected_num=%d\n",
-			actual_num, expected_num);
-		return GSI_STATUS_INVALID_PARAMS;
-	}
-
-	if (sys->ep->xfer_notify_valid) {
-		*notify = sys->ep->xfer_notify;
-		sys->ep->xfer_notify_valid = false;
-		idx++;
-	}
-	if (expected_num == idx) {
-		*actual_num = idx;
-		return GSI_STATUS_SUCCESS;
-	}
-
-	ret = gsi_poll_n_channel(sys->ep->gsi_chan_hdl,
-		&notify[idx], expected_num - idx, &poll_num);
-	if (ret == GSI_STATUS_POLL_EMPTY) {
-		if (idx) {
-			*actual_num = idx;
-			return GSI_STATUS_SUCCESS;
-		}
-		*actual_num = 0;
-		return ret;
-	} else if (ret != GSI_STATUS_SUCCESS) {
-		if (idx) {
-			*actual_num = idx;
-			return GSI_STATUS_SUCCESS;
-		}
-		*actual_num = 0;
-		IPAERR("Poll channel err: %d\n", ret);
-		return ret;
-	}
-
-	*actual_num = idx + poll_num;
-	return ret;
-}
-
-/**
- * ipa3_rx_poll() - Poll the rx packets from IPA HW. This
- * function is exectued in the softirq context
- *
- * if input budget is zero, the driver switches back to
- * interrupt mode.
- *
- * return number of polled packets, on error 0(zero)
- */
-int ipa3_rx_poll(u32 clnt_hdl, int weight)
-{
-	struct ipa3_ep_context *ep;
-	int ret;
-	int cnt = 0;
-	int num = 0;
-	int remain_aggr_weight;
-	struct ipa_active_client_logging_info log;
-	struct gsi_chan_xfer_notify notify[IPA_WAN_NAPI_MAX_FRAMES];
-
-	IPA_ACTIVE_CLIENTS_PREP_SPECIAL(log, "NAPI");
-
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-		ipa3_ctx->ep[clnt_hdl].valid == 0) {
-		IPAERR("bad parm 0x%x\n", clnt_hdl);
-		return cnt;
-	}
-
-	remain_aggr_weight = weight / IPA_WAN_AGGR_PKT_CNT;
-
-	if (remain_aggr_weight > IPA_WAN_NAPI_MAX_FRAMES) {
-		IPAERR("NAPI weight is higher than expected\n");
-		IPAERR("expected %d got %d\n",
-			IPA_WAN_NAPI_MAX_FRAMES, remain_aggr_weight);
-		return -EINVAL;
-	}
-
-	ep = &ipa3_ctx->ep[clnt_hdl];
-start_poll:
-	while (remain_aggr_weight > 0 &&
-			atomic_read(&ep->sys->curr_polling_state)) {
-		atomic_set(&ipa3_ctx->transport_pm.eot_activity, 1);
-		if (ipa3_ctx->enable_napi_chain) {
-			ret = ipa_poll_gsi_n_pkt(ep->sys, notify,
-				remain_aggr_weight, &num);
-		} else {
-			ret = ipa_poll_gsi_n_pkt(ep->sys, notify,
-				1, &num);
-		}
-		if (ret)
-			break;
-
-		trace_ipa3_rx_poll_num(num);
-		ipa3_wq_rx_napi_chain(ep->sys, notify, num);
-		remain_aggr_weight -= num;
-
-		trace_ipa3_rx_poll_cnt(ep->sys->len);
-		if (ep->sys->len == 0) {
-			if (remain_aggr_weight == 0)
-				cnt--;
-			break;
-		}
-	}
-	cnt += weight - remain_aggr_weight * IPA_WAN_AGGR_PKT_CNT;
-	/* call repl_hdlr before napi_reschedule / napi_complete */
-	if (cnt)
-		ep->sys->repl_hdlr(ep->sys);
-	if (cnt < weight) {
-		napi_complete(ep->sys->napi_obj);
-		ret = ipa3_rx_switch_to_intr_mode(ep->sys);
-		if (ret == -GSI_STATUS_PENDING_IRQ &&
-				napi_reschedule(ep->sys->napi_obj))
-			goto start_poll;
-		ipa_pm_deferred_deactivate(ep->sys->pm_hdl);
-	}
-	return cnt;
-}
-
-static unsigned long tag_to_pointer_wa(uint64_t tag)
-{
-	return 0xFFFF000000000000 | (unsigned long) tag;
-}
-
-static uint64_t pointer_to_tag_wa(struct ipa3_tx_pkt_wrapper *tx_pkt)
-{
-	u16 temp;
-	/* Add the check but it might have throughput issue */
-	if (BITS_PER_LONG == 64) {
-		temp = (u16) (~((unsigned long) tx_pkt &
-			0xFFFF000000000000) >> 48);
-		if (temp) {
-			IPAERR("The 16 prefix is not all 1s (%pK)\n",
-			tx_pkt);
-			/*
-			 * We need all addresses starting at 0xFFFF to
-			 * pass it to HW.
-			 */
-			ipa_assert();
-		}
-	}
-	return (unsigned long)tx_pkt & 0x0000FFFFFFFFFFFF;
-}
-
-/**
- * ipa_gsi_ch20_wa() - software workaround for IPA GSI channel 20
- *
- * A hardware limitation requires to avoid using GSI physical channel 20.
- * This function allocates GSI physical channel 20 and holds it to prevent
- * others to use it.
- *
- * Return codes: 0 on success, negative on failure
- */
-int ipa_gsi_ch20_wa(void)
-{
-	struct gsi_chan_props gsi_channel_props;
-	dma_addr_t dma_addr;
-	int result;
-	int i;
-	unsigned long chan_hdl[IPA_GSI_CH_20_WA_NUM_CH_TO_ALLOC];
-	unsigned long chan_hdl_to_keep;
-
-
-	memset(&gsi_channel_props, 0, sizeof(gsi_channel_props));
-	gsi_channel_props.prot = GSI_CHAN_PROT_GPI;
-	gsi_channel_props.dir = GSI_CHAN_DIR_TO_GSI;
-	gsi_channel_props.evt_ring_hdl = ~0;
-	gsi_channel_props.re_size = GSI_CHAN_RE_SIZE_16B;
-	gsi_channel_props.ring_len = 4 * gsi_channel_props.re_size;
-	gsi_channel_props.ring_base_vaddr =
-		dma_alloc_coherent(ipa3_ctx->pdev, gsi_channel_props.ring_len,
-		&dma_addr, 0);
-	gsi_channel_props.ring_base_addr = dma_addr;
-
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0)
-		gsi_channel_props.use_db_eng = GSI_CHAN_DIRECT_MODE;
-	else
-		gsi_channel_props.use_db_eng = GSI_CHAN_DB_MODE;
-
-	gsi_channel_props.max_prefetch = GSI_ONE_PREFETCH_SEG;
-	gsi_channel_props.low_weight = 1;
-	gsi_channel_props.err_cb = ipa_gsi_chan_err_cb;
-	gsi_channel_props.xfer_cb = ipa_gsi_irq_tx_notify_cb;
-
-	/* first allocate channels up to channel 20 */
-	for (i = 0; i < IPA_GSI_CH_20_WA_NUM_CH_TO_ALLOC; i++) {
-		gsi_channel_props.ch_id = i;
-		result = gsi_alloc_channel(&gsi_channel_props,
-			ipa3_ctx->gsi_dev_hdl,
-			&chan_hdl[i]);
-		if (result != GSI_STATUS_SUCCESS) {
-			IPAERR("failed to alloc channel %d err %d\n",
-				i, result);
-			return result;
-		}
-	}
-
-	/* allocate channel 20 */
-	gsi_channel_props.ch_id = IPA_GSI_CH_20_WA_VIRT_CHAN;
-	result = gsi_alloc_channel(&gsi_channel_props, ipa3_ctx->gsi_dev_hdl,
-		&chan_hdl_to_keep);
-	if (result != GSI_STATUS_SUCCESS) {
-		IPAERR("failed to alloc channel %d err %d\n",
-			i, result);
-		return result;
-	}
-
-	/* release all other channels */
-	for (i = 0; i < IPA_GSI_CH_20_WA_NUM_CH_TO_ALLOC; i++) {
-		result = gsi_dealloc_channel(chan_hdl[i]);
-		if (result != GSI_STATUS_SUCCESS) {
-			IPAERR("failed to dealloc channel %d err %d\n",
-				i, result);
-			return result;
-		}
-	}
-
-	/* DMA memory shall not be freed as it is used by channel 20 */
-	return 0;
-}
-
-/**
- * ipa_adjust_ra_buff_base_sz()
- *
- * Return value: the largest power of two which is smaller
- * than the input value
- */
-static u32 ipa_adjust_ra_buff_base_sz(u32 aggr_byte_limit)
-{
-	aggr_byte_limit += IPA_MTU;
-	aggr_byte_limit += IPA_GENERIC_RX_BUFF_LIMIT;
-	aggr_byte_limit--;
-	aggr_byte_limit |= aggr_byte_limit >> 1;
-	aggr_byte_limit |= aggr_byte_limit >> 2;
-	aggr_byte_limit |= aggr_byte_limit >> 4;
-	aggr_byte_limit |= aggr_byte_limit >> 8;
-	aggr_byte_limit |= aggr_byte_limit >> 16;
-	aggr_byte_limit++;
-	return aggr_byte_limit >> 1;
-}

ipa/ipa_v3/ipa_dt_replacement.c

@@ -1,874 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/errno.h>
-#include <linux/string.h>
-#include <linux/ioport.h>
-#include <linux/msm_ipa.h>
-#include "ipa_i.h"
-#include "ipa_emulation_stubs.h"
-
-# undef strsame
-# define strsame(x, y) \
-	(!strcmp((x), (y)))
-
-/*
- * The following enum values used to index tables below.
- */
-enum dtsi_index_e {
-	DTSI_INDEX_3_5_1 = 0,
-	DTSI_INDEX_4_0   = 1,
-	DTSI_INDEX_4_5   = 2,
-};
-
-struct dtsi_replacement_u32 {
-	char *key;
-	u32 value;
-};
-
-struct dtsi_replacement_u32_table {
-	struct dtsi_replacement_u32 *p_table;
-	u32 num_entries;
-};
-
-struct dtsi_replacement_bool {
-	char *key;
-	bool value;
-};
-
-struct dtsi_replacement_bool_table {
-	struct dtsi_replacement_bool *p_table;
-	u32 num_entries;
-};
-
-struct dtsi_replacement_u32_array {
-	char *key;
-	u32 *p_value;
-	u32 num_elements;
-};
-
-struct dtsi_replacement_u32_array_table {
-	struct dtsi_replacement_u32_array *p_table;
-	u32 num_entries;
-};
-
-struct dtsi_replacement_resource_table {
-	struct resource *p_table;
-	u32 num_entries;
-};
-
-/*
- * Any of the data below with _4_5 in the name represent data taken
- * from the 4.5 dtsi file.
- *
- * Any of the data below with _4_0 in the name represent data taken
- * from the 4.0 dtsi file.
- *
- * Any of the data below with _3_5_1 in the name represent data taken
- * from the 3.5.1 dtsi file.
- */
-static struct dtsi_replacement_bool ipa3_plat_drv_bool_4_5[] = {
-	{"qcom,use-ipa-tethering-bridge",       true},
-	{"qcom,modem-cfg-emb-pipe-flt",         true},
-	{"qcom,ipa-wdi2",                       false},
-	{"qcom,use-64-bit-dma-mask",            false},
-	{"qcom,bandwidth-vote-for-ipa",         true},
-	{"qcom,skip-uc-pipe-reset",             false},
-	{"qcom,tethered-flow-control",          false},
-	{"qcom,use-rg10-limitation-mitigation", false},
-	{"qcom,do-not-use-ch-gsi-20",           false},
-	{"qcom,use-ipa-pm",                     true},
-	{"qcom,register-collection-on-crash",   true},
-	{"qcom,testbus-collection-on-crash",    true},
-	{"qcom,non-tn-collection-on-crash",     true},
-};
-
-static struct dtsi_replacement_bool ipa3_plat_drv_bool_4_0[] = {
-	{"qcom,use-ipa-tethering-bridge",       true},
-	{"qcom,modem-cfg-emb-pipe-flt",         true},
-	{"qcom,ipa-wdi2",                       true},
-	{"qcom,use-64-bit-dma-mask",            false},
-	{"qcom,bandwidth-vote-for-ipa",         false},
-	{"qcom,skip-uc-pipe-reset",             false},
-	{"qcom,tethered-flow-control",          true},
-	{"qcom,use-rg10-limitation-mitigation", false},
-	{"qcom,do-not-use-ch-gsi-20",           false},
-	{"qcom,use-ipa-pm",                     false},
-	{"qcom,register-collection-on-crash",   true},
-	{"qcom,testbus-collection-on-crash",    true},
-	{"qcom,non-tn-collection-on-crash",     true},
-};
-
-static struct dtsi_replacement_bool ipa3_plat_drv_bool_3_5_1[] = {
-	{"qcom,use-ipa-tethering-bridge",       true},
-	{"qcom,modem-cfg-emb-pipe-flt",         true},
-	{"qcom,ipa-wdi2",                       true},
-	{"qcom,use-64-bit-dma-mask",            false},
-	{"qcom,bandwidth-vote-for-ipa",         true},
-	{"qcom,skip-uc-pipe-reset",             false},
-	{"qcom,tethered-flow-control",          false},
-	{"qcom,use-rg10-limitation-mitigation", false},
-	{"qcom,do-not-use-ch-gsi-20",           false},
-	{"qcom,use-ipa-pm",                     false},
-	{"qcom,register-collection-on-crash",   true},
-	{"qcom,testbus-collection-on-crash",    true},
-	{"qcom,non-tn-collection-on-crash",     true},
-};
-
-static struct dtsi_replacement_bool_table
-ipa3_plat_drv_bool_table[] = {
-	{ ipa3_plat_drv_bool_3_5_1,
-	  ARRAY_SIZE(ipa3_plat_drv_bool_3_5_1) },
-	{ ipa3_plat_drv_bool_4_0,
-	  ARRAY_SIZE(ipa3_plat_drv_bool_4_0) },
-	{ ipa3_plat_drv_bool_4_5,
-	  ARRAY_SIZE(ipa3_plat_drv_bool_4_5) },
-};
-
-static struct dtsi_replacement_u32 ipa3_plat_drv_u32_4_5[] = {
-	{"qcom,ipa-hw-ver",                     IPA_HW_v4_5},
-	{"qcom,ipa-hw-mode",                    3},
-	{"qcom,wan-rx-ring-size",               192},
-	{"qcom,lan-rx-ring-size",               192},
-	{"qcom,ee",                             0},
-	{"qcom,msm-bus,num-cases",              5},
-	{"emulator-bar0-offset",                0x01C00000},
-	{"qcom,entire-ipa-block-size",          0x00100000},
-};
-
-static struct dtsi_replacement_u32 ipa3_plat_drv_u32_4_0[] = {
-	{"qcom,ipa-hw-ver",                     IPA_HW_v4_0},
-	{"qcom,ipa-hw-mode",                    3},
-	{"qcom,wan-rx-ring-size",               192},
-	{"qcom,lan-rx-ring-size",               192},
-	{"qcom,ee",                             0},
-	{"emulator-bar0-offset",                0x01C00000},
-	{"qcom,entire-ipa-block-size",          0x00100000},
-};
-
-static struct dtsi_replacement_u32 ipa3_plat_drv_u32_3_5_1[] = {
-	{"qcom,ipa-hw-ver",                     IPA_HW_v3_5_1},
-	{"qcom,ipa-hw-mode",                    3},
-	{"qcom,wan-rx-ring-size",               192},
-	{"qcom,lan-rx-ring-size",               192},
-	{"qcom,ee",                             0},
-	{"emulator-bar0-offset",                0x01C00000},
-	{"qcom,entire-ipa-block-size",          0x00100000},
-};
-
-static struct dtsi_replacement_u32_table ipa3_plat_drv_u32_table[] = {
-	{ ipa3_plat_drv_u32_3_5_1,
-	  ARRAY_SIZE(ipa3_plat_drv_u32_3_5_1) },
-	{ ipa3_plat_drv_u32_4_0,
-	  ARRAY_SIZE(ipa3_plat_drv_u32_4_0) },
-	{ ipa3_plat_drv_u32_4_5,
-	  ARRAY_SIZE(ipa3_plat_drv_u32_4_5) },
-};
-
-static u32 mhi_event_ring_id_limits_array_4_5[] = {
-	9, 10
-};
-
-static u32 mhi_event_ring_id_limits_array_4_0[] = {
-	9, 10
-};
-
-static u32 mhi_event_ring_id_limits_array_3_5_1[] = {
-	IPA_MHI_GSI_EVENT_RING_ID_START, IPA_MHI_GSI_EVENT_RING_ID_END
-};
-
-static u32 ipa_tz_unlock_reg_array_4_5[] = {
-	0x04043583c, 0x00001000
-};
-
-static u32 ipa_throughput_thresh_array_4_5[] = {
-	310, 600, 1000
-};
-
-static u32 ipa_tz_unlock_reg_array_4_0[] = {
-	0x04043583c, 0x00001000
-};
-
-static u32 ipa_tz_unlock_reg_array_3_5_1[] = {
-	0x04043583c, 0x00001000
-};
-
-struct dtsi_replacement_u32_array ipa3_plat_drv_u32_array_4_5[] = {
-	{"qcom,mhi-event-ring-id-limits",
-	 mhi_event_ring_id_limits_array_4_5,
-	 ARRAY_SIZE(mhi_event_ring_id_limits_array_4_5) },
-	{"qcom,ipa-tz-unlock-reg",
-	 ipa_tz_unlock_reg_array_4_5,
-	 ARRAY_SIZE(ipa_tz_unlock_reg_array_4_5) },
-	{"qcom,throughput-threshold",
-	 ipa_throughput_thresh_array_4_5,
-	 ARRAY_SIZE(ipa_throughput_thresh_array_4_5) },
-};
-
-struct dtsi_replacement_u32_array ipa3_plat_drv_u32_array_4_0[] = {
-	{"qcom,mhi-event-ring-id-limits",
-	 mhi_event_ring_id_limits_array_4_0,
-	 ARRAY_SIZE(mhi_event_ring_id_limits_array_4_0) },
-	{"qcom,ipa-tz-unlock-reg",
-	 ipa_tz_unlock_reg_array_4_0,
-	 ARRAY_SIZE(ipa_tz_unlock_reg_array_4_0) },
-};
-
-struct dtsi_replacement_u32_array ipa3_plat_drv_u32_array_3_5_1[] = {
-	{"qcom,mhi-event-ring-id-limits",
-	 mhi_event_ring_id_limits_array_3_5_1,
-	 ARRAY_SIZE(mhi_event_ring_id_limits_array_3_5_1) },
-	{"qcom,ipa-tz-unlock-reg",
-	 ipa_tz_unlock_reg_array_3_5_1,
-	 ARRAY_SIZE(ipa_tz_unlock_reg_array_3_5_1) },
-};
-
-struct dtsi_replacement_u32_array_table
-ipa3_plat_drv_u32_array_table[] = {
-	{ ipa3_plat_drv_u32_array_3_5_1,
-	  ARRAY_SIZE(ipa3_plat_drv_u32_array_3_5_1) },
-	{ ipa3_plat_drv_u32_array_4_0,
-	  ARRAY_SIZE(ipa3_plat_drv_u32_array_4_0) },
-	{ ipa3_plat_drv_u32_array_4_5,
-	  ARRAY_SIZE(ipa3_plat_drv_u32_array_4_5) },
-};
-
-#define INTCTRL_OFFSET       0x083C0000
-#define INTCTRL_SIZE         0x00000110
-
-#define IPA_BASE_OFFSET_4_5  0x01e00000
-#define IPA_BASE_SIZE_4_5    0x000c0000
-#define GSI_BASE_OFFSET_4_5  0x01e04000
-#define GSI_BASE_SIZE_4_5    0x00023000
-
-struct resource ipa3_plat_drv_resource_4_5[] = {
-	/*
-	 * PLEASE NOTE: The following offset values below ("ipa-base",
-	 * "gsi-base", and "intctrl-base") are used to calculate
-	 * offsets relative to the PCI BAR0 address provided by the
-	 * PCI probe.  After their use to calculate the offsets, they
-	 * are not used again, since PCI ultimately dictates where
-	 * things live.
-	 */
-	{
-		IPA_BASE_OFFSET_4_5,
-		(IPA_BASE_OFFSET_4_5 + IPA_BASE_SIZE_4_5),
-		"ipa-base",
-		IORESOURCE_MEM,
-		0,
-		NULL,
-		NULL,
-		NULL
-	},
-
-	{
-		GSI_BASE_OFFSET_4_5,
-		(GSI_BASE_OFFSET_4_5 + GSI_BASE_SIZE_4_5),
-		"gsi-base",
-		IORESOURCE_MEM,
-		0,
-		NULL,
-		NULL,
-		NULL
-	},
-
-	/*
-	 * The following entry is germane only to the emulator
-	 * environment.  It is needed to locate the emulator's PCI
-	 * interrupt controller...
-	 */
-	{
-		INTCTRL_OFFSET,
-		(INTCTRL_OFFSET + INTCTRL_SIZE),
-		"intctrl-base",
-		IORESOURCE_MEM,
-		0,
-		NULL,
-		NULL,
-		NULL
-	},
-
-	{
-		IPA_PIPE_MEM_START_OFST,
-		(IPA_PIPE_MEM_START_OFST + IPA_PIPE_MEM_SIZE),
-		"ipa-pipe-mem",
-		IORESOURCE_MEM,
-		0,
-		NULL,
-		NULL,
-		NULL
-	},
-
-	{
-		0,
-		0,
-		"gsi-irq",
-		IORESOURCE_IRQ,
-		0,
-		NULL,
-		NULL,
-		NULL
-	},
-
-	{
-		0,
-		0,
-		"ipa-irq",
-		IORESOURCE_IRQ,
-		0,
-		NULL,
-		NULL,
-		NULL
-	},
-};
-
-#define IPA_BASE_OFFSET_4_0  0x01e00000
-#define IPA_BASE_SIZE_4_0    0x00034000
-#define GSI_BASE_OFFSET_4_0  0x01e04000
-#define GSI_BASE_SIZE_4_0    0x00028000
-
-struct resource ipa3_plat_drv_resource_4_0[] = {
-	/*
-	 * PLEASE NOTE: The following offset values below ("ipa-base",
-	 * "gsi-base", and "intctrl-base") are used to calculate
-	 * offsets relative to the PCI BAR0 address provided by the
-	 * PCI probe.  After their use to calculate the offsets, they
-	 * are not used again, since PCI ultimately dictates where
-	 * things live.
-	 */
-	{
-		IPA_BASE_OFFSET_4_0,
-		(IPA_BASE_OFFSET_4_0 + IPA_BASE_SIZE_4_0),
-		"ipa-base",
-		IORESOURCE_MEM,
-		0,
-		NULL,
-		NULL,
-		NULL
-	},
-
-	{
-		GSI_BASE_OFFSET_4_0,
-		(GSI_BASE_OFFSET_4_0 + GSI_BASE_SIZE_4_0),
-		"gsi-base",
-		IORESOURCE_MEM,
-		0,
-		NULL,
-		NULL,
-		NULL
-	},
-
-	/*
-	 * The following entry is germane only to the emulator
-	 * environment.  It is needed to locate the emulator's PCI
-	 * interrupt controller...
-	 */
-	{
-		INTCTRL_OFFSET,
-		(INTCTRL_OFFSET + INTCTRL_SIZE),
-		"intctrl-base",
-		IORESOURCE_MEM,
-		0,
-		NULL,
-		NULL,
-		NULL
-	},
-
-	{
-		IPA_PIPE_MEM_START_OFST,
-		(IPA_PIPE_MEM_START_OFST + IPA_PIPE_MEM_SIZE),
-		"ipa-pipe-mem",
-		IORESOURCE_MEM,
-		0,
-		NULL,
-		NULL,
-		NULL
-	},
-
-	{
-		0,
-		0,
-		"gsi-irq",
-		IORESOURCE_IRQ,
-		0,
-		NULL,
-		NULL,
-		NULL
-	},
-
-	{
-		0,
-		0,
-		"ipa-irq",
-		IORESOURCE_IRQ,
-		0,
-		NULL,
-		NULL,
-		NULL
-	},
-};
-
-#define IPA_BASE_OFFSET_3_5_1  0x01e00000
-#define IPA_BASE_SIZE_3_5_1    0x00034000
-#define GSI_BASE_OFFSET_3_5_1  0x01e04000
-#define GSI_BASE_SIZE_3_5_1    0x0002c000
-
-struct resource ipa3_plat_drv_resource_3_5_1[] = {
-	/*
-	 * PLEASE NOTE: The following offset values below ("ipa-base",
-	 * "gsi-base", and "intctrl-base") are used to calculate
-	 * offsets relative to the PCI BAR0 address provided by the
-	 * PCI probe.  After their use to calculate the offsets, they
-	 * are not used again, since PCI ultimately dictates where
-	 * things live.
-	 */
-	{
-		IPA_BASE_OFFSET_3_5_1,
-		(IPA_BASE_OFFSET_3_5_1 + IPA_BASE_SIZE_3_5_1),
-		"ipa-base",
-		IORESOURCE_MEM,
-		0,
-		NULL,
-		NULL,
-		NULL
-	},
-
-	{
-		GSI_BASE_OFFSET_3_5_1,
-		(GSI_BASE_OFFSET_3_5_1 + GSI_BASE_SIZE_3_5_1),
-		"gsi-base",
-		IORESOURCE_MEM,
-		0,
-		NULL,
-		NULL,
-		NULL
-	},
-
-	/*
-	 * The following entry is germane only to the emulator
-	 * environment.  It is needed to locate the emulator's PCI
-	 * interrupt controller...
-	 */
-	{
-		INTCTRL_OFFSET,
-		(INTCTRL_OFFSET + INTCTRL_SIZE),
-		"intctrl-base",
-		IORESOURCE_MEM,
-		0,
-		NULL,
-		NULL,
-		NULL
-	},
-
-	{
-		IPA_PIPE_MEM_START_OFST,
-		(IPA_PIPE_MEM_START_OFST + IPA_PIPE_MEM_SIZE),
-		"ipa-pipe-mem",
-		IORESOURCE_MEM,
-		0,
-		NULL,
-		NULL,
-		NULL
-	},
-
-	{
-		0,
-		0,
-		"gsi-irq",
-		IORESOURCE_IRQ,
-		0,
-		NULL,
-		NULL,
-		NULL
-	},
-
-	{
-		0,
-		0,
-		"ipa-irq",
-		IORESOURCE_IRQ,
-		0,
-		NULL,
-		NULL,
-		NULL
-	},
-};
-
-struct dtsi_replacement_resource_table
-ipa3_plat_drv_resource_table[] = {
-	{ ipa3_plat_drv_resource_3_5_1,
-	  ARRAY_SIZE(ipa3_plat_drv_resource_3_5_1) },
-	{ ipa3_plat_drv_resource_4_0,
-	  ARRAY_SIZE(ipa3_plat_drv_resource_4_0) },
-	{ ipa3_plat_drv_resource_4_5,
-	  ARRAY_SIZE(ipa3_plat_drv_resource_4_5) },
-};
-
-/*
- * The following code uses the data above...
- */
-static u32 emulator_type_to_index(void)
-{
-	/*
-	 * Use the input parameter to the IPA driver loadable module,
-	 * which specifies the type of hardware the driver is running
-	 * on.
-	 */
-	u32 index = DTSI_INDEX_4_0;
-	uint emulation_type = ipa3_get_emulation_type();
-
-	switch (emulation_type) {
-	case IPA_HW_v3_5_1:
-		index = DTSI_INDEX_3_5_1;
-		break;
-	case IPA_HW_v4_0:
-		index = DTSI_INDEX_4_0;
-		break;
-	case IPA_HW_v4_5:
-		index = DTSI_INDEX_4_5;
-		break;
-	default:
-		break;
-	}
-
-	IPADBG("emulation_type(%u) emulation_index(%u)\n",
-	       emulation_type, index);
-
-	return index;
-}
-
-/* From include/linux/of.h */
-/**
- * emulator_of_property_read_bool - Find from a property
- * @np:         device node from which the property value is to be read.
- * @propname:   name of the property to be searched.
- *
- * Search for a property in a device node.
- * Returns true if the property exists false otherwise.
- */
-bool emulator_of_property_read_bool(
-	const struct device_node *np,
-	const char *propname)
-{
-	u16 i;
-	u32 index;
-	struct dtsi_replacement_bool *ipa3_plat_drv_boolP;
-
-	/*
-	 * Get the index for the type of hardware we're running on.
-	 * This is used as a table index.
-	 */
-	index = emulator_type_to_index();
-	if (index >= ARRAY_SIZE(ipa3_plat_drv_bool_table)) {
-		IPADBG(
-		    "Did not find ipa3_plat_drv_bool_table for index %u\n",
-		    index);
-		return false;
-	}
-
-	ipa3_plat_drv_boolP =
-	    ipa3_plat_drv_bool_table[index].p_table;
-
-	for (i = 0;
-	     i < ipa3_plat_drv_bool_table[index].num_entries;
-	     i++) {
-		if (strsame(ipa3_plat_drv_boolP[i].key, propname)) {
-			IPADBG(
-			    "Found value %u for propname %s index %u\n",
-			    ipa3_plat_drv_boolP[i].value,
-			    propname,
-			    index);
-			return ipa3_plat_drv_boolP[i].value;
-		}
-	}
-
-	IPADBG("Did not find match for propname %s index %u\n",
-	       propname,
-	       index);
-
-	return false;
-}
-
-/* From include/linux/of.h */
-int emulator_of_property_read_u32(
-	const struct device_node *np,
-	const char *propname,
-	u32 *out_value)
-{
-	u16 i;
-	u32 index;
-	struct dtsi_replacement_u32 *ipa3_plat_drv_u32P;
-
-	/*
-	 * Get the index for the type of hardware we're running on.
-	 * This is used as a table index.
-	 */
-	index = emulator_type_to_index();
-	if (index >= ARRAY_SIZE(ipa3_plat_drv_u32_table)) {
-		IPADBG(
-		    "Did not find ipa3_plat_drv_u32_table for index %u\n",
-		    index);
-		return false;
-	}
-
-	ipa3_plat_drv_u32P =
-	    ipa3_plat_drv_u32_table[index].p_table;
-
-	for (i = 0;
-	     i < ipa3_plat_drv_u32_table[index].num_entries;
-	     i++) {
-		if (strsame(ipa3_plat_drv_u32P[i].key, propname)) {
-			*out_value = ipa3_plat_drv_u32P[i].value;
-			IPADBG(
-			    "Found value %u for propname %s index %u\n",
-			    ipa3_plat_drv_u32P[i].value,
-			    propname,
-			    index);
-			return 0;
-		}
-	}
-
-	IPADBG("Did not find match for propname %s index %u\n",
-	       propname,
-	       index);
-
-	return -EINVAL;
-}
-
-/* From include/linux/of.h */
-/**
- * emulator_of_property_read_u32_array - Find and read an array of 32
- * bit integers from a property.
- *
- * @np:         device node from which the property value is to be read.
- * @propname:   name of the property to be searched.
- * @out_values: pointer to return value, modified only if return value is 0.
- * @sz:         number of array elements to read
- *
- * Search for a property in a device node and read 32-bit value(s) from
- * it. Returns 0 on success, -EINVAL if the property does not exist,
- * -ENODATA if property does not have a value, and -EOVERFLOW if the
- * property data isn't large enough.
- *
- * The out_values is modified only if a valid u32 value can be decoded.
- */
-int emulator_of_property_read_u32_array(
-	const struct device_node *np,
-	const char *propname,
-	u32 *out_values,
-	size_t sz)
-{
-	u16 i;
-	u32 index;
-	struct dtsi_replacement_u32_array *u32_arrayP;
-
-	/*
-	 * Get the index for the type of hardware we're running on.
-	 * This is used as a table index.
-	 */
-	index = emulator_type_to_index();
-	if (index >= ARRAY_SIZE(ipa3_plat_drv_u32_array_table)) {
-		IPADBG(
-		    "Did not find ipa3_plat_drv_u32_array_table for index %u\n",
-		    index);
-		return false;
-	}
-
-	u32_arrayP =
-		ipa3_plat_drv_u32_array_table[index].p_table;
-	for (i = 0;
-	     i < ipa3_plat_drv_u32_array_table[index].num_entries;
-	     i++) {
-		if (strsame(
-			u32_arrayP[i].key, propname)) {
-			u32 num_elements =
-			    u32_arrayP[i].num_elements;
-			u32 *p_element =
-			    &u32_arrayP[i].p_value[0];
-			size_t j = 0;
-
-			if (num_elements > sz) {
-				IPAERR(
-				    "Found array of %u values for propname %s; only room for %u elements in copy buffer\n",
-				    num_elements,
-				    propname,
-				    (unsigned int) sz);
-				return -EOVERFLOW;
-			}
-
-			while (j++ < num_elements)
-				*out_values++ = *p_element++;
-
-			IPADBG(
-			    "Found array of values starting with %u for propname %s index %u\n",
-			    u32_arrayP[i].p_value[0],
-			    propname,
-			    index);
-
-			return 0;
-		}
-	}
-
-	IPADBG("Did not find match for propname %s index %u\n",
-	       propname,
-	       index);
-
-	return -EINVAL;
-}
-
-/* From drivers/base/platform.c */
-/**
- * emulator_platform_get_resource_byname - get a resource for a device by name
- * @dev: platform device
- * @type: resource type
- * @name: resource name
- */
-struct resource *emulator_platform_get_resource_byname(
-	struct platform_device *dev,
-	unsigned int type,
-	const char *name)
-{
-	u16 i;
-	u32 index;
-	struct resource *ipa3_plat_drv_resourceP;
-
-	/*
-	 * Get the index for the type of hardware we're running on.
-	 * This is used as a table index.
-	 */
-	index = emulator_type_to_index();
-	if (index >= ARRAY_SIZE(ipa3_plat_drv_resource_table)) {
-		IPADBG(
-		    "Did not find ipa3_plat_drv_resource_table for index %u\n",
-		    index);
-		return false;
-	}
-
-	ipa3_plat_drv_resourceP =
-		ipa3_plat_drv_resource_table[index].p_table;
-	for (i = 0;
-	     i < ipa3_plat_drv_resource_table[index].num_entries;
-	     i++) {
-		struct resource *r = &ipa3_plat_drv_resourceP[i];
-
-		if (type == resource_type(r) && strsame(r->name, name)) {
-			IPADBG(
-			    "Found start 0x%x size %u for name %s index %u\n",
-			    (unsigned int) (r->start),
-			    (unsigned int) (resource_size(r)),
-			    name,
-			    index);
-			return r;
-		}
-	}
-
-	IPADBG("Did not find match for name %s index %u\n",
-	       name,
-	       index);
-
-	return NULL;
-}
-
-/* From drivers/of/base.c */
-/**
- * emulator_of_property_count_elems_of_size - Count the number of
- * elements in a property
- *
- * @np:         device node from which the property value is to
- *              be read. Not used.
- * @propname:   name of the property to be searched.
- * @elem_size:  size of the individual element
- *
- * Search for a property and count the number of elements of size
- * elem_size in it. Returns number of elements on success, -EINVAL if
- * the property does not exist or its length does not match a multiple
- * of elem_size and -ENODATA if the property does not have a value.
- */
-int emulator_of_property_count_elems_of_size(
-	const struct device_node *np,
-	const char *propname,
-	int elem_size)
-{
-	u32 index;
-
-	/*
-	 * Get the index for the type of hardware we're running on.
-	 * This is used as a table index.
-	 */
-	index = emulator_type_to_index();
-
-	/*
-	 * Use elem_size to determine which table to search for the
-	 * specified property name
-	 */
-	if (elem_size == sizeof(u32)) {
-		u16 i;
-		struct dtsi_replacement_u32_array *u32_arrayP;
-
-		if (index >= ARRAY_SIZE(ipa3_plat_drv_u32_array_table)) {
-			IPADBG(
-			    "Did not find ipa3_plat_drv_u32_array_table for index %u\n",
-			    index);
-			return false;
-		}
-
-		u32_arrayP =
-			ipa3_plat_drv_u32_array_table[index].p_table;
-
-		for (i = 0;
-		     i < ipa3_plat_drv_u32_array_table[index].num_entries;
-		     i++) {
-			if (strsame(u32_arrayP[i].key, propname)) {
-				if (u32_arrayP[i].p_value == NULL) {
-					IPADBG(
-					    "Found no elements for propname %s index %u\n",
-					    propname,
-					    index);
-					return -ENODATA;
-				}
-
-				IPADBG(
-				    "Found %u elements for propname %s index %u\n",
-				    u32_arrayP[i].num_elements,
-				    propname,
-				    index);
-
-				return u32_arrayP[i].num_elements;
-			}
-		}
-
-		IPADBG(
-		    "Found no match in table with elem_size %d for propname %s index %u\n",
-		    elem_size,
-		    propname,
-		    index);
-
-		return -EINVAL;
-	}
-
-	IPAERR(
-	    "Found no tables with element size %u to search for propname %s index %u\n",
-	    elem_size,
-	    propname,
-	    index);
-
-	return -EINVAL;
-}
-
-int emulator_of_property_read_variable_u32_array(
-	const struct device_node *np,
-	const char *propname,
-	u32 *out_values,
-	size_t sz_min,
-	size_t sz_max)
-{
-	return emulator_of_property_read_u32_array(
-	    np, propname, out_values, sz_max);
-}
-
-resource_size_t emulator_resource_size(const struct resource *res)
-{
-	return res->end - res->start + 1;
-}

ipa/ipa_v3/ipa_emulation_stubs.h

@@ -1,121 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2018, The Linux Foundation. All rights reserved.
- */
-
-#if !defined(_IPA_EMULATION_STUBS_H_)
-# define _IPA_EMULATION_STUBS_H_
-
-# define outer_flush_range(x, y)
-# define __flush_dcache_area(x, y)
-# define __cpuc_flush_dcache_area(x, y) __flush_dcache_area(x, y)
-
-/* Point several API calls to these new EMULATION functions */
-# define of_property_read_bool(np, propname)	 \
-	emulator_of_property_read_bool(NULL, propname)
-# define of_property_read_u32(np, propname, out_value)   \
-	emulator_of_property_read_u32(NULL, propname, out_value)
-# define of_property_read_u32_array(np, propname, out_values, sz)	\
-	emulator_of_property_read_u32_array(NULL, propname, out_values, sz)
-# define platform_get_resource_byname(dev, type, name) \
-	emulator_platform_get_resource_byname(NULL, type, name)
-# define of_property_count_elems_of_size(np, propname, elem_size) \
-	emulator_of_property_count_elems_of_size(NULL, propname, elem_size)
-# define of_property_read_variable_u32_array( \
-	np, propname, out_values, sz_min, sz_max) \
-	emulator_of_property_read_variable_u32_array( \
-		NULL, propname, out_values, sz_min, sz_max)
-# define resource_size(res) \
-	emulator_resource_size(res)
-
-/**
- * emulator_of_property_read_bool - Findfrom a property
- * @np:         device node used to find the property value. (not used)
- * @propname:   name of the property to be searched.
- *
- * Search for a property in a device node.
- * Returns true if the property exists false otherwise.
- */
-bool emulator_of_property_read_bool(
-	const struct device_node *np,
-	const char *propname);
-
-int emulator_of_property_read_u32(
-	const struct device_node *np,
-	const char *propname,
-	u32 *out_value);
-
-/**
- * emulator_of_property_read_u32_array - Find and read an array of 32
- * bit integers from a property.
- *
- * @np:         device node used to find the property value. (not used)
- * @propname:   name of the property to be searched.
- * @out_values: pointer to return value, modified only if return value is 0.
- * @sz:         number of array elements to read
- *
- * Search for a property in a device node and read 32-bit value(s) from
- * it. Returns 0 on success, -EINVAL if the property does not exist,
- * -ENODATA if property does not have a value, and -EOVERFLOW if the
- * property data isn't large enough.
- *
- * The out_values is modified only if a valid u32 value can be decoded.
- */
-int emulator_of_property_read_u32_array(
-	const struct device_node *np,
-	const char *propname,
-	u32 *out_values,
-	size_t sz);
-
-/**
- * emulator_platform_get_resource_byname - get a resource for a device
- * by name
- *
- * @dev: platform device
- * @type: resource type
- * @name: resource name
- */
-struct resource *emulator_platform_get_resource_byname(
-	struct platform_device *dev,
-	unsigned int type,
-	const char *name);
-
-/**
- * emulator_of_property_count_elems_of_size - Count the number of
- * elements in a property
- *
- * @np:         device node used to find the property value. (not used)
- * @propname:   name of the property to be searched.
- * @elem_size:  size of the individual element
- *
- * Search for a property and count the number of elements of size
- * elem_size in it. Returns number of elements on success, -EINVAL if
- * the property does not exist or its length does not match a multiple
- * of elem_size and -ENODATA if the property does not have a value.
- */
-int emulator_of_property_count_elems_of_size(
-	const struct device_node *np,
-	const char *propname,
-	int elem_size);
-
-int emulator_of_property_read_variable_u32_array(
-	const struct device_node *np,
-	const char *propname,
-	u32 *out_values,
-	size_t sz_min,
-	size_t sz_max);
-
-resource_size_t emulator_resource_size(
-	const struct resource *res);
-
-static inline bool is_device_dma_coherent(struct device *dev)
-{
-	return false;
-}
-
-static inline phys_addr_t qcom_smem_virt_to_phys(void *addr)
-{
-	return 0;
-}
-
-#endif /* #if !defined(_IPA_EMULATION_STUBS_H_) */

ipa/ipa_v3/ipa_flt.c

@@ -1,2110 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2012-2019, The Linux Foundation. All rights reserved.
- */
-
-#include "ipa_i.h"
-#include "ipahal/ipahal.h"
-#include "ipahal/ipahal_fltrt.h"
-
-#define IPA_FLT_STATUS_OF_ADD_FAILED		(-1)
-#define IPA_FLT_STATUS_OF_DEL_FAILED		(-1)
-#define IPA_FLT_STATUS_OF_MDFY_FAILED		(-1)
-
-#define IPA_FLT_GET_RULE_TYPE(__entry) \
-	( \
-	((__entry)->rule.hashable) ? \
-	(IPA_RULE_HASHABLE):(IPA_RULE_NON_HASHABLE) \
-	)
-
-/**
- * ipa3_generate_flt_hw_rule() - generates the filtering hardware rule
- * @ip: the ip address family type
- * @entry: filtering entry
- * @buf: output buffer, buf == NULL means
- *		caller wants to know the size of the rule as seen
- *		by HW so they did not pass a valid buffer, we will use a
- *		scratch buffer instead.
- *		With this scheme we are going to
- *		generate the rule twice, once to know size using scratch
- *		buffer and second to write the rule to the actual caller
- *		supplied buffer which is of required size
- *
- * Returns:	0 on success, negative on failure
- *
- * caller needs to hold any needed locks to ensure integrity
- *
- */
-static int ipa3_generate_flt_hw_rule(enum ipa_ip_type ip,
-		struct ipa3_flt_entry *entry, u8 *buf)
-{
-	struct ipahal_flt_rule_gen_params gen_params;
-	int res = 0;
-
-	memset(&gen_params, 0, sizeof(gen_params));
-
-	if (entry->rule.hashable) {
-		if (entry->rule.attrib.attrib_mask & IPA_FLT_IS_PURE_ACK
-			&& !entry->rule.eq_attrib_type) {
-			IPAERR_RL("PURE_ACK rule atrb used with hash rule\n");
-			WARN_ON_RATELIMIT_IPA(1);
-			return -EPERM;
-		}
-		/*
-		 * tos_eq_present field has two meanings:
-		 * tos equation for IPA ver < 4.5 (as the field name reveals)
-		 * pure_ack equation for IPA ver >= 4.5
-		 */
-		if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_5 &&
-			entry->rule.eq_attrib_type &&
-			entry->rule.eq_attrib.tos_eq_present) {
-			IPAERR_RL("PURE_ACK rule eq used with hash rule\n");
-			return -EPERM;
-		}
-	}
-
-	gen_params.ipt = ip;
-	if (entry->rt_tbl)
-		gen_params.rt_tbl_idx = entry->rt_tbl->idx;
-	else
-		gen_params.rt_tbl_idx = entry->rule.rt_tbl_idx;
-
-	gen_params.priority = entry->prio;
-	gen_params.id = entry->rule_id;
-	gen_params.rule = (const struct ipa_flt_rule_i *)&entry->rule;
-	gen_params.cnt_idx = entry->cnt_idx;
-
-	res = ipahal_flt_generate_hw_rule(&gen_params, &entry->hw_len, buf);
-	if (res) {
-		IPAERR_RL("failed to generate flt h/w rule\n");
-		return res;
-	}
-
-	return 0;
-}
-
-static void __ipa_reap_sys_flt_tbls(enum ipa_ip_type ip, enum ipa_rule_type rlt)
-{
-	struct ipa3_flt_tbl *tbl;
-	int i;
-
-	IPADBG_LOW("reaping sys flt tbls ip=%d rlt=%d\n", ip, rlt);
-
-	for (i = 0; i < ipa3_ctx->ipa_num_pipes; i++) {
-		if (!ipa_is_ep_support_flt(i))
-			continue;
-
-		tbl = &ipa3_ctx->flt_tbl[i][ip];
-		if (tbl->prev_mem[rlt].phys_base) {
-			IPADBG_LOW("reaping flt tbl (prev) pipe=%d\n", i);
-			ipahal_free_dma_mem(&tbl->prev_mem[rlt]);
-		}
-
-		if (list_empty(&tbl->head_flt_rule_list)) {
-			if (tbl->curr_mem[rlt].phys_base) {
-				IPADBG_LOW("reaping flt tbl (curr) pipe=%d\n",
-					i);
-				ipahal_free_dma_mem(&tbl->curr_mem[rlt]);
-			}
-		}
-	}
-}
-
-/**
- * ipa_prep_flt_tbl_for_cmt() - preparing the flt table for commit
- *  assign priorities to the rules, calculate their sizes and calculate
- *  the overall table size
- * @ip: the ip address family type
- * @tbl: the flt tbl to be prepared
- * @pipe_idx: the ep pipe appropriate for the given tbl
- *
- * Return: 0 on success, negative on failure
- */
-static int ipa_prep_flt_tbl_for_cmt(enum ipa_ip_type ip,
-	struct ipa3_flt_tbl *tbl, int pipe_idx)
-{
-	struct ipa3_flt_entry *entry;
-	int prio_i;
-	int max_prio;
-	u32 hdr_width;
-
-	tbl->sz[IPA_RULE_HASHABLE] = 0;
-	tbl->sz[IPA_RULE_NON_HASHABLE] = 0;
-
-	max_prio = ipahal_get_rule_max_priority();
-
-	prio_i = max_prio;
-	list_for_each_entry(entry, &tbl->head_flt_rule_list, link) {
-
-		if (entry->rule.max_prio) {
-			entry->prio = max_prio;
-		} else {
-			if (ipahal_rule_decrease_priority(&prio_i)) {
-				IPAERR("cannot decrease rule priority - %d\n",
-					prio_i);
-				return -EPERM;
-			}
-			entry->prio = prio_i;
-		}
-
-		if (ipa3_generate_flt_hw_rule(ip, entry, NULL)) {
-			IPAERR("failed to calculate HW FLT rule size\n");
-			return -EPERM;
-		}
-		IPADBG_LOW("pipe %d rule_id(handle) %u hw_len %d priority %u\n",
-			pipe_idx, entry->rule_id, entry->hw_len, entry->prio);
-
-		if (entry->rule.hashable)
-			tbl->sz[IPA_RULE_HASHABLE] += entry->hw_len;
-		else
-			tbl->sz[IPA_RULE_NON_HASHABLE] += entry->hw_len;
-	}
-
-	if ((tbl->sz[IPA_RULE_HASHABLE] +
-		tbl->sz[IPA_RULE_NON_HASHABLE]) == 0) {
-		IPADBG_LOW("flt tbl pipe %d is with zero total size\n",
-			pipe_idx);
-		return 0;
-	}
-
-	hdr_width = ipahal_get_hw_tbl_hdr_width();
-
-	/* for the header word */
-	if (tbl->sz[IPA_RULE_HASHABLE])
-		tbl->sz[IPA_RULE_HASHABLE] += hdr_width;
-	if (tbl->sz[IPA_RULE_NON_HASHABLE])
-		tbl->sz[IPA_RULE_NON_HASHABLE] += hdr_width;
-
-	IPADBG_LOW("FLT tbl pipe idx %d hash sz %u non-hash sz %u\n", pipe_idx,
-		tbl->sz[IPA_RULE_HASHABLE], tbl->sz[IPA_RULE_NON_HASHABLE]);
-
-	return 0;
-}
-
-/**
- * ipa_translate_flt_tbl_to_hw_fmt() - translate the flt driver structures
- *  (rules and tables) to HW format and fill it in the given buffers
- * @ip: the ip address family type
- * @rlt: the type of the rules to translate (hashable or non-hashable)
- * @base: the rules body buffer to be filled
- * @hdr: the rules header (addresses/offsets) buffer to be filled
- * @body_ofst: the offset of the rules body from the rules header at
- *  ipa sram
- *
- * Returns: 0 on success, negative on failure
- *
- * caller needs to hold any needed locks to ensure integrity
- *
- */
-static int ipa_translate_flt_tbl_to_hw_fmt(enum ipa_ip_type ip,
-	enum ipa_rule_type rlt, u8 *base, u8 *hdr, u32 body_ofst)
-{
-	u64 offset;
-	u8 *body_i;
-	int res;
-	struct ipa3_flt_entry *entry;
-	u8 *tbl_mem_buf;
-	struct ipa_mem_buffer tbl_mem;
-	struct ipa3_flt_tbl *tbl;
-	int i;
-	int hdr_idx = 0;
-
-	body_i = base;
-	for (i = 0; i < ipa3_ctx->ipa_num_pipes; i++) {
-		if (!ipa_is_ep_support_flt(i))
-			continue;
-		tbl = &ipa3_ctx->flt_tbl[i][ip];
-		if (tbl->sz[rlt] == 0) {
-			hdr_idx++;
-			continue;
-		}
-		if (tbl->in_sys[rlt]) {
-			/* only body (no header) */
-			tbl_mem.size = tbl->sz[rlt] -
-				ipahal_get_hw_tbl_hdr_width();
-			if (ipahal_fltrt_allocate_hw_sys_tbl(&tbl_mem)) {
-				IPAERR("fail to alloc sys tbl of size %d\n",
-					tbl_mem.size);
-				goto err;
-			}
-
-			if (ipahal_fltrt_write_addr_to_hdr(tbl_mem.phys_base,
-				hdr, hdr_idx, true)) {
-				IPAERR("fail to wrt sys tbl addr to hdr\n");
-				goto hdr_update_fail;
-			}
-
-			tbl_mem_buf = tbl_mem.base;
-
-			/* generate the rule-set */
-			list_for_each_entry(entry, &tbl->head_flt_rule_list,
-				link) {
-				if (IPA_FLT_GET_RULE_TYPE(entry) != rlt)
-					continue;
-				res = ipa3_generate_flt_hw_rule(
-					ip, entry, tbl_mem_buf);
-				if (res) {
-					IPAERR("failed to gen HW FLT rule\n");
-					goto hdr_update_fail;
-				}
-				tbl_mem_buf += entry->hw_len;
-			}
-
-			if (tbl->curr_mem[rlt].phys_base) {
-				WARN_ON(tbl->prev_mem[rlt].phys_base);
-				tbl->prev_mem[rlt] = tbl->curr_mem[rlt];
-			}
-			tbl->curr_mem[rlt] = tbl_mem;
-		} else {
-			offset = body_i - base + body_ofst;
-
-			/* update the hdr at the right index */
-			if (ipahal_fltrt_write_addr_to_hdr(offset, hdr,
-				hdr_idx, true)) {
-				IPAERR("fail to wrt lcl tbl ofst to hdr\n");
-				goto hdr_update_fail;
-			}
-
-			/* generate the rule-set */
-			list_for_each_entry(entry, &tbl->head_flt_rule_list,
-				link) {
-				if (IPA_FLT_GET_RULE_TYPE(entry) != rlt)
-					continue;
-				res = ipa3_generate_flt_hw_rule(
-					ip, entry, body_i);
-				if (res) {
-					IPAERR("failed to gen HW FLT rule\n");
-					goto err;
-				}
-				body_i += entry->hw_len;
-			}
-
-			/**
-			 * advance body_i to next table alignment as local
-			 * tables are order back-to-back
-			 */
-			body_i += ipahal_get_lcl_tbl_addr_alignment();
-			body_i = (u8 *)((long)body_i &
-				~ipahal_get_lcl_tbl_addr_alignment());
-		}
-		hdr_idx++;
-	}
-
-	return 0;
-
-hdr_update_fail:
-	ipahal_free_dma_mem(&tbl_mem);
-err:
-	return -EPERM;
-}
-
-/**
- * ipa_generate_flt_hw_tbl_img() - generates the flt hw tbls.
- *  headers and bodies are being created into buffers that will be filled into
- *  the local memory (sram)
- * @ip: the ip address family type
- * @alloc_params: In and Out parameters for the allocations of the buffers
- *  4 buffers: hdr and bdy, each hashable and non-hashable
- *
- * Return: 0 on success, negative on failure
- */
-static int ipa_generate_flt_hw_tbl_img(enum ipa_ip_type ip,
-	struct ipahal_fltrt_alloc_imgs_params *alloc_params)
-{
-	u32 hash_bdy_start_ofst, nhash_bdy_start_ofst;
-	int rc = 0;
-
-	if (ip == IPA_IP_v4) {
-		nhash_bdy_start_ofst = IPA_MEM_PART(apps_v4_flt_nhash_ofst) -
-			IPA_MEM_PART(v4_flt_nhash_ofst);
-		hash_bdy_start_ofst = IPA_MEM_PART(apps_v4_flt_hash_ofst) -
-			IPA_MEM_PART(v4_flt_hash_ofst);
-	} else {
-		nhash_bdy_start_ofst = IPA_MEM_PART(apps_v6_flt_nhash_ofst) -
-			IPA_MEM_PART(v6_flt_nhash_ofst);
-		hash_bdy_start_ofst = IPA_MEM_PART(apps_v6_flt_hash_ofst) -
-			IPA_MEM_PART(v6_flt_hash_ofst);
-	}
-
-	if (ipahal_fltrt_allocate_hw_tbl_imgs(alloc_params)) {
-		IPAERR_RL("fail to allocate FLT HW TBL images. IP %d\n", ip);
-		rc = -ENOMEM;
-		goto allocate_failed;
-	}
-
-	if (ipa_translate_flt_tbl_to_hw_fmt(ip, IPA_RULE_HASHABLE,
-		alloc_params->hash_bdy.base, alloc_params->hash_hdr.base,
-		hash_bdy_start_ofst)) {
-		IPAERR_RL("fail to translate hashable flt tbls to hw format\n");
-		rc = -EPERM;
-		goto translate_fail;
-	}
-	if (ipa_translate_flt_tbl_to_hw_fmt(ip, IPA_RULE_NON_HASHABLE,
-		alloc_params->nhash_bdy.base, alloc_params->nhash_hdr.base,
-		nhash_bdy_start_ofst)) {
-		IPAERR_RL("fail to translate non-hash flt tbls to hw format\n");
-		rc = -EPERM;
-		goto translate_fail;
-	}
-
-	return rc;
-
-translate_fail:
-	if (alloc_params->hash_hdr.size)
-		ipahal_free_dma_mem(&alloc_params->hash_hdr);
-	ipahal_free_dma_mem(&alloc_params->nhash_hdr);
-	if (alloc_params->hash_bdy.size)
-		ipahal_free_dma_mem(&alloc_params->hash_bdy);
-	if (alloc_params->nhash_bdy.size)
-		ipahal_free_dma_mem(&alloc_params->nhash_bdy);
-allocate_failed:
-	return rc;
-}
-
-/**
- * ipa_flt_valid_lcl_tbl_size() - validate if the space allocated for flt
- * tbl bodies at the sram is enough for the commit
- * @ipt: the ip address family type
- * @rlt: the rule type (hashable or non-hashable)
- *
- * Return: true if enough space available or false in other cases
- */
-static bool ipa_flt_valid_lcl_tbl_size(enum ipa_ip_type ipt,
-	enum ipa_rule_type rlt, struct ipa_mem_buffer *bdy)
-{
-	u16 avail;
-
-	if (!bdy) {
-		IPAERR("Bad parameters, bdy = NULL\n");
-		return false;
-	}
-
-	if (ipt == IPA_IP_v4)
-		avail = (rlt == IPA_RULE_HASHABLE) ?
-			IPA_MEM_PART(apps_v4_flt_hash_size) :
-			IPA_MEM_PART(apps_v4_flt_nhash_size);
-	else
-		avail = (rlt == IPA_RULE_HASHABLE) ?
-			IPA_MEM_PART(apps_v6_flt_hash_size) :
-			IPA_MEM_PART(apps_v6_flt_nhash_size);
-
-	if (bdy->size <= avail)
-		return true;
-
-	IPAERR("tbl too big, needed %d avail %d ipt %d rlt %d\n",
-	       bdy->size, avail, ipt, rlt);
-	return false;
-}
-
-/**
- * ipa_flt_alloc_cmd_buffers() - alloc descriptors and imm cmds
- *  payload pointers buffers for headers and bodies of flt structure
- *  as well as place for flush imm.
- * @ipt: the ip address family type
- * @entries: the number of entries
- * @desc: [OUT] descriptor buffer
- * @cmd: [OUT] imm commands payload pointers buffer
- *
- * Return: 0 on success, negative on failure
- */
-static int ipa_flt_alloc_cmd_buffers(enum ipa_ip_type ip, u16 entries,
-	struct ipa3_desc **desc, struct ipahal_imm_cmd_pyld ***cmd_pyld)
-{
-	*desc = kcalloc(entries, sizeof(**desc), GFP_ATOMIC);
-	if (*desc == NULL) {
-		IPAERR("fail to alloc desc blob ip %d\n", ip);
-		goto fail_desc_alloc;
-	}
-
-	*cmd_pyld = kcalloc(entries, sizeof(**cmd_pyld), GFP_ATOMIC);
-	if (*cmd_pyld == NULL) {
-		IPAERR("fail to alloc cmd pyld blob ip %d\n", ip);
-		goto fail_cmd_alloc;
-	}
-
-	return 0;
-
-fail_cmd_alloc:
-	kfree(*desc);
-fail_desc_alloc:
-	return -ENOMEM;
-}
-
-/**
- * ipa_flt_skip_pipe_config() - skip ep flt configuration or not?
- *  will skip according to pre-configuration or modem pipes
- * @pipe: the EP pipe index
- *
- * Return: true if to skip, false otherwize
- */
-static bool ipa_flt_skip_pipe_config(int pipe)
-{
-	struct ipa3_ep_context *ep;
-
-	if (ipa_is_modem_pipe(pipe)) {
-		IPADBG_LOW("skip %d - modem owned pipe\n", pipe);
-		return true;
-	}
-
-	if (ipa3_ctx->skip_ep_cfg_shadow[pipe]) {
-		IPADBG_LOW("skip %d\n", pipe);
-		return true;
-	}
-
-	ep = &ipa3_ctx->ep[pipe];
-
-	if ((ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_PROD) == pipe
-		&& ipa3_ctx->modem_cfg_emb_pipe_flt)
-		&& ep->client == IPA_CLIENT_APPS_WAN_PROD) {
-		IPADBG_LOW("skip %d\n", pipe);
-		return true;
-	}
-
-	return false;
-}
-
-/**
- * __ipa_commit_flt_v3() - commit flt tables to the hw
- *  commit the headers and the bodies if are local with internal cache flushing.
- *  The headers (and local bodies) will first be created into dma buffers and
- *  then written via IC to the SRAM
- * @ipt: the ip address family type
- *
- * Return: 0 on success, negative on failure
- */
-int __ipa_commit_flt_v3(enum ipa_ip_type ip)
-{
-	struct ipahal_fltrt_alloc_imgs_params alloc_params;
-	int rc = 0;
-	struct ipa3_desc *desc;
-	struct ipahal_imm_cmd_register_write reg_write_cmd = {0};
-	struct ipahal_imm_cmd_dma_shared_mem mem_cmd = {0};
-	struct ipahal_imm_cmd_pyld **cmd_pyld;
-	int num_cmd = 0;
-	int i;
-	int hdr_idx;
-	u32 lcl_hash_hdr, lcl_nhash_hdr;
-	u32 lcl_hash_bdy, lcl_nhash_bdy;
-	bool lcl_hash, lcl_nhash;
-	struct ipahal_reg_fltrt_hash_flush flush;
-	struct ipahal_reg_valmask valmask;
-	u32 tbl_hdr_width;
-	struct ipa3_flt_tbl *tbl;
-	u16 entries;
-
-	tbl_hdr_width = ipahal_get_hw_tbl_hdr_width();
-	memset(&alloc_params, 0, sizeof(alloc_params));
-	alloc_params.ipt = ip;
-	alloc_params.tbls_num = ipa3_ctx->ep_flt_num;
-
-	if (ip == IPA_IP_v4) {
-		lcl_hash_hdr = ipa3_ctx->smem_restricted_bytes +
-			IPA_MEM_PART(v4_flt_hash_ofst) +
-			tbl_hdr_width; /* to skip the bitmap */
-		lcl_nhash_hdr = ipa3_ctx->smem_restricted_bytes +
-			IPA_MEM_PART(v4_flt_nhash_ofst) +
-			tbl_hdr_width; /* to skip the bitmap */
-		lcl_hash_bdy = ipa3_ctx->smem_restricted_bytes +
-			IPA_MEM_PART(apps_v4_flt_hash_ofst);
-		lcl_nhash_bdy = ipa3_ctx->smem_restricted_bytes +
-			IPA_MEM_PART(apps_v4_flt_nhash_ofst);
-		lcl_hash = ipa3_ctx->ip4_flt_tbl_hash_lcl;
-		lcl_nhash = ipa3_ctx->ip4_flt_tbl_nhash_lcl;
-	} else {
-		lcl_hash_hdr = ipa3_ctx->smem_restricted_bytes +
-			IPA_MEM_PART(v6_flt_hash_ofst) +
-			tbl_hdr_width; /* to skip the bitmap */
-		lcl_nhash_hdr = ipa3_ctx->smem_restricted_bytes +
-			IPA_MEM_PART(v6_flt_nhash_ofst) +
-			tbl_hdr_width; /* to skip the bitmap */
-		lcl_hash_bdy = ipa3_ctx->smem_restricted_bytes +
-			IPA_MEM_PART(apps_v6_flt_hash_ofst);
-		lcl_nhash_bdy = ipa3_ctx->smem_restricted_bytes +
-			IPA_MEM_PART(apps_v6_flt_nhash_ofst);
-		lcl_hash = ipa3_ctx->ip6_flt_tbl_hash_lcl;
-		lcl_nhash = ipa3_ctx->ip6_flt_tbl_nhash_lcl;
-	}
-
-	for (i = 0; i < ipa3_ctx->ipa_num_pipes; i++) {
-		if (!ipa_is_ep_support_flt(i))
-			continue;
-		tbl = &ipa3_ctx->flt_tbl[i][ip];
-		if (ipa_prep_flt_tbl_for_cmt(ip, tbl, i)) {
-			rc = -EPERM;
-			goto prep_failed;
-		}
-		if (!tbl->in_sys[IPA_RULE_HASHABLE] &&
-			tbl->sz[IPA_RULE_HASHABLE]) {
-			alloc_params.num_lcl_hash_tbls++;
-			alloc_params.total_sz_lcl_hash_tbls +=
-				tbl->sz[IPA_RULE_HASHABLE];
-			alloc_params.total_sz_lcl_hash_tbls -= tbl_hdr_width;
-
-		}
-		if (!tbl->in_sys[IPA_RULE_NON_HASHABLE] &&
-			tbl->sz[IPA_RULE_NON_HASHABLE]) {
-			alloc_params.num_lcl_nhash_tbls++;
-			alloc_params.total_sz_lcl_nhash_tbls +=
-				tbl->sz[IPA_RULE_NON_HASHABLE];
-			alloc_params.total_sz_lcl_nhash_tbls -= tbl_hdr_width;
-		}
-	}
-
-	if (ipa_generate_flt_hw_tbl_img(ip, &alloc_params)) {
-		IPAERR_RL("fail to generate FLT HW TBL image. IP %d\n", ip);
-		rc = -EFAULT;
-		goto prep_failed;
-	}
-
-	if (!ipa_flt_valid_lcl_tbl_size(ip, IPA_RULE_HASHABLE,
-		&alloc_params.hash_bdy)) {
-		rc = -EFAULT;
-		goto fail_size_valid;
-	}
-	if (!ipa_flt_valid_lcl_tbl_size(ip, IPA_RULE_NON_HASHABLE,
-		&alloc_params.nhash_bdy)) {
-		rc = -EFAULT;
-		goto fail_size_valid;
-	}
-
-	/* +3: 2 for bodies (hashable and non-hashable) and 1 for flushing */
-	entries = (ipa3_ctx->ep_flt_num) * 2 + 3;
-
-	if (ipa_flt_alloc_cmd_buffers(ip, entries, &desc, &cmd_pyld)) {
-		rc = -ENOMEM;
-		goto fail_size_valid;
-	}
-
-	/*
-	 * SRAM memory not allocated to hash tables. Sending
-	 * command to hash tables(filer/routing) operation not supported.
-	 */
-	if (!ipa3_ctx->ipa_fltrt_not_hashable) {
-		/* flushing ipa internal hashable flt rules cache */
-		memset(&flush, 0, sizeof(flush));
-		if (ip == IPA_IP_v4)
-			flush.v4_flt = true;
-		else
-			flush.v6_flt = true;
-		ipahal_get_fltrt_hash_flush_valmask(&flush, &valmask);
-		reg_write_cmd.skip_pipeline_clear = false;
-		reg_write_cmd.pipeline_clear_options = IPAHAL_HPS_CLEAR;
-		reg_write_cmd.offset = ipahal_get_reg_ofst(
-					IPA_FILT_ROUT_HASH_FLUSH);
-		reg_write_cmd.value = valmask.val;
-		reg_write_cmd.value_mask = valmask.mask;
-		cmd_pyld[0] = ipahal_construct_imm_cmd(
-				IPA_IMM_CMD_REGISTER_WRITE, &reg_write_cmd,
-							false);
-		if (!cmd_pyld[0]) {
-			IPAERR(
-			"fail construct register_write imm cmd: IP %d\n", ip);
-			rc = -EFAULT;
-			goto fail_reg_write_construct;
-		}
-		ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]);
-		++num_cmd;
-	}
-
-	hdr_idx = 0;
-	for (i = 0; i < ipa3_ctx->ipa_num_pipes; i++) {
-		if (!ipa_is_ep_support_flt(i)) {
-			IPADBG_LOW("skip %d - not filtering pipe\n", i);
-			continue;
-		}
-
-		if (ipa_flt_skip_pipe_config(i)) {
-			hdr_idx++;
-			continue;
-		}
-
-		if (num_cmd + 1 >= entries) {
-			IPAERR("number of commands is out of range: IP = %d\n",
-				ip);
-			rc = -ENOBUFS;
-			goto fail_imm_cmd_construct;
-		}
-
-		IPADBG_LOW("Prepare imm cmd for hdr at index %d for pipe %d\n",
-			hdr_idx, i);
-
-		mem_cmd.is_read = false;
-		mem_cmd.skip_pipeline_clear = false;
-		mem_cmd.pipeline_clear_options = IPAHAL_HPS_CLEAR;
-		mem_cmd.size = tbl_hdr_width;
-		mem_cmd.system_addr = alloc_params.nhash_hdr.phys_base +
-			hdr_idx * tbl_hdr_width;
-		mem_cmd.local_addr = lcl_nhash_hdr +
-			hdr_idx * tbl_hdr_width;
-		cmd_pyld[num_cmd] = ipahal_construct_imm_cmd(
-			IPA_IMM_CMD_DMA_SHARED_MEM, &mem_cmd, false);
-		if (!cmd_pyld[num_cmd]) {
-			IPAERR("fail construct dma_shared_mem cmd: IP = %d\n",
-				ip);
-			rc = -ENOMEM;
-			goto fail_imm_cmd_construct;
-		}
-		ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]);
-		++num_cmd;
-
-		/*
-		 * SRAM memory not allocated to hash tables. Sending command
-		 * to hash tables(filer/routing) operation not supported.
-		 */
-		if (!ipa3_ctx->ipa_fltrt_not_hashable) {
-			mem_cmd.is_read = false;
-			mem_cmd.skip_pipeline_clear = false;
-			mem_cmd.pipeline_clear_options = IPAHAL_HPS_CLEAR;
-			mem_cmd.size = tbl_hdr_width;
-			mem_cmd.system_addr = alloc_params.hash_hdr.phys_base +
-				hdr_idx * tbl_hdr_width;
-			mem_cmd.local_addr = lcl_hash_hdr +
-				hdr_idx * tbl_hdr_width;
-			cmd_pyld[num_cmd] = ipahal_construct_imm_cmd(
-					IPA_IMM_CMD_DMA_SHARED_MEM,
-						&mem_cmd, false);
-			if (!cmd_pyld[num_cmd]) {
-				IPAERR(
-				"fail construct dma_shared_mem cmd: IP = %d\n",
-						ip);
-				rc = -ENOMEM;
-				goto fail_imm_cmd_construct;
-			}
-			ipa3_init_imm_cmd_desc(&desc[num_cmd],
-						cmd_pyld[num_cmd]);
-			++num_cmd;
-		}
-		++hdr_idx;
-	}
-
-	if (lcl_nhash) {
-		if (num_cmd >= entries) {
-			IPAERR("number of commands is out of range: IP = %d\n",
-				ip);
-			rc = -ENOBUFS;
-			goto fail_imm_cmd_construct;
-		}
-
-		mem_cmd.is_read = false;
-		mem_cmd.skip_pipeline_clear = false;
-		mem_cmd.pipeline_clear_options = IPAHAL_HPS_CLEAR;
-		mem_cmd.size = alloc_params.nhash_bdy.size;
-		mem_cmd.system_addr = alloc_params.nhash_bdy.phys_base;
-		mem_cmd.local_addr = lcl_nhash_bdy;
-		cmd_pyld[num_cmd] = ipahal_construct_imm_cmd(
-			IPA_IMM_CMD_DMA_SHARED_MEM, &mem_cmd, false);
-		if (!cmd_pyld[num_cmd]) {
-			IPAERR("fail construct dma_shared_mem cmd: IP = %d\n",
-				ip);
-			rc = -ENOMEM;
-			goto fail_imm_cmd_construct;
-		}
-		ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]);
-		++num_cmd;
-	}
-	if (lcl_hash) {
-		if (num_cmd >= entries) {
-			IPAERR("number of commands is out of range: IP = %d\n",
-				ip);
-			rc = -ENOBUFS;
-			goto fail_imm_cmd_construct;
-		}
-
-		mem_cmd.is_read = false;
-		mem_cmd.skip_pipeline_clear = false;
-		mem_cmd.pipeline_clear_options = IPAHAL_HPS_CLEAR;
-		mem_cmd.size = alloc_params.hash_bdy.size;
-		mem_cmd.system_addr = alloc_params.hash_bdy.phys_base;
-		mem_cmd.local_addr = lcl_hash_bdy;
-		cmd_pyld[num_cmd] = ipahal_construct_imm_cmd(
-			IPA_IMM_CMD_DMA_SHARED_MEM, &mem_cmd, false);
-		if (!cmd_pyld[num_cmd]) {
-			IPAERR("fail construct dma_shared_mem cmd: IP = %d\n",
-				ip);
-			rc = -ENOMEM;
-			goto fail_imm_cmd_construct;
-		}
-		ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]);
-		++num_cmd;
-	}
-
-	if (ipa3_send_cmd(num_cmd, desc)) {
-		IPAERR("fail to send immediate command\n");
-		rc = -EFAULT;
-		goto fail_imm_cmd_construct;
-	}
-
-	IPADBG_LOW("Hashable HEAD\n");
-	IPA_DUMP_BUFF(alloc_params.hash_hdr.base,
-		alloc_params.hash_hdr.phys_base, alloc_params.hash_hdr.size);
-
-	IPADBG_LOW("Non-Hashable HEAD\n");
-	IPA_DUMP_BUFF(alloc_params.nhash_hdr.base,
-		alloc_params.nhash_hdr.phys_base, alloc_params.nhash_hdr.size);
-
-	if (alloc_params.hash_bdy.size) {
-		IPADBG_LOW("Hashable BODY\n");
-		IPA_DUMP_BUFF(alloc_params.hash_bdy.base,
-			alloc_params.hash_bdy.phys_base,
-			alloc_params.hash_bdy.size);
-	}
-
-	if (alloc_params.nhash_bdy.size) {
-		IPADBG_LOW("Non-Hashable BODY\n");
-		IPA_DUMP_BUFF(alloc_params.nhash_bdy.base,
-			alloc_params.nhash_bdy.phys_base,
-			alloc_params.nhash_bdy.size);
-	}
-
-	__ipa_reap_sys_flt_tbls(ip, IPA_RULE_HASHABLE);
-	__ipa_reap_sys_flt_tbls(ip, IPA_RULE_NON_HASHABLE);
-
-fail_imm_cmd_construct:
-	for (i = 0 ; i < num_cmd ; i++)
-		ipahal_destroy_imm_cmd(cmd_pyld[i]);
-fail_reg_write_construct:
-	kfree(desc);
-	kfree(cmd_pyld);
-fail_size_valid:
-	if (alloc_params.hash_hdr.size)
-		ipahal_free_dma_mem(&alloc_params.hash_hdr);
-	ipahal_free_dma_mem(&alloc_params.nhash_hdr);
-	if (alloc_params.hash_bdy.size)
-		ipahal_free_dma_mem(&alloc_params.hash_bdy);
-	if (alloc_params.nhash_bdy.size)
-		ipahal_free_dma_mem(&alloc_params.nhash_bdy);
-prep_failed:
-	return rc;
-}
-
-static int __ipa_validate_flt_rule(const struct ipa_flt_rule_i *rule,
-		struct ipa3_rt_tbl **rt_tbl, enum ipa_ip_type ip)
-{
-	int index;
-
-	if (rule->action != IPA_PASS_TO_EXCEPTION) {
-		if (!rule->eq_attrib_type) {
-			if (!rule->rt_tbl_hdl) {
-				IPAERR_RL("invalid RT tbl\n");
-				goto error;
-			}
-
-			*rt_tbl = ipa3_id_find(rule->rt_tbl_hdl);
-			if (*rt_tbl == NULL) {
-				IPAERR_RL("RT tbl not found\n");
-				goto error;
-			}
-
-			if ((*rt_tbl)->cookie != IPA_RT_TBL_COOKIE) {
-				IPAERR_RL("RT table cookie is invalid\n");
-				goto error;
-			}
-		} else {
-			if (rule->rt_tbl_idx > ((ip == IPA_IP_v4) ?
-				IPA_MEM_PART(v4_modem_rt_index_hi) :
-				IPA_MEM_PART(v6_modem_rt_index_hi))) {
-				IPAERR_RL("invalid RT tbl\n");
-				goto error;
-			}
-		}
-	} else {
-		if (rule->rt_tbl_idx > 0) {
-			IPAERR_RL("invalid RT tbl\n");
-			goto error;
-		}
-	}
-
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0) {
-		if (rule->pdn_idx) {
-			if (rule->action == IPA_PASS_TO_EXCEPTION ||
-				rule->action == IPA_PASS_TO_ROUTING) {
-				IPAERR_RL(
-					"PDN index should be 0 when action is not pass to NAT\n");
-				goto error;
-			} else {
-				if (rule->pdn_idx >= IPA_MAX_PDN_NUM) {
-					IPAERR_RL("PDN index %d is too large\n",
-						rule->pdn_idx);
-					goto error;
-				}
-			}
-		}
-	}
-
-	if (rule->rule_id) {
-		if ((rule->rule_id < ipahal_get_rule_id_hi_bit()) ||
-		(rule->rule_id >= ((ipahal_get_rule_id_hi_bit()<<1)-1))) {
-			IPAERR_RL("invalid rule_id provided 0x%x\n"
-				"rule_id with bit 0x%x are auto generated\n",
-				rule->rule_id, ipahal_get_rule_id_hi_bit());
-			goto error;
-		}
-	}
-
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_5) {
-		if (rule->enable_stats && rule->cnt_idx) {
-			if (!ipahal_is_rule_cnt_id_valid(rule->cnt_idx)) {
-				IPAERR_RL(
-					"invalid cnt_idx %hhu out of range\n",
-					rule->cnt_idx);
-				goto error;
-			}
-			index = rule->cnt_idx - 1;
-			if (!ipa3_ctx->flt_rt_counters.used_hw[index]) {
-				IPAERR_RL(
-					"invalid cnt_idx %hhu not alloc by driver\n",
-					rule->cnt_idx);
-				goto error;
-			}
-		}
-	} else {
-		if (rule->enable_stats) {
-			IPAERR_RL(
-				"enable_stats won't support on ipa_hw_type %d\n",
-				ipa3_ctx->ipa_hw_type);
-			goto error;
-		}
-	}
-	return 0;
-
-error:
-	return -EPERM;
-}
-
-static int __ipa_create_flt_entry(struct ipa3_flt_entry **entry,
-		const struct ipa_flt_rule_i *rule, struct ipa3_rt_tbl *rt_tbl,
-		struct ipa3_flt_tbl *tbl, bool user)
-{
-	int id;
-
-	*entry = kmem_cache_zalloc(ipa3_ctx->flt_rule_cache, GFP_KERNEL);
-	if (!*entry)
-		goto error;
-	INIT_LIST_HEAD(&((*entry)->link));
-	(*entry)->rule = *rule;
-	(*entry)->cookie = IPA_FLT_COOKIE;
-	(*entry)->rt_tbl = rt_tbl;
-	(*entry)->tbl = tbl;
-	if (rule->rule_id) {
-		id = rule->rule_id;
-	} else {
-		id = ipa3_alloc_rule_id(tbl->rule_ids);
-		if (id < 0) {
-			IPAERR_RL("failed to allocate rule id\n");
-			WARN_ON_RATELIMIT_IPA(1);
-			goto rule_id_fail;
-		}
-	}
-	(*entry)->rule_id = id;
-	(*entry)->ipacm_installed = user;
-	if (rule->enable_stats)
-		(*entry)->cnt_idx = rule->cnt_idx;
-	else
-		(*entry)->cnt_idx = 0;
-	return 0;
-
-rule_id_fail:
-	kmem_cache_free(ipa3_ctx->flt_rule_cache, *entry);
-error:
-	return -EPERM;
-}
-
-static int __ipa_finish_flt_rule_add(struct ipa3_flt_tbl *tbl,
-		struct ipa3_flt_entry *entry, u32 *rule_hdl)
-{
-	int id;
-
-	tbl->rule_cnt++;
-	if (entry->rt_tbl)
-		entry->rt_tbl->ref_cnt++;
-	id = ipa3_id_alloc(entry);
-	if (id < 0) {
-		IPAERR_RL("failed to add to tree\n");
-		WARN_ON_RATELIMIT_IPA(1);
-		goto ipa_insert_failed;
-	}
-	*rule_hdl = id;
-	entry->id = id;
-	IPADBG_LOW("add flt rule rule_cnt=%d\n", tbl->rule_cnt);
-
-	return 0;
-ipa_insert_failed:
-	if (entry->rt_tbl)
-		entry->rt_tbl->ref_cnt--;
-	tbl->rule_cnt--;
-	return -EPERM;
-}
-
-static int __ipa_add_flt_rule(struct ipa3_flt_tbl *tbl, enum ipa_ip_type ip,
-			      const struct ipa_flt_rule_i *rule, u8 add_rear,
-			      u32 *rule_hdl, bool user)
-{
-	struct ipa3_flt_entry *entry;
-	struct ipa3_rt_tbl *rt_tbl = NULL;
-
-	if (__ipa_validate_flt_rule(rule, &rt_tbl, ip))
-		goto error;
-
-	if (__ipa_create_flt_entry(&entry, rule, rt_tbl, tbl, user))
-		goto error;
-
-	if (add_rear) {
-		if (tbl->sticky_rear)
-			list_add_tail(&entry->link,
-					tbl->head_flt_rule_list.prev);
-		else
-			list_add_tail(&entry->link, &tbl->head_flt_rule_list);
-	} else {
-		list_add(&entry->link, &tbl->head_flt_rule_list);
-	}
-
-	if (__ipa_finish_flt_rule_add(tbl, entry, rule_hdl))
-		goto ipa_insert_failed;
-
-	return 0;
-ipa_insert_failed:
-	list_del(&entry->link);
-	/* if rule id was allocated from idr, remove it */
-	if ((entry->rule_id < ipahal_get_rule_id_hi_bit()) &&
-		(entry->rule_id >= ipahal_get_low_rule_id()))
-		idr_remove(entry->tbl->rule_ids, entry->rule_id);
-	kmem_cache_free(ipa3_ctx->flt_rule_cache, entry);
-
-error:
-	return -EPERM;
-}
-
-static int __ipa_add_flt_rule_after(struct ipa3_flt_tbl *tbl,
-				const struct ipa_flt_rule_i *rule,
-				u32 *rule_hdl,
-				enum ipa_ip_type ip,
-				struct ipa3_flt_entry **add_after_entry)
-{
-	struct ipa3_flt_entry *entry;
-	struct ipa3_rt_tbl *rt_tbl = NULL;
-
-	if (!*add_after_entry)
-		goto error;
-
-	if (rule == NULL || rule_hdl == NULL) {
-		IPAERR_RL("bad parms rule=%pK rule_hdl=%pK\n", rule,
-				rule_hdl);
-		goto error;
-	}
-
-	if (__ipa_validate_flt_rule(rule, &rt_tbl, ip))
-		goto error;
-
-	if (__ipa_create_flt_entry(&entry, rule, rt_tbl, tbl, true))
-		goto error;
-
-	list_add(&entry->link, &((*add_after_entry)->link));
-
-	if (__ipa_finish_flt_rule_add(tbl, entry, rule_hdl))
-		goto ipa_insert_failed;
-
-	/*
-	 * prepare for next insertion
-	 */
-	*add_after_entry = entry;
-
-	return 0;
-
-ipa_insert_failed:
-	list_del(&entry->link);
-	/* if rule id was allocated from idr, remove it */
-	if ((entry->rule_id < ipahal_get_rule_id_hi_bit()) &&
-		(entry->rule_id >= ipahal_get_low_rule_id()))
-		idr_remove(entry->tbl->rule_ids, entry->rule_id);
-	kmem_cache_free(ipa3_ctx->flt_rule_cache, entry);
-
-error:
-	*add_after_entry = NULL;
-	return -EPERM;
-}
-
-static int __ipa_del_flt_rule(u32 rule_hdl)
-{
-	struct ipa3_flt_entry *entry;
-	int id;
-
-	entry = ipa3_id_find(rule_hdl);
-	if (entry == NULL) {
-		IPAERR_RL("lookup failed\n");
-		return -EINVAL;
-	}
-
-	if (entry->cookie != IPA_FLT_COOKIE) {
-		IPAERR_RL("bad params\n");
-		return -EINVAL;
-	}
-	id = entry->id;
-
-	list_del(&entry->link);
-	entry->tbl->rule_cnt--;
-	if (entry->rt_tbl)
-		entry->rt_tbl->ref_cnt--;
-	IPADBG("del flt rule rule_cnt=%d rule_id=%d\n",
-		entry->tbl->rule_cnt, entry->rule_id);
-	entry->cookie = 0;
-	/* if rule id was allocated from idr, remove it */
-	if ((entry->rule_id < ipahal_get_rule_id_hi_bit()) &&
-		(entry->rule_id >= ipahal_get_low_rule_id()))
-		idr_remove(entry->tbl->rule_ids, entry->rule_id);
-
-	kmem_cache_free(ipa3_ctx->flt_rule_cache, entry);
-
-	/* remove the handle from the database */
-	ipa3_id_remove(id);
-
-	return 0;
-}
-
-static int __ipa_mdfy_flt_rule(struct ipa_flt_rule_mdfy_i *frule,
-		enum ipa_ip_type ip)
-{
-	struct ipa3_flt_entry *entry;
-	struct ipa3_rt_tbl *rt_tbl = NULL;
-
-	entry = ipa3_id_find(frule->rule_hdl);
-	if (entry == NULL) {
-		IPAERR_RL("lookup failed\n");
-		goto error;
-	}
-
-	if (entry->cookie != IPA_FLT_COOKIE) {
-		IPAERR_RL("bad params\n");
-		goto error;
-	}
-
-	if (__ipa_validate_flt_rule(&frule->rule, &rt_tbl, ip))
-		goto error;
-
-	if (entry->rt_tbl)
-		entry->rt_tbl->ref_cnt--;
-
-	entry->rule = frule->rule;
-	entry->rt_tbl = rt_tbl;
-	if (entry->rt_tbl)
-		entry->rt_tbl->ref_cnt++;
-	entry->hw_len = 0;
-	entry->prio = 0;
-	if (frule->rule.enable_stats)
-		entry->cnt_idx = frule->rule.cnt_idx;
-	else
-		entry->cnt_idx = 0;
-
-	return 0;
-
-error:
-	return -EPERM;
-}
-
-static int __ipa_add_flt_get_ep_idx(enum ipa_client_type ep, int *ipa_ep_idx)
-{
-	*ipa_ep_idx = ipa3_get_ep_mapping(ep);
-	if (*ipa_ep_idx < 0) {
-		IPAERR_RL("ep not valid ep=%d\n", ep);
-		return -EINVAL;
-	}
-	if (ipa3_ctx->ep[*ipa_ep_idx].valid == 0)
-		IPADBG("ep not connected ep_idx=%d\n", *ipa_ep_idx);
-
-	if (!ipa_is_ep_support_flt(*ipa_ep_idx)) {
-		IPAERR("ep do not support filtering ep=%d\n", ep);
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int __ipa_add_ep_flt_rule(enum ipa_ip_type ip, enum ipa_client_type ep,
-				 const struct ipa_flt_rule_i *rule, u8 add_rear,
-				 u32 *rule_hdl, bool user)
-{
-	struct ipa3_flt_tbl *tbl;
-	int ipa_ep_idx;
-
-	if (rule == NULL || rule_hdl == NULL || ep >= IPA_CLIENT_MAX) {
-		IPAERR_RL("bad parms rule=%pK rule_hdl=%pK ep=%d\n", rule,
-				rule_hdl, ep);
-		return -EINVAL;
-	}
-
-	if (__ipa_add_flt_get_ep_idx(ep, &ipa_ep_idx))
-		return -EINVAL;
-
-	if (ipa_ep_idx >= IPA3_MAX_NUM_PIPES) {
-		IPAERR_RL("invalid ipa_ep_idx=%d\n", ipa_ep_idx);
-		return -EINVAL;
-	}
-
-	tbl = &ipa3_ctx->flt_tbl[ipa_ep_idx][ip];
-	IPADBG_LOW("add ep flt rule ip=%d ep=%d\n", ip, ep);
-
-	return __ipa_add_flt_rule(tbl, ip, rule, add_rear, rule_hdl, user);
-}
-
-static void __ipa_convert_flt_rule_in(struct ipa_flt_rule rule_in,
-	struct ipa_flt_rule_i *rule_out)
-{
-	if (unlikely(sizeof(struct ipa_flt_rule) >
-			sizeof(struct ipa_flt_rule_i))) {
-		IPAERR_RL("invalid size in:%d size out:%d\n",
-			sizeof(struct ipa_flt_rule_i),
-			sizeof(struct ipa_flt_rule));
-		return;
-	}
-	memset(rule_out, 0, sizeof(struct ipa_flt_rule_i));
-	memcpy(rule_out, &rule_in, sizeof(struct ipa_flt_rule));
-}
-
-static void __ipa_convert_flt_rule_out(struct ipa_flt_rule_i rule_in,
-	struct ipa_flt_rule *rule_out)
-{
-	if (unlikely(sizeof(struct ipa_flt_rule) >
-			sizeof(struct ipa_flt_rule_i))) {
-		IPAERR_RL("invalid size in:%d size out:%d\n",
-			sizeof(struct ipa_flt_rule_i),
-			sizeof(struct ipa_flt_rule));
-		return;
-	}
-	memset(rule_out, 0, sizeof(struct ipa_flt_rule));
-	memcpy(rule_out, &rule_in, sizeof(struct ipa_flt_rule));
-}
-
-static void __ipa_convert_flt_mdfy_in(struct ipa_flt_rule_mdfy rule_in,
-	struct ipa_flt_rule_mdfy_i *rule_out)
-{
-	if (unlikely(sizeof(struct ipa_flt_rule_mdfy) >
-			sizeof(struct ipa_flt_rule_mdfy_i))) {
-		IPAERR_RL("invalid size in:%d size out:%d\n",
-			sizeof(struct ipa_flt_rule_mdfy),
-			sizeof(struct ipa_flt_rule_mdfy_i));
-		return;
-	}
-	memset(rule_out, 0, sizeof(struct ipa_flt_rule_mdfy_i));
-	memcpy(&rule_out->rule, &rule_in.rule,
-		sizeof(struct ipa_flt_rule));
-	rule_out->rule_hdl = rule_in.rule_hdl;
-	rule_out->status = rule_in.status;
-}
-
-static void __ipa_convert_flt_mdfy_out(struct ipa_flt_rule_mdfy_i rule_in,
-	struct ipa_flt_rule_mdfy *rule_out)
-{
-	if (unlikely(sizeof(struct ipa_flt_rule_mdfy) >
-			sizeof(struct ipa_flt_rule_mdfy_i))) {
-		IPAERR_RL("invalid size in:%d size out:%d\n",
-			sizeof(struct ipa_flt_rule_mdfy),
-			sizeof(struct ipa_flt_rule_mdfy_i));
-		return;
-	}
-	memset(rule_out, 0, sizeof(struct ipa_flt_rule_mdfy));
-	memcpy(&rule_out->rule, &rule_in.rule,
-		sizeof(struct ipa_flt_rule));
-	rule_out->rule_hdl = rule_in.rule_hdl;
-	rule_out->status = rule_in.status;
-}
-
-/**
- * ipa3_add_flt_rule() - Add the specified filtering rules to SW and optionally
- * commit to IPA HW
- * @rules:	[inout] set of filtering rules to add
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_add_flt_rule(struct ipa_ioc_add_flt_rule *rules)
-{
-	return ipa3_add_flt_rule_usr(rules, false);
-}
-
-/**
- * ipa3_add_flt_rule_v2() - Add the specified filtering rules to
- * SW and optionally commit to IPA HW
- * @rules:	[inout] set of filtering rules to add
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_add_flt_rule_v2(struct ipa_ioc_add_flt_rule_v2 *rules)
-{
-	return ipa3_add_flt_rule_usr_v2(rules, false);
-}
-
-
-/**
- * ipa3_add_flt_rule_usr() - Add the specified filtering rules to
- * SW and optionally commit to IPA HW
- * @rules:	[inout] set of filtering rules to add
- * @user_only:	[in] indicate rules installed by userspace
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_add_flt_rule_usr(struct ipa_ioc_add_flt_rule *rules, bool user_only)
-{
-	int i;
-	int result;
-	struct ipa_flt_rule_i rule;
-
-	if (rules == NULL || rules->num_rules == 0 ||
-			rules->ip >= IPA_IP_MAX) {
-		IPAERR_RL("bad parm\n");
-		return -EINVAL;
-	}
-
-	mutex_lock(&ipa3_ctx->lock);
-	for (i = 0; i < rules->num_rules; i++) {
-		if (!rules->global) {
-			/* if hashing not supported, all table entry
-			 * are non-hash tables
-			 */
-			if (ipa3_ctx->ipa_fltrt_not_hashable)
-				rules->rules[i].rule.hashable = false;
-			__ipa_convert_flt_rule_in(
-				rules->rules[i].rule, &rule);
-			result = __ipa_add_ep_flt_rule(rules->ip,
-					rules->ep,
-					&rule,
-					rules->rules[i].at_rear,
-					&rules->rules[i].flt_rule_hdl,
-					user_only);
-			__ipa_convert_flt_rule_out(rule,
-				&rules->rules[i].rule);
-		} else
-			result = -1;
-		if (result) {
-			IPAERR_RL("failed to add flt rule %d\n", i);
-			rules->rules[i].status = IPA_FLT_STATUS_OF_ADD_FAILED;
-		} else {
-			rules->rules[i].status = 0;
-		}
-	}
-
-	if (rules->global) {
-		IPAERR_RL("no support for global filter rules\n");
-		result = -EPERM;
-		goto bail;
-	}
-
-	if (rules->commit)
-		if (ipa3_ctx->ctrl->ipa3_commit_flt(rules->ip)) {
-			result = -EPERM;
-			goto bail;
-		}
-	result = 0;
-bail:
-	mutex_unlock(&ipa3_ctx->lock);
-
-	return result;
-}
-
-/**
- * ipa3_add_flt_rule_usr_v2() - Add the specified filtering
- * rules to SW and optionally commit to IPA HW
- * @rules:	[inout] set of filtering rules to add
- * @user_only:	[in] indicate rules installed by userspace
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_add_flt_rule_usr_v2(struct ipa_ioc_add_flt_rule_v2
-	*rules, bool user_only)
-{
-	int i;
-	int result;
-
-	if (rules == NULL || rules->num_rules == 0 ||
-			rules->ip >= IPA_IP_MAX) {
-		IPAERR_RL("bad parm\n");
-		return -EINVAL;
-	}
-
-	mutex_lock(&ipa3_ctx->lock);
-	for (i = 0; i < rules->num_rules; i++) {
-		if (!rules->global) {
-			/* if hashing not supported, all table entry
-			 * are non-hash tables
-			 */
-			if (ipa3_ctx->ipa_fltrt_not_hashable)
-				((struct ipa_flt_rule_add_i *)
-				rules->rules)[i].rule.hashable = false;
-			result = __ipa_add_ep_flt_rule(rules->ip,
-					rules->ep,
-					&(((struct ipa_flt_rule_add_i *)
-					rules->rules)[i].rule),
-					((struct ipa_flt_rule_add_i *)
-					rules->rules)[i].at_rear,
-					&(((struct ipa_flt_rule_add_i *)
-					rules->rules)[i].flt_rule_hdl),
-					user_only);
-		} else
-			result = -1;
-
-		if (result) {
-			IPAERR_RL("failed to add flt rule %d\n", i);
-			((struct ipa_flt_rule_add_i *)
-			rules->rules)[i].status = IPA_FLT_STATUS_OF_ADD_FAILED;
-		} else {
-			((struct ipa_flt_rule_add_i *)
-			rules->rules)[i].status = 0;
-		}
-	}
-
-	if (rules->global) {
-		IPAERR_RL("no support for global filter rules\n");
-		result = -EPERM;
-		goto bail;
-	}
-
-	if (rules->commit)
-		if (ipa3_ctx->ctrl->ipa3_commit_flt(rules->ip)) {
-			result = -EPERM;
-			goto bail;
-		}
-	result = 0;
-bail:
-	mutex_unlock(&ipa3_ctx->lock);
-
-	return result;
-}
-
-/**
- * ipa3_add_flt_rule_after() - Add the specified filtering rules to SW after
- *  the rule which its handle is given and optionally commit to IPA HW
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_add_flt_rule_after(struct ipa_ioc_add_flt_rule_after *rules)
-{
-	int i;
-	int result;
-	struct ipa3_flt_tbl *tbl;
-	int ipa_ep_idx;
-	struct ipa3_flt_entry *entry;
-	struct ipa_flt_rule_i rule;
-
-	if (rules == NULL || rules->num_rules == 0 ||
-			rules->ip >= IPA_IP_MAX) {
-		IPAERR_RL("bad parm\n");
-		return -EINVAL;
-	}
-
-	if (rules->ep >= IPA_CLIENT_MAX) {
-		IPAERR_RL("bad parms ep=%d\n", rules->ep);
-		return -EINVAL;
-	}
-
-	mutex_lock(&ipa3_ctx->lock);
-
-	if (__ipa_add_flt_get_ep_idx(rules->ep, &ipa_ep_idx)) {
-		result = -EINVAL;
-		goto bail;
-	}
-
-	if (ipa_ep_idx >= IPA3_MAX_NUM_PIPES || ipa_ep_idx < 0) {
-		IPAERR_RL("invalid ipa_ep_idx=%u\n", ipa_ep_idx);
-		result = -EINVAL;
-		goto bail;
-	}
-
-	tbl = &ipa3_ctx->flt_tbl[ipa_ep_idx][rules->ip];
-
-	entry = ipa3_id_find(rules->add_after_hdl);
-	if (entry == NULL) {
-		IPAERR_RL("lookup failed\n");
-		result = -EINVAL;
-		goto bail;
-	}
-
-	if (entry->cookie != IPA_FLT_COOKIE) {
-		IPAERR_RL("Invalid cookie value =  %u flt hdl id = %d\n",
-			entry->cookie, rules->add_after_hdl);
-		result = -EINVAL;
-		goto bail;
-	}
-
-	if (entry->tbl != tbl) {
-		IPAERR_RL("given entry does not match the table\n");
-		result = -EINVAL;
-		goto bail;
-	}
-
-	if (tbl->sticky_rear)
-		if (&entry->link == tbl->head_flt_rule_list.prev) {
-			IPAERR_RL("cannot add rule at end of a sticky table");
-			result = -EINVAL;
-			goto bail;
-		}
-
-	IPADBG("add ep flt rule ip=%d ep=%d after hdl %d\n",
-			rules->ip, rules->ep, rules->add_after_hdl);
-
-	/*
-	 * we add all rules one after the other, if one insertion fails, it cuts
-	 * the chain (all following will receive fail status) following calls to
-	 * __ipa_add_flt_rule_after will fail (entry == NULL)
-	 */
-
-	for (i = 0; i < rules->num_rules; i++) {
-		/* if hashing not supported, all tables are non-hash tables*/
-		if (ipa3_ctx->ipa_fltrt_not_hashable)
-			rules->rules[i].rule.hashable = false;
-
-		__ipa_convert_flt_rule_in(
-				rules->rules[i].rule, &rule);
-
-		result = __ipa_add_flt_rule_after(tbl,
-				&rule,
-				&rules->rules[i].flt_rule_hdl,
-				rules->ip,
-				&entry);
-
-		__ipa_convert_flt_rule_out(rule,
-				&rules->rules[i].rule);
-
-		if (result) {
-			IPAERR_RL("failed to add flt rule %d\n", i);
-			rules->rules[i].status = IPA_FLT_STATUS_OF_ADD_FAILED;
-		} else {
-			rules->rules[i].status = 0;
-		}
-	}
-
-	if (rules->commit)
-		if (ipa3_ctx->ctrl->ipa3_commit_flt(rules->ip)) {
-			IPAERR("failed to commit flt rules\n");
-			result = -EPERM;
-			goto bail;
-		}
-	result = 0;
-bail:
-	mutex_unlock(&ipa3_ctx->lock);
-
-	return result;
-}
-
-/**
- * ipa3_add_flt_rule_after_v2() - Add the specified filtering
- *  rules to SW after the rule which its handle is given and
- *  optionally commit to IPA HW
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_add_flt_rule_after_v2(struct ipa_ioc_add_flt_rule_after_v2
-	*rules)
-{
-	int i;
-	int result;
-	struct ipa3_flt_tbl *tbl;
-	int ipa_ep_idx;
-	struct ipa3_flt_entry *entry;
-
-	if (rules == NULL || rules->num_rules == 0 ||
-			rules->ip >= IPA_IP_MAX) {
-		IPAERR_RL("bad parm\n");
-		return -EINVAL;
-	}
-
-	if (rules->ep >= IPA_CLIENT_MAX) {
-		IPAERR_RL("bad parms ep=%d\n", rules->ep);
-		return -EINVAL;
-	}
-
-	mutex_lock(&ipa3_ctx->lock);
-
-	if (__ipa_add_flt_get_ep_idx(rules->ep, &ipa_ep_idx)) {
-		result = -EINVAL;
-		goto bail;
-	}
-
-	if (ipa_ep_idx >= IPA3_MAX_NUM_PIPES ||
-		ipa_ep_idx < 0) {
-		IPAERR_RL("invalid ipa_ep_idx=%u\n", ipa_ep_idx);
-		result = -EINVAL;
-		goto bail;
-	}
-
-	tbl = &ipa3_ctx->flt_tbl[ipa_ep_idx][rules->ip];
-
-	entry = ipa3_id_find(rules->add_after_hdl);
-	if (entry == NULL) {
-		IPAERR_RL("lookup failed\n");
-		result = -EINVAL;
-		goto bail;
-	}
-
-	if (entry->cookie != IPA_FLT_COOKIE) {
-		IPAERR_RL("Invalid cookie value =  %u flt hdl id = %d\n",
-			entry->cookie, rules->add_after_hdl);
-		result = -EINVAL;
-		goto bail;
-	}
-
-	if (entry->tbl != tbl) {
-		IPAERR_RL("given entry does not match the table\n");
-		result = -EINVAL;
-		goto bail;
-	}
-
-	if (tbl->sticky_rear)
-		if (&entry->link == tbl->head_flt_rule_list.prev) {
-			IPAERR_RL("cannot add rule at end of a sticky table");
-			result = -EINVAL;
-			goto bail;
-		}
-
-	IPADBG("add ep flt rule ip=%d ep=%d after hdl %d\n",
-			rules->ip, rules->ep, rules->add_after_hdl);
-
-	/*
-	 * we add all rules one after the other, if one insertion fails, it cuts
-	 * the chain (all following will receive fail status) following calls to
-	 * __ipa_add_flt_rule_after will fail (entry == NULL)
-	 */
-
-	for (i = 0; i < rules->num_rules; i++) {
-		/* if hashing not supported, all tables are non-hash tables*/
-		if (ipa3_ctx->ipa_fltrt_not_hashable)
-			((struct ipa_flt_rule_add_i *)
-			rules->rules)[i].rule.hashable = false;
-		result = __ipa_add_flt_rule_after(tbl,
-				&(((struct ipa_flt_rule_add_i *)
-				rules->rules)[i].rule),
-				&(((struct ipa_flt_rule_add_i *)
-				rules->rules)[i].flt_rule_hdl),
-				rules->ip,
-				&entry);
-		if (result) {
-			IPAERR_RL("failed to add flt rule %d\n", i);
-			((struct ipa_flt_rule_add_i *)
-			rules->rules)[i].status = IPA_FLT_STATUS_OF_ADD_FAILED;
-		} else {
-			((struct ipa_flt_rule_add_i *)
-			rules->rules)[i].status = 0;
-		}
-	}
-
-	if (rules->commit)
-		if (ipa3_ctx->ctrl->ipa3_commit_flt(rules->ip)) {
-			IPAERR("failed to commit flt rules\n");
-			result = -EPERM;
-			goto bail;
-		}
-	result = 0;
-bail:
-	mutex_unlock(&ipa3_ctx->lock);
-
-	return result;
-}
-
-/**
- * ipa3_del_flt_rule() - Remove the specified filtering rules from SW and
- * optionally commit to IPA HW
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_del_flt_rule(struct ipa_ioc_del_flt_rule *hdls)
-{
-	int i;
-	int result;
-
-	if (hdls == NULL || hdls->num_hdls == 0 || hdls->ip >= IPA_IP_MAX) {
-		IPAERR_RL("bad param\n");
-		return -EINVAL;
-	}
-
-	mutex_lock(&ipa3_ctx->lock);
-	for (i = 0; i < hdls->num_hdls; i++) {
-		if (__ipa_del_flt_rule(hdls->hdl[i].hdl)) {
-			IPAERR_RL("failed to del flt rule %i\n", i);
-			hdls->hdl[i].status = IPA_FLT_STATUS_OF_DEL_FAILED;
-		} else {
-			hdls->hdl[i].status = 0;
-		}
-	}
-
-	if (hdls->commit)
-		if (ipa3_ctx->ctrl->ipa3_commit_flt(hdls->ip)) {
-			result = -EPERM;
-			goto bail;
-		}
-	result = 0;
-bail:
-	mutex_unlock(&ipa3_ctx->lock);
-
-	return result;
-}
-
-/**
- * ipa3_mdfy_flt_rule() - Modify the specified filtering rules in SW and
- * optionally commit to IPA HW
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_mdfy_flt_rule(struct ipa_ioc_mdfy_flt_rule *hdls)
-{
-	int i;
-	int result;
-	struct ipa_flt_rule_mdfy_i rule;
-
-	if (hdls == NULL || hdls->num_rules == 0 || hdls->ip >= IPA_IP_MAX) {
-		IPAERR_RL("bad parm\n");
-		return -EINVAL;
-	}
-
-	mutex_lock(&ipa3_ctx->lock);
-	for (i = 0; i < hdls->num_rules; i++) {
-		/* if hashing not supported, all tables are non-hash tables*/
-		if (ipa3_ctx->ipa_fltrt_not_hashable)
-			hdls->rules[i].rule.hashable = false;
-		__ipa_convert_flt_mdfy_in(hdls->rules[i], &rule);
-		if (__ipa_mdfy_flt_rule(&rule, hdls->ip)) {
-			IPAERR_RL("failed to mdfy flt rule %i\n", i);
-			hdls->rules[i].status = IPA_FLT_STATUS_OF_MDFY_FAILED;
-		} else {
-			hdls->rules[i].status = 0;
-			__ipa_convert_flt_mdfy_out(rule, &hdls->rules[i]);
-		}
-	}
-
-	if (hdls->commit)
-		if (ipa3_ctx->ctrl->ipa3_commit_flt(hdls->ip)) {
-			result = -EPERM;
-			goto bail;
-		}
-	result = 0;
-bail:
-	mutex_unlock(&ipa3_ctx->lock);
-
-	return result;
-}
-
-/**
- * ipa3_mdfy_flt_rule_v2() - Modify the specified filtering
- * rules in SW and optionally commit to IPA HW
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_mdfy_flt_rule_v2(struct ipa_ioc_mdfy_flt_rule_v2 *hdls)
-{
-	int i;
-	int result;
-
-	if (hdls == NULL || hdls->num_rules == 0 || hdls->ip >= IPA_IP_MAX) {
-		IPAERR_RL("bad parm\n");
-		return -EINVAL;
-	}
-
-	mutex_lock(&ipa3_ctx->lock);
-	for (i = 0; i < hdls->num_rules; i++) {
-		/* if hashing not supported, all tables are non-hash tables*/
-		if (ipa3_ctx->ipa_fltrt_not_hashable)
-			((struct ipa_flt_rule_mdfy_i *)
-			hdls->rules)[i].rule.hashable = false;
-		if (__ipa_mdfy_flt_rule(&(((struct ipa_flt_rule_mdfy_i *)
-			hdls->rules)[i]), hdls->ip)) {
-			IPAERR_RL("failed to mdfy flt rule %i\n", i);
-			((struct ipa_flt_rule_mdfy_i *)
-			hdls->rules)[i].status = IPA_FLT_STATUS_OF_MDFY_FAILED;
-		} else {
-			((struct ipa_flt_rule_mdfy_i *)
-			hdls->rules)[i].status = 0;
-		}
-	}
-
-	if (hdls->commit)
-		if (ipa3_ctx->ctrl->ipa3_commit_flt(hdls->ip)) {
-			result = -EPERM;
-			goto bail;
-		}
-	result = 0;
-bail:
-	mutex_unlock(&ipa3_ctx->lock);
-
-	return result;
-}
-
-/**
- * ipa3_commit_flt() - Commit the current SW filtering table of specified type
- * to IPA HW
- * @ip:	[in] the family of routing tables
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_commit_flt(enum ipa_ip_type ip)
-{
-	int result;
-
-	if (ip >= IPA_IP_MAX) {
-		IPAERR_RL("bad param\n");
-		return -EINVAL;
-	}
-
-	mutex_lock(&ipa3_ctx->lock);
-
-	if (ipa3_ctx->ctrl->ipa3_commit_flt(ip)) {
-		result = -EPERM;
-		goto bail;
-	}
-	result = 0;
-
-bail:
-	mutex_unlock(&ipa3_ctx->lock);
-
-	return result;
-}
-
-/**
- * ipa3_reset_flt() - Reset the current SW filtering table of specified type
- * (does not commit to HW)
- * @ip:	[in] the family of routing tables
- * @user_only:	[in] indicate rules deleted by userspace
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_reset_flt(enum ipa_ip_type ip, bool user_only)
-{
-	struct ipa3_flt_tbl *tbl;
-	struct ipa3_flt_entry *entry;
-	struct ipa3_flt_entry *next;
-	int i;
-	int id;
-	int rule_id;
-
-	if (ip >= IPA_IP_MAX) {
-		IPAERR_RL("bad parm\n");
-		return -EINVAL;
-	}
-
-	mutex_lock(&ipa3_ctx->lock);
-	for (i = 0; i < ipa3_ctx->ipa_num_pipes; i++) {
-		if (!ipa_is_ep_support_flt(i))
-			continue;
-
-		tbl = &ipa3_ctx->flt_tbl[i][ip];
-		list_for_each_entry_safe(entry, next, &tbl->head_flt_rule_list,
-				link) {
-			if (ipa3_id_find(entry->id) == NULL) {
-				WARN_ON_RATELIMIT_IPA(1);
-				mutex_unlock(&ipa3_ctx->lock);
-				return -EFAULT;
-			}
-
-			if (!user_only ||
-					entry->ipacm_installed) {
-				list_del(&entry->link);
-				entry->tbl->rule_cnt--;
-				if (entry->rt_tbl)
-					entry->rt_tbl->ref_cnt--;
-				/* if rule id was allocated from idr, remove */
-				rule_id = entry->rule_id;
-				id = entry->id;
-				if ((rule_id < ipahal_get_rule_id_hi_bit()) &&
-					(rule_id >= ipahal_get_low_rule_id()))
-					idr_remove(entry->tbl->rule_ids,
-						rule_id);
-				entry->cookie = 0;
-				kmem_cache_free(ipa3_ctx->flt_rule_cache,
-								entry);
-
-				/* remove the handle from the database */
-				ipa3_id_remove(id);
-			}
-		}
-	}
-
-	/* commit the change to IPA-HW */
-	if (ipa3_ctx->ctrl->ipa3_commit_flt(IPA_IP_v4) ||
-		ipa3_ctx->ctrl->ipa3_commit_flt(IPA_IP_v6)) {
-		IPAERR("fail to commit flt-rule\n");
-		WARN_ON_RATELIMIT_IPA(1);
-		mutex_unlock(&ipa3_ctx->lock);
-		return -EPERM;
-	}
-	mutex_unlock(&ipa3_ctx->lock);
-	return 0;
-}
-
-void ipa3_install_dflt_flt_rules(u32 ipa_ep_idx)
-{
-	struct ipa3_flt_tbl *tbl;
-	struct ipa3_ep_context *ep;
-	struct ipa_flt_rule_i rule;
-
-	if (ipa_ep_idx >= IPA3_MAX_NUM_PIPES) {
-		IPAERR("invalid ipa_ep_idx=%u\n", ipa_ep_idx);
-		ipa_assert();
-		return;
-	}
-
-	ep = &ipa3_ctx->ep[ipa_ep_idx];
-
-	if (!ipa_is_ep_support_flt(ipa_ep_idx)) {
-		IPADBG("cannot add flt rules to non filtering pipe num %d\n",
-			ipa_ep_idx);
-		return;
-	}
-
-	memset(&rule, 0, sizeof(rule));
-
-	mutex_lock(&ipa3_ctx->lock);
-	tbl = &ipa3_ctx->flt_tbl[ipa_ep_idx][IPA_IP_v4];
-	rule.action = IPA_PASS_TO_EXCEPTION;
-	__ipa_add_flt_rule(tbl, IPA_IP_v4, &rule, true,
-			&ep->dflt_flt4_rule_hdl, false);
-	ipa3_ctx->ctrl->ipa3_commit_flt(IPA_IP_v4);
-	tbl->sticky_rear = true;
-
-	tbl = &ipa3_ctx->flt_tbl[ipa_ep_idx][IPA_IP_v6];
-	rule.action = IPA_PASS_TO_EXCEPTION;
-	__ipa_add_flt_rule(tbl, IPA_IP_v6, &rule, true,
-			&ep->dflt_flt6_rule_hdl, false);
-	ipa3_ctx->ctrl->ipa3_commit_flt(IPA_IP_v6);
-	tbl->sticky_rear = true;
-	mutex_unlock(&ipa3_ctx->lock);
-}
-
-void ipa3_delete_dflt_flt_rules(u32 ipa_ep_idx)
-{
-	struct ipa3_ep_context *ep = &ipa3_ctx->ep[ipa_ep_idx];
-	struct ipa3_flt_tbl *tbl;
-
-	mutex_lock(&ipa3_ctx->lock);
-	if (ep->dflt_flt4_rule_hdl) {
-		tbl = &ipa3_ctx->flt_tbl[ipa_ep_idx][IPA_IP_v4];
-		__ipa_del_flt_rule(ep->dflt_flt4_rule_hdl);
-		ipa3_ctx->ctrl->ipa3_commit_flt(IPA_IP_v4);
-		/* Reset the sticky flag. */
-		tbl->sticky_rear = false;
-		ep->dflt_flt4_rule_hdl = 0;
-	}
-	if (ep->dflt_flt6_rule_hdl) {
-		tbl = &ipa3_ctx->flt_tbl[ipa_ep_idx][IPA_IP_v6];
-		__ipa_del_flt_rule(ep->dflt_flt6_rule_hdl);
-		ipa3_ctx->ctrl->ipa3_commit_flt(IPA_IP_v6);
-		/* Reset the sticky flag. */
-		tbl->sticky_rear = false;
-		ep->dflt_flt6_rule_hdl = 0;
-	}
-	mutex_unlock(&ipa3_ctx->lock);
-}
-
-/**
- * ipa3_set_flt_tuple_mask() - Sets the flt tuple masking for the given pipe
- *  Pipe must be for AP EP (not modem) and support filtering
- *  updates the the filtering masking values without changing the rt ones.
- *
- * @pipe_idx: filter pipe index to configure the tuple masking
- * @tuple: the tuple members masking
- * Returns:	0 on success, negative on failure
- *
- */
-int ipa3_set_flt_tuple_mask(int pipe_idx, struct ipahal_reg_hash_tuple *tuple)
-{
-	struct ipahal_reg_fltrt_hash_tuple fltrt_tuple;
-
-	if (!tuple) {
-		IPAERR_RL("bad tuple\n");
-		return -EINVAL;
-	}
-
-	if (pipe_idx >= ipa3_ctx->ipa_num_pipes || pipe_idx < 0) {
-		IPAERR("bad pipe index!\n");
-		return -EINVAL;
-	}
-
-	if (!ipa_is_ep_support_flt(pipe_idx)) {
-		IPAERR("pipe %d not filtering pipe\n", pipe_idx);
-		return -EINVAL;
-	}
-
-	if (ipa_is_modem_pipe(pipe_idx)) {
-		IPAERR("modem pipe tuple is not configured by AP\n");
-		return -EINVAL;
-	}
-
-	ipahal_read_reg_n_fields(IPA_ENDP_FILTER_ROUTER_HSH_CFG_n,
-		pipe_idx, &fltrt_tuple);
-	fltrt_tuple.flt = *tuple;
-	ipahal_write_reg_n_fields(IPA_ENDP_FILTER_ROUTER_HSH_CFG_n,
-		pipe_idx, &fltrt_tuple);
-
-	return 0;
-}
-
-/**
- * ipa3_flt_read_tbl_from_hw() -Read filtering table from IPA HW
- * @pipe_idx: IPA endpoint index
- * @ip_type: IPv4 or IPv6 table
- * @hashable: hashable or non-hashable table
- * @entry: array to fill the table entries
- * @num_entry: number of entries in entry array. set by the caller to indicate
- *  entry array size. Then set by this function as an output parameter to
- *  indicate the number of entries in the array
- *
- * This function reads the filtering table from IPA SRAM and prepares an array
- * of entries. This function is mainly used for debugging purposes.
- *
- * If empty table or Modem Apps table, zero entries will be returned.
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa3_flt_read_tbl_from_hw(u32 pipe_idx, enum ipa_ip_type ip_type,
-	bool hashable, struct ipahal_flt_rule_entry entry[], int *num_entry)
-{
-	void *ipa_sram_mmio;
-	u64 hdr_base_ofst;
-	int tbl_entry_idx;
-	int i;
-	int res = 0;
-	u64 tbl_addr;
-	bool is_sys;
-	u8 *rule_addr;
-	struct ipa_mem_buffer *sys_tbl_mem;
-	int rule_idx;
-	struct ipa3_flt_tbl *flt_tbl_ptr;
-
-	IPADBG("pipe_idx=%d ip=%d hashable=%d entry=0x%pK num_entry=0x%pK\n",
-		pipe_idx, ip_type, hashable, entry, num_entry);
-
-	/*
-	 * SRAM memory not allocated to hash tables. Reading of hash table
-	 * rules operation not supported
-	 */
-	if (hashable && ipa3_ctx->ipa_fltrt_not_hashable) {
-		IPAERR_RL("Reading hashable rules not supported\n");
-		*num_entry = 0;
-		return 0;
-	}
-
-	if (pipe_idx >= ipa3_ctx->ipa_num_pipes ||
-		pipe_idx >= IPA3_MAX_NUM_PIPES || ip_type >= IPA_IP_MAX ||
-		!entry || !num_entry) {
-		IPAERR_RL("Invalid pipe_idx=%u\n", pipe_idx);
-		return -EFAULT;
-	}
-
-	if (!ipa_is_ep_support_flt(pipe_idx)) {
-		IPAERR_RL("pipe %d does not support filtering\n", pipe_idx);
-		return -EINVAL;
-	}
-
-	flt_tbl_ptr = &ipa3_ctx->flt_tbl[pipe_idx][ip_type];
-	/* map IPA SRAM */
-	ipa_sram_mmio = ioremap(ipa3_ctx->ipa_wrapper_base +
-		ipa3_ctx->ctrl->ipa_reg_base_ofst +
-		ipahal_get_reg_n_ofst(IPA_SW_AREA_RAM_DIRECT_ACCESS_n,
-			ipa3_ctx->smem_restricted_bytes / 4),
-		ipa3_ctx->smem_sz);
-	if (!ipa_sram_mmio) {
-		IPAERR("fail to ioremap IPA SRAM\n");
-		return -ENOMEM;
-	}
-
-	memset(entry, 0, sizeof(*entry) * (*num_entry));
-	if (hashable) {
-		if (ip_type == IPA_IP_v4)
-			hdr_base_ofst =
-				IPA_MEM_PART(v4_flt_hash_ofst);
-		else
-			hdr_base_ofst =
-				IPA_MEM_PART(v6_flt_hash_ofst);
-	} else {
-		if (ip_type == IPA_IP_v4)
-			hdr_base_ofst =
-				IPA_MEM_PART(v4_flt_nhash_ofst);
-		else
-			hdr_base_ofst =
-				IPA_MEM_PART(v6_flt_nhash_ofst);
-	}
-
-	/* calculate the index of the tbl entry */
-	tbl_entry_idx = 1; /* skip the bitmap */
-	for (i = 0; i < pipe_idx; i++)
-		if (ipa3_ctx->ep_flt_bitmap & (1 << i))
-			tbl_entry_idx++;
-
-	IPADBG("hdr_base_ofst=0x%llx tbl_entry_idx=%d\n",
-		hdr_base_ofst, tbl_entry_idx);
-
-	res = ipahal_fltrt_read_addr_from_hdr(ipa_sram_mmio + hdr_base_ofst,
-		tbl_entry_idx, &tbl_addr, &is_sys);
-	if (res) {
-		IPAERR("failed to read table address from header structure\n");
-		goto bail;
-	}
-	IPADBG("flt tbl ep=%d: tbl_addr=0x%llx is_sys=%d\n",
-		pipe_idx, tbl_addr, is_sys);
-	if (!tbl_addr) {
-		IPAERR("invalid flt tbl addr\n");
-		res = -EFAULT;
-		goto bail;
-	}
-
-	/* for tables resides in DDR access it from the virtual memory */
-	if (is_sys) {
-		sys_tbl_mem =
-			&flt_tbl_ptr->curr_mem[hashable ? IPA_RULE_HASHABLE :
-			IPA_RULE_NON_HASHABLE];
-		if (sys_tbl_mem->phys_base &&
-			sys_tbl_mem->phys_base != tbl_addr) {
-			IPAERR("mismatch addr: parsed=%llx sw=%pad\n",
-				tbl_addr, &sys_tbl_mem->phys_base);
-		}
-		if (sys_tbl_mem->phys_base)
-			rule_addr = sys_tbl_mem->base;
-		else
-			rule_addr = NULL;
-	} else {
-		rule_addr = ipa_sram_mmio + hdr_base_ofst + tbl_addr;
-	}
-
-	IPADBG("First rule addr 0x%pK\n", rule_addr);
-
-	if (!rule_addr) {
-		/* Modem table in system memory or empty table */
-		*num_entry = 0;
-		goto bail;
-	}
-
-	rule_idx = 0;
-	while (rule_idx < *num_entry) {
-		res = ipahal_flt_parse_hw_rule(rule_addr, &entry[rule_idx]);
-		if (res) {
-			IPAERR("failed parsing flt rule\n");
-			goto bail;
-		}
-
-		IPADBG("rule_size=%d\n", entry[rule_idx].rule_size);
-		if (!entry[rule_idx].rule_size)
-			break;
-
-		rule_addr += entry[rule_idx].rule_size;
-		rule_idx++;
-	}
-	*num_entry = rule_idx;
-bail:
-	iounmap(ipa_sram_mmio);
-	return 0;
-}

ipa/ipa_v3/ipa_hdr.c

@@ -1,1335 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2012-2019, The Linux Foundation. All rights reserved.
- */
-
-#include "ipa_i.h"
-#include "ipahal/ipahal.h"
-
-static const u32 ipa_hdr_bin_sz[IPA_HDR_BIN_MAX] = { 8, 16, 24, 36, 64};
-static const u32 ipa_hdr_proc_ctx_bin_sz[IPA_HDR_PROC_CTX_BIN_MAX] = { 32, 64};
-
-#define HDR_TYPE_IS_VALID(type) \
-	((type) >= 0 && (type) < IPA_HDR_L2_MAX)
-
-#define HDR_PROC_TYPE_IS_VALID(type) \
-	((type) >= 0 && (type) < IPA_HDR_PROC_MAX)
-
-/**
- * ipa3_generate_hdr_hw_tbl() - generates the headers table
- * @mem:	[out] buffer to put the header table
- *
- * Returns:	0 on success, negative on failure
- */
-static int ipa3_generate_hdr_hw_tbl(struct ipa_mem_buffer *mem)
-{
-	struct ipa3_hdr_entry *entry;
-	gfp_t flag = GFP_KERNEL;
-
-	mem->size = ipa3_ctx->hdr_tbl.end;
-
-	if (mem->size == 0) {
-		IPAERR("hdr tbl empty\n");
-		return -EPERM;
-	}
-	IPADBG_LOW("tbl_sz=%d\n", ipa3_ctx->hdr_tbl.end);
-
-alloc:
-	mem->base = dma_zalloc_coherent(ipa3_ctx->pdev, mem->size,
-			&mem->phys_base, flag);
-	if (!mem->base) {
-		if (flag == GFP_KERNEL) {
-			flag = GFP_ATOMIC;
-			goto alloc;
-		}
-		IPAERR("fail to alloc DMA buff of size %d\n", mem->size);
-		return -ENOMEM;
-	}
-
-	list_for_each_entry(entry, &ipa3_ctx->hdr_tbl.head_hdr_entry_list,
-			link) {
-		if (entry->is_hdr_proc_ctx)
-			continue;
-		IPADBG_LOW("hdr of len %d ofst=%d\n", entry->hdr_len,
-				entry->offset_entry->offset);
-		ipahal_cp_hdr_to_hw_buff(mem->base, entry->offset_entry->offset,
-				entry->hdr, entry->hdr_len);
-	}
-
-	return 0;
-}
-
-static int ipa3_hdr_proc_ctx_to_hw_format(struct ipa_mem_buffer *mem,
-	u64 hdr_base_addr)
-{
-	struct ipa3_hdr_proc_ctx_entry *entry;
-	int ret;
-	int ep;
-	struct ipa_ep_cfg *cfg_ptr;
-	struct ipa_l2tp_header_remove_procparams *l2p_hdr_rm_ptr;
-
-	list_for_each_entry(entry,
-			&ipa3_ctx->hdr_proc_ctx_tbl.head_proc_ctx_entry_list,
-			link) {
-		IPADBG_LOW("processing type %d ofst=%d\n",
-			entry->type, entry->offset_entry->offset);
-
-		if (entry->l2tp_params.is_dst_pipe_valid) {
-			ep = ipa3_get_ep_mapping(entry->l2tp_params.dst_pipe);
-
-			if (ep >= 0) {
-				cfg_ptr = &ipa3_ctx->ep[ep].cfg;
-				l2p_hdr_rm_ptr =
-					&entry->l2tp_params.hdr_remove_param;
-				l2p_hdr_rm_ptr->hdr_ofst_pkt_size_valid =
-					cfg_ptr->hdr.hdr_ofst_pkt_size_valid;
-				l2p_hdr_rm_ptr->hdr_ofst_pkt_size =
-					cfg_ptr->hdr.hdr_ofst_pkt_size;
-				l2p_hdr_rm_ptr->hdr_endianness =
-					cfg_ptr->hdr_ext.hdr_little_endian ?
-					0 : 1;
-			}
-		}
-
-		ret = ipahal_cp_proc_ctx_to_hw_buff(entry->type, mem->base,
-				entry->offset_entry->offset,
-				entry->hdr->hdr_len,
-				entry->hdr->is_hdr_proc_ctx,
-				entry->hdr->phys_base,
-				hdr_base_addr,
-				entry->hdr->offset_entry,
-				entry->l2tp_params,
-				ipa3_ctx->use_64_bit_dma_mask);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
-/**
- * ipa3_generate_hdr_proc_ctx_hw_tbl() -
- * generates the headers processing context table.
- * @mem:		[out] buffer to put the processing context table
- * @aligned_mem:	[out] actual processing context table (with alignment).
- *			Processing context table needs to be 8 Bytes aligned.
- *
- * Returns:	0 on success, negative on failure
- */
-static int ipa3_generate_hdr_proc_ctx_hw_tbl(u64 hdr_sys_addr,
-	struct ipa_mem_buffer *mem, struct ipa_mem_buffer *aligned_mem)
-{
-	u64 hdr_base_addr;
-
-	mem->size = (ipa3_ctx->hdr_proc_ctx_tbl.end) ? : 4;
-
-	/* make sure table is aligned */
-	mem->size += IPA_HDR_PROC_CTX_TABLE_ALIGNMENT_BYTE;
-
-	IPADBG_LOW("tbl_sz=%d\n", ipa3_ctx->hdr_proc_ctx_tbl.end);
-
-	mem->base = dma_alloc_coherent(ipa3_ctx->pdev, mem->size,
-			&mem->phys_base, GFP_KERNEL);
-	if (!mem->base) {
-		IPAERR("fail to alloc DMA buff of size %d\n", mem->size);
-		return -ENOMEM;
-	}
-
-	aligned_mem->phys_base =
-		IPA_HDR_PROC_CTX_TABLE_ALIGNMENT(mem->phys_base);
-	aligned_mem->base = mem->base +
-		(aligned_mem->phys_base - mem->phys_base);
-	aligned_mem->size = mem->size - IPA_HDR_PROC_CTX_TABLE_ALIGNMENT_BYTE;
-	memset(aligned_mem->base, 0, aligned_mem->size);
-	hdr_base_addr = (ipa3_ctx->hdr_tbl_lcl) ? IPA_MEM_PART(apps_hdr_ofst) :
-		hdr_sys_addr;
-	return ipa3_hdr_proc_ctx_to_hw_format(aligned_mem, hdr_base_addr);
-}
-
-/**
- * __ipa_commit_hdr_v3_0() - Commits the header table from memory to HW
- *
- * Returns:	0 on success, negative on failure
- */
-int __ipa_commit_hdr_v3_0(void)
-{
-	struct ipa3_desc desc[2];
-	struct ipa_mem_buffer hdr_mem;
-	struct ipa_mem_buffer ctx_mem;
-	struct ipa_mem_buffer aligned_ctx_mem;
-	struct ipahal_imm_cmd_dma_shared_mem dma_cmd_hdr = {0};
-	struct ipahal_imm_cmd_dma_shared_mem dma_cmd_ctx = {0};
-	struct ipahal_imm_cmd_register_write reg_write_cmd = {0};
-	struct ipahal_imm_cmd_hdr_init_system hdr_init_cmd = {0};
-	struct ipahal_imm_cmd_pyld *hdr_cmd_pyld = NULL;
-	struct ipahal_imm_cmd_pyld *ctx_cmd_pyld = NULL;
-	int rc = -EFAULT;
-	u32 proc_ctx_size;
-	u32 proc_ctx_ofst;
-	u32 proc_ctx_size_ddr;
-
-	memset(desc, 0, 2 * sizeof(struct ipa3_desc));
-
-	if (ipa3_generate_hdr_hw_tbl(&hdr_mem)) {
-		IPAERR("fail to generate HDR HW TBL\n");
-		goto end;
-	}
-
-	if (ipa3_generate_hdr_proc_ctx_hw_tbl(hdr_mem.phys_base, &ctx_mem,
-	    &aligned_ctx_mem)) {
-		IPAERR("fail to generate HDR PROC CTX HW TBL\n");
-		goto end;
-	}
-
-	if (ipa3_ctx->hdr_tbl_lcl) {
-		if (hdr_mem.size > IPA_MEM_PART(apps_hdr_size)) {
-			IPAERR("tbl too big needed %d avail %d\n", hdr_mem.size,
-				IPA_MEM_PART(apps_hdr_size));
-			goto end;
-		} else {
-			dma_cmd_hdr.is_read = false; /* write operation */
-			dma_cmd_hdr.skip_pipeline_clear = false;
-			dma_cmd_hdr.pipeline_clear_options = IPAHAL_HPS_CLEAR;
-			dma_cmd_hdr.system_addr = hdr_mem.phys_base;
-			dma_cmd_hdr.size = hdr_mem.size;
-			dma_cmd_hdr.local_addr =
-				ipa3_ctx->smem_restricted_bytes +
-				IPA_MEM_PART(apps_hdr_ofst);
-			hdr_cmd_pyld = ipahal_construct_imm_cmd(
-				IPA_IMM_CMD_DMA_SHARED_MEM,
-				&dma_cmd_hdr, false);
-			if (!hdr_cmd_pyld) {
-				IPAERR("fail construct dma_shared_mem cmd\n");
-				goto end;
-			}
-		}
-	} else {
-		if (hdr_mem.size > IPA_MEM_PART(apps_hdr_size_ddr)) {
-			IPAERR("tbl too big needed %d avail %d\n", hdr_mem.size,
-				IPA_MEM_PART(apps_hdr_size_ddr));
-			goto end;
-		} else {
-			hdr_init_cmd.hdr_table_addr = hdr_mem.phys_base;
-			hdr_cmd_pyld = ipahal_construct_imm_cmd(
-				IPA_IMM_CMD_HDR_INIT_SYSTEM,
-				&hdr_init_cmd, false);
-			if (!hdr_cmd_pyld) {
-				IPAERR("fail construct hdr_init_system cmd\n");
-				goto end;
-			}
-		}
-	}
-	ipa3_init_imm_cmd_desc(&desc[0], hdr_cmd_pyld);
-	IPA_DUMP_BUFF(hdr_mem.base, hdr_mem.phys_base, hdr_mem.size);
-
-	proc_ctx_size = IPA_MEM_PART(apps_hdr_proc_ctx_size);
-	proc_ctx_ofst = IPA_MEM_PART(apps_hdr_proc_ctx_ofst);
-	if (ipa3_ctx->hdr_proc_ctx_tbl_lcl) {
-		if (aligned_ctx_mem.size > proc_ctx_size) {
-			IPAERR("tbl too big needed %d avail %d\n",
-				aligned_ctx_mem.size,
-				proc_ctx_size);
-			goto end;
-		} else {
-			dma_cmd_ctx.is_read = false; /* Write operation */
-			dma_cmd_ctx.skip_pipeline_clear = false;
-			dma_cmd_ctx.pipeline_clear_options = IPAHAL_HPS_CLEAR;
-			dma_cmd_ctx.system_addr = aligned_ctx_mem.phys_base;
-			dma_cmd_ctx.size = aligned_ctx_mem.size;
-			dma_cmd_ctx.local_addr =
-				ipa3_ctx->smem_restricted_bytes +
-				proc_ctx_ofst;
-			ctx_cmd_pyld = ipahal_construct_imm_cmd(
-				IPA_IMM_CMD_DMA_SHARED_MEM,
-				&dma_cmd_ctx, false);
-			if (!ctx_cmd_pyld) {
-				IPAERR("fail construct dma_shared_mem cmd\n");
-				goto end;
-			}
-		}
-	} else {
-		proc_ctx_size_ddr = IPA_MEM_PART(apps_hdr_proc_ctx_size_ddr);
-		if (aligned_ctx_mem.size > proc_ctx_size_ddr) {
-			IPAERR("tbl too big, needed %d avail %d\n",
-				aligned_ctx_mem.size,
-				proc_ctx_size_ddr);
-			goto end;
-		} else {
-			reg_write_cmd.skip_pipeline_clear = false;
-			reg_write_cmd.pipeline_clear_options =
-				IPAHAL_HPS_CLEAR;
-			reg_write_cmd.offset =
-				ipahal_get_reg_ofst(
-				IPA_SYS_PKT_PROC_CNTXT_BASE);
-			reg_write_cmd.value = aligned_ctx_mem.phys_base;
-			reg_write_cmd.value_mask =
-				~(IPA_HDR_PROC_CTX_TABLE_ALIGNMENT_BYTE - 1);
-			ctx_cmd_pyld = ipahal_construct_imm_cmd(
-				IPA_IMM_CMD_REGISTER_WRITE,
-				&reg_write_cmd, false);
-			if (!ctx_cmd_pyld) {
-				IPAERR("fail construct register_write cmd\n");
-				goto end;
-			}
-		}
-	}
-	ipa3_init_imm_cmd_desc(&desc[1], ctx_cmd_pyld);
-	IPA_DUMP_BUFF(ctx_mem.base, ctx_mem.phys_base, ctx_mem.size);
-
-	if (ipa3_send_cmd(2, desc))
-		IPAERR("fail to send immediate command\n");
-	else
-		rc = 0;
-
-	if (ipa3_ctx->hdr_tbl_lcl) {
-		dma_free_coherent(ipa3_ctx->pdev, hdr_mem.size, hdr_mem.base,
-			hdr_mem.phys_base);
-	} else {
-		if (!rc) {
-			if (ipa3_ctx->hdr_mem.phys_base)
-				dma_free_coherent(ipa3_ctx->pdev,
-				ipa3_ctx->hdr_mem.size,
-				ipa3_ctx->hdr_mem.base,
-				ipa3_ctx->hdr_mem.phys_base);
-			ipa3_ctx->hdr_mem = hdr_mem;
-		}
-	}
-
-	if (ipa3_ctx->hdr_proc_ctx_tbl_lcl) {
-		dma_free_coherent(ipa3_ctx->pdev, ctx_mem.size, ctx_mem.base,
-			ctx_mem.phys_base);
-	} else {
-		if (!rc) {
-			if (ipa3_ctx->hdr_proc_ctx_mem.phys_base)
-				dma_free_coherent(ipa3_ctx->pdev,
-					ipa3_ctx->hdr_proc_ctx_mem.size,
-					ipa3_ctx->hdr_proc_ctx_mem.base,
-					ipa3_ctx->hdr_proc_ctx_mem.phys_base);
-			ipa3_ctx->hdr_proc_ctx_mem = ctx_mem;
-		}
-	}
-
-end:
-	if (ctx_cmd_pyld)
-		ipahal_destroy_imm_cmd(ctx_cmd_pyld);
-
-	if (hdr_cmd_pyld)
-		ipahal_destroy_imm_cmd(hdr_cmd_pyld);
-
-	return rc;
-}
-
-static int __ipa_add_hdr_proc_ctx(struct ipa_hdr_proc_ctx_add *proc_ctx,
-	bool add_ref_hdr, bool user_only)
-{
-	struct ipa3_hdr_entry *hdr_entry;
-	struct ipa3_hdr_proc_ctx_entry *entry;
-	struct ipa3_hdr_proc_ctx_offset_entry *offset;
-	u32 bin;
-	struct ipa3_hdr_proc_ctx_tbl *htbl = &ipa3_ctx->hdr_proc_ctx_tbl;
-	int id;
-	int needed_len;
-	int mem_size;
-
-	IPADBG_LOW("Add processing type %d hdr_hdl %d\n",
-		proc_ctx->type, proc_ctx->hdr_hdl);
-
-	if (!HDR_PROC_TYPE_IS_VALID(proc_ctx->type)) {
-		IPAERR_RL("invalid processing type %d\n", proc_ctx->type);
-		return -EINVAL;
-	}
-
-	hdr_entry = ipa3_id_find(proc_ctx->hdr_hdl);
-	if (!hdr_entry) {
-		IPAERR_RL("hdr_hdl is invalid\n");
-		return -EINVAL;
-	}
-	if (hdr_entry->cookie != IPA_HDR_COOKIE) {
-		IPAERR_RL("Invalid header cookie %u\n", hdr_entry->cookie);
-		WARN_ON_RATELIMIT_IPA(1);
-		return -EINVAL;
-	}
-	IPADBG("Associated header is name=%s is_hdr_proc_ctx=%d\n",
-		hdr_entry->name, hdr_entry->is_hdr_proc_ctx);
-
-	entry = kmem_cache_zalloc(ipa3_ctx->hdr_proc_ctx_cache, GFP_KERNEL);
-	if (!entry) {
-		IPAERR("failed to alloc proc_ctx object\n");
-		return -ENOMEM;
-	}
-
-	INIT_LIST_HEAD(&entry->link);
-
-	entry->type = proc_ctx->type;
-	entry->hdr = hdr_entry;
-	entry->l2tp_params = proc_ctx->l2tp_params;
-	if (add_ref_hdr)
-		hdr_entry->ref_cnt++;
-	entry->cookie = IPA_PROC_HDR_COOKIE;
-	entry->ipacm_installed = user_only;
-
-	needed_len = ipahal_get_proc_ctx_needed_len(proc_ctx->type);
-
-	if (needed_len <= ipa_hdr_proc_ctx_bin_sz[IPA_HDR_PROC_CTX_BIN0]) {
-		bin = IPA_HDR_PROC_CTX_BIN0;
-	} else if (needed_len <=
-			ipa_hdr_proc_ctx_bin_sz[IPA_HDR_PROC_CTX_BIN1]) {
-		bin = IPA_HDR_PROC_CTX_BIN1;
-	} else {
-		IPAERR_RL("unexpected needed len %d\n", needed_len);
-		WARN_ON_RATELIMIT_IPA(1);
-		goto bad_len;
-	}
-
-	mem_size = (ipa3_ctx->hdr_proc_ctx_tbl_lcl) ?
-		IPA_MEM_PART(apps_hdr_proc_ctx_size) :
-		IPA_MEM_PART(apps_hdr_proc_ctx_size_ddr);
-	if (list_empty(&htbl->head_free_offset_list[bin])) {
-		if (htbl->end + ipa_hdr_proc_ctx_bin_sz[bin] > mem_size) {
-			IPAERR_RL("hdr proc ctx table overflow\n");
-			goto bad_len;
-		}
-
-		offset = kmem_cache_zalloc(ipa3_ctx->hdr_proc_ctx_offset_cache,
-					   GFP_KERNEL);
-		if (!offset) {
-			IPAERR("failed to alloc offset object\n");
-			goto bad_len;
-		}
-		INIT_LIST_HEAD(&offset->link);
-		/*
-		 * for a first item grow, set the bin and offset which are set
-		 * in stone
-		 */
-		offset->offset = htbl->end;
-		offset->bin = bin;
-		offset->ipacm_installed = user_only;
-		htbl->end += ipa_hdr_proc_ctx_bin_sz[bin];
-		list_add(&offset->link,
-				&htbl->head_offset_list[bin]);
-	} else {
-		/* get the first free slot */
-		offset =
-		    list_first_entry(&htbl->head_free_offset_list[bin],
-				struct ipa3_hdr_proc_ctx_offset_entry, link);
-		offset->ipacm_installed = user_only;
-		list_move(&offset->link, &htbl->head_offset_list[bin]);
-	}
-
-	entry->offset_entry = offset;
-	list_add(&entry->link, &htbl->head_proc_ctx_entry_list);
-	htbl->proc_ctx_cnt++;
-	IPADBG("add proc ctx of sz=%d cnt=%d ofst=%d\n", needed_len,
-			htbl->proc_ctx_cnt, offset->offset);
-
-	id = ipa3_id_alloc(entry);
-	if (id < 0) {
-		IPAERR_RL("failed to alloc id\n");
-		WARN_ON_RATELIMIT_IPA(1);
-		goto ipa_insert_failed;
-	}
-	entry->id = id;
-	proc_ctx->proc_ctx_hdl = id;
-	entry->ref_cnt++;
-
-	return 0;
-
-ipa_insert_failed:
-	list_move(&offset->link,
-		&htbl->head_free_offset_list[offset->bin]);
-	entry->offset_entry = NULL;
-	list_del(&entry->link);
-	htbl->proc_ctx_cnt--;
-
-bad_len:
-	if (add_ref_hdr)
-		hdr_entry->ref_cnt--;
-	entry->cookie = 0;
-	kmem_cache_free(ipa3_ctx->hdr_proc_ctx_cache, entry);
-	return -EPERM;
-}
-
-
-static int __ipa_add_hdr(struct ipa_hdr_add *hdr, bool user)
-{
-	struct ipa3_hdr_entry *entry;
-	struct ipa_hdr_offset_entry *offset = NULL;
-	u32 bin;
-	struct ipa3_hdr_tbl *htbl = &ipa3_ctx->hdr_tbl;
-	int id;
-	int mem_size;
-
-	if (hdr->hdr_len == 0 || hdr->hdr_len > IPA_HDR_MAX_SIZE) {
-		IPAERR_RL("bad param\n");
-		goto error;
-	}
-
-	if (!HDR_TYPE_IS_VALID(hdr->type)) {
-		IPAERR_RL("invalid hdr type %d\n", hdr->type);
-		goto error;
-	}
-
-	entry = kmem_cache_zalloc(ipa3_ctx->hdr_cache, GFP_KERNEL);
-	if (!entry)
-		goto error;
-
-	INIT_LIST_HEAD(&entry->link);
-
-	memcpy(entry->hdr, hdr->hdr, hdr->hdr_len);
-	entry->hdr_len = hdr->hdr_len;
-	strlcpy(entry->name, hdr->name, IPA_RESOURCE_NAME_MAX);
-	entry->is_partial = hdr->is_partial;
-	entry->type = hdr->type;
-	entry->is_eth2_ofst_valid = hdr->is_eth2_ofst_valid;
-	entry->eth2_ofst = hdr->eth2_ofst;
-	entry->cookie = IPA_HDR_COOKIE;
-	entry->ipacm_installed = user;
-
-	if (hdr->hdr_len <= ipa_hdr_bin_sz[IPA_HDR_BIN0])
-		bin = IPA_HDR_BIN0;
-	else if (hdr->hdr_len <= ipa_hdr_bin_sz[IPA_HDR_BIN1])
-		bin = IPA_HDR_BIN1;
-	else if (hdr->hdr_len <= ipa_hdr_bin_sz[IPA_HDR_BIN2])
-		bin = IPA_HDR_BIN2;
-	else if (hdr->hdr_len <= ipa_hdr_bin_sz[IPA_HDR_BIN3])
-		bin = IPA_HDR_BIN3;
-	else if (hdr->hdr_len <= ipa_hdr_bin_sz[IPA_HDR_BIN4])
-		bin = IPA_HDR_BIN4;
-	else {
-		IPAERR_RL("unexpected hdr len %d\n", hdr->hdr_len);
-		goto bad_hdr_len;
-	}
-
-	mem_size = (ipa3_ctx->hdr_tbl_lcl) ? IPA_MEM_PART(apps_hdr_size) :
-		IPA_MEM_PART(apps_hdr_size_ddr);
-
-	if (list_empty(&htbl->head_free_offset_list[bin])) {
-		/* if header does not fit to table, place it in DDR */
-		if (htbl->end + ipa_hdr_bin_sz[bin] > mem_size) {
-			entry->is_hdr_proc_ctx = true;
-			entry->phys_base = dma_map_single(ipa3_ctx->pdev,
-				entry->hdr,
-				entry->hdr_len,
-				DMA_TO_DEVICE);
-			if (dma_mapping_error(ipa3_ctx->pdev,
-				entry->phys_base)) {
-				IPAERR("dma_map_single failure for entry\n");
-				goto fail_dma_mapping;
-			}
-		} else {
-			entry->is_hdr_proc_ctx = false;
-			offset = kmem_cache_zalloc(ipa3_ctx->hdr_offset_cache,
-						   GFP_KERNEL);
-			if (!offset) {
-				IPAERR("failed to alloc hdr offset object\n");
-				goto bad_hdr_len;
-			}
-			INIT_LIST_HEAD(&offset->link);
-			/*
-			 * for a first item grow, set the bin and offset which
-			 * are set in stone
-			 */
-			offset->offset = htbl->end;
-			offset->bin = bin;
-			htbl->end += ipa_hdr_bin_sz[bin];
-			list_add(&offset->link,
-					&htbl->head_offset_list[bin]);
-			entry->offset_entry = offset;
-			offset->ipacm_installed = user;
-		}
-	} else {
-		entry->is_hdr_proc_ctx = false;
-		/* get the first free slot */
-		offset = list_first_entry(&htbl->head_free_offset_list[bin],
-			struct ipa_hdr_offset_entry, link);
-		list_move(&offset->link, &htbl->head_offset_list[bin]);
-		entry->offset_entry = offset;
-		offset->ipacm_installed = user;
-	}
-
-	list_add(&entry->link, &htbl->head_hdr_entry_list);
-	htbl->hdr_cnt++;
-	if (entry->is_hdr_proc_ctx)
-		IPADBG("add hdr of sz=%d hdr_cnt=%d phys_base=%pa\n",
-			hdr->hdr_len,
-			htbl->hdr_cnt,
-			&entry->phys_base);
-	else
-		IPADBG("add hdr of sz=%d hdr_cnt=%d ofst=%d\n",
-			hdr->hdr_len,
-			htbl->hdr_cnt,
-			entry->offset_entry->offset);
-
-	id = ipa3_id_alloc(entry);
-	if (id < 0) {
-		IPAERR_RL("failed to alloc id\n");
-		WARN_ON_RATELIMIT_IPA(1);
-		goto ipa_insert_failed;
-	}
-	entry->id = id;
-	hdr->hdr_hdl = id;
-	entry->ref_cnt++;
-
-	if (entry->is_hdr_proc_ctx) {
-		struct ipa_hdr_proc_ctx_add proc_ctx;
-
-		IPADBG("adding processing context for header %s\n", hdr->name);
-		proc_ctx.type = IPA_HDR_PROC_NONE;
-		proc_ctx.hdr_hdl = id;
-		if (__ipa_add_hdr_proc_ctx(&proc_ctx, false, user)) {
-			IPAERR("failed to add hdr proc ctx\n");
-			goto fail_add_proc_ctx;
-		}
-		entry->proc_ctx = ipa3_id_find(proc_ctx.proc_ctx_hdl);
-	}
-
-	return 0;
-
-fail_add_proc_ctx:
-	entry->ref_cnt--;
-	hdr->hdr_hdl = 0;
-	ipa3_id_remove(id);
-ipa_insert_failed:
-	if (entry->is_hdr_proc_ctx) {
-		dma_unmap_single(ipa3_ctx->pdev, entry->phys_base,
-			entry->hdr_len, DMA_TO_DEVICE);
-	} else {
-		if (offset)
-			list_move(&offset->link,
-			&htbl->head_free_offset_list[offset->bin]);
-		entry->offset_entry = NULL;
-	}
-	htbl->hdr_cnt--;
-	list_del(&entry->link);
-
-fail_dma_mapping:
-	entry->is_hdr_proc_ctx = false;
-
-bad_hdr_len:
-	entry->cookie = 0;
-	kmem_cache_free(ipa3_ctx->hdr_cache, entry);
-error:
-	return -EPERM;
-}
-
-static int __ipa3_del_hdr_proc_ctx(u32 proc_ctx_hdl,
-	bool release_hdr, bool by_user)
-{
-	struct ipa3_hdr_proc_ctx_entry *entry;
-	struct ipa3_hdr_proc_ctx_tbl *htbl = &ipa3_ctx->hdr_proc_ctx_tbl;
-
-	entry = ipa3_id_find(proc_ctx_hdl);
-	if (!entry || (entry->cookie != IPA_PROC_HDR_COOKIE)) {
-		IPAERR_RL("bad param\n");
-		return -EINVAL;
-	}
-
-	IPADBG("del proc ctx cnt=%d ofst=%d\n",
-		htbl->proc_ctx_cnt, entry->offset_entry->offset);
-
-	if (by_user && entry->user_deleted) {
-		IPAERR_RL("proc_ctx already deleted by user\n");
-		return -EINVAL;
-	}
-
-	if (by_user)
-		entry->user_deleted = true;
-
-	if (--entry->ref_cnt) {
-		IPADBG("proc_ctx_hdl %x ref_cnt %d\n",
-			proc_ctx_hdl, entry->ref_cnt);
-		return 0;
-	}
-
-	if (release_hdr)
-		__ipa3_del_hdr(entry->hdr->id, false);
-
-	/* move the offset entry to appropriate free list */
-	list_move(&entry->offset_entry->link,
-		&htbl->head_free_offset_list[entry->offset_entry->bin]);
-	list_del(&entry->link);
-	htbl->proc_ctx_cnt--;
-	entry->cookie = 0;
-	kmem_cache_free(ipa3_ctx->hdr_proc_ctx_cache, entry);
-
-	/* remove the handle from the database */
-	ipa3_id_remove(proc_ctx_hdl);
-
-	return 0;
-}
-
-int __ipa3_del_hdr(u32 hdr_hdl, bool by_user)
-{
-	struct ipa3_hdr_entry *entry;
-	struct ipa3_hdr_tbl *htbl = &ipa3_ctx->hdr_tbl;
-
-	entry = ipa3_id_find(hdr_hdl);
-	if (entry == NULL) {
-		IPAERR_RL("lookup failed\n");
-		return -EINVAL;
-	}
-
-	if (entry->cookie != IPA_HDR_COOKIE) {
-		IPAERR_RL("bad parm\n");
-		return -EINVAL;
-	}
-
-	if (entry->is_hdr_proc_ctx)
-		IPADBG("del hdr of len=%d hdr_cnt=%d phys_base=%pa\n",
-			entry->hdr_len, htbl->hdr_cnt, &entry->phys_base);
-	else
-		IPADBG("del hdr of len=%d hdr_cnt=%d ofst=%d\n",
-			entry->hdr_len, htbl->hdr_cnt,
-			entry->offset_entry->offset);
-
-	if (by_user && entry->user_deleted) {
-		IPAERR_RL("proc_ctx already deleted by user\n");
-		return -EINVAL;
-	}
-
-	if (by_user) {
-		if (!strcmp(entry->name, IPA_LAN_RX_HDR_NAME)) {
-			IPADBG("Trying to delete hdr %s offset=%u\n",
-				entry->name, entry->offset_entry->offset);
-			if (!entry->offset_entry->offset) {
-				IPAERR_RL(
-				"User cannot delete default header\n");
-				return -EPERM;
-			}
-		}
-		entry->user_deleted = true;
-	}
-
-	if (--entry->ref_cnt) {
-		IPADBG("hdr_hdl %x ref_cnt %d\n", hdr_hdl, entry->ref_cnt);
-		return 0;
-	}
-
-	if (entry->is_hdr_proc_ctx) {
-		dma_unmap_single(ipa3_ctx->pdev,
-			entry->phys_base,
-			entry->hdr_len,
-			DMA_TO_DEVICE);
-		__ipa3_del_hdr_proc_ctx(entry->proc_ctx->id, false, false);
-	} else {
-		/* move the offset entry to appropriate free list */
-		list_move(&entry->offset_entry->link,
-			&htbl->head_free_offset_list[entry->offset_entry->bin]);
-	}
-	list_del(&entry->link);
-	htbl->hdr_cnt--;
-	entry->cookie = 0;
-	kmem_cache_free(ipa3_ctx->hdr_cache, entry);
-
-	/* remove the handle from the database */
-	ipa3_id_remove(hdr_hdl);
-
-	return 0;
-}
-
-/**
- * ipa3_add_hdr() - add the specified headers to SW and optionally commit them
- * to IPA HW
- * @hdrs:	[inout] set of headers to add
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_add_hdr(struct ipa_ioc_add_hdr *hdrs)
-{
-	return ipa3_add_hdr_usr(hdrs, false);
-}
-
-/**
- * ipa3_add_hdr_usr() - add the specified headers to SW
- * and optionally commit them to IPA HW
- * @hdrs:		[inout] set of headers to add
- * @user_only:	[in] indicate installed from user
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_add_hdr_usr(struct ipa_ioc_add_hdr *hdrs, bool user_only)
-{
-	int i;
-	int result = -EFAULT;
-
-	if (hdrs == NULL || hdrs->num_hdrs == 0) {
-		IPAERR_RL("bad parm\n");
-		return -EINVAL;
-	}
-
-	mutex_lock(&ipa3_ctx->lock);
-	IPADBG("adding %d headers to IPA driver internal data struct\n",
-			hdrs->num_hdrs);
-	for (i = 0; i < hdrs->num_hdrs; i++) {
-		if (__ipa_add_hdr(&hdrs->hdr[i], user_only)) {
-			IPAERR_RL("failed to add hdr %d\n", i);
-			hdrs->hdr[i].status = -1;
-		} else {
-			hdrs->hdr[i].status = 0;
-		}
-	}
-
-	if (hdrs->commit) {
-		IPADBG("committing all headers to IPA core");
-		if (ipa3_ctx->ctrl->ipa3_commit_hdr()) {
-			result = -EPERM;
-			goto bail;
-		}
-	}
-	result = 0;
-bail:
-	mutex_unlock(&ipa3_ctx->lock);
-	return result;
-}
-
-/**
- * ipa3_del_hdr_by_user() - Remove the specified headers
- * from SW and optionally commit them to IPA HW
- * @hdls:	[inout] set of headers to delete
- * @by_user:	Operation requested by user?
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_del_hdr_by_user(struct ipa_ioc_del_hdr *hdls, bool by_user)
-{
-	int i;
-	int result = -EFAULT;
-
-	if (hdls == NULL || hdls->num_hdls == 0) {
-		IPAERR_RL("bad parm\n");
-		return -EINVAL;
-	}
-
-	mutex_lock(&ipa3_ctx->lock);
-	for (i = 0; i < hdls->num_hdls; i++) {
-		if (__ipa3_del_hdr(hdls->hdl[i].hdl, by_user)) {
-			IPAERR_RL("failed to del hdr %i\n", i);
-			hdls->hdl[i].status = -1;
-		} else {
-			hdls->hdl[i].status = 0;
-		}
-	}
-
-	if (hdls->commit) {
-		if (ipa3_ctx->ctrl->ipa3_commit_hdr()) {
-			result = -EPERM;
-			goto bail;
-		}
-	}
-	result = 0;
-bail:
-	mutex_unlock(&ipa3_ctx->lock);
-	return result;
-}
-
-/**
- * ipa3_del_hdr() - Remove the specified headers from SW
- * and optionally commit them to IPA HW
- * @hdls:	[inout] set of headers to delete
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_del_hdr(struct ipa_ioc_del_hdr *hdls)
-{
-	return ipa3_del_hdr_by_user(hdls, false);
-}
-
-/**
- * ipa3_add_hdr_proc_ctx() - add the specified headers to SW
- * and optionally commit them to IPA HW
- * @proc_ctxs:	[inout] set of processing context headers to add
- * @user_only:	[in] indicate installed by user-space module
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_add_hdr_proc_ctx(struct ipa_ioc_add_hdr_proc_ctx *proc_ctxs,
-							bool user_only)
-{
-	int i;
-	int result = -EFAULT;
-
-	if (proc_ctxs == NULL || proc_ctxs->num_proc_ctxs == 0) {
-		IPAERR_RL("bad parm\n");
-		return -EINVAL;
-	}
-
-	mutex_lock(&ipa3_ctx->lock);
-	IPADBG("adding %d header processing contextes to IPA driver\n",
-			proc_ctxs->num_proc_ctxs);
-	for (i = 0; i < proc_ctxs->num_proc_ctxs; i++) {
-		if (__ipa_add_hdr_proc_ctx(&proc_ctxs->proc_ctx[i],
-				true, user_only)) {
-			IPAERR_RL("failed to add hdr pric ctx %d\n", i);
-			proc_ctxs->proc_ctx[i].status = -1;
-		} else {
-			proc_ctxs->proc_ctx[i].status = 0;
-		}
-	}
-
-	if (proc_ctxs->commit) {
-		IPADBG("committing all headers to IPA core");
-		if (ipa3_ctx->ctrl->ipa3_commit_hdr()) {
-			result = -EPERM;
-			goto bail;
-		}
-	}
-	result = 0;
-bail:
-	mutex_unlock(&ipa3_ctx->lock);
-	return result;
-}
-
-/**
- * ipa3_del_hdr_proc_ctx_by_user() -
- * Remove the specified processing context headers from SW and
- * optionally commit them to IPA HW.
- * @hdls:	[inout] set of processing context headers to delete
- * @by_user:	Operation requested by user?
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_del_hdr_proc_ctx_by_user(struct ipa_ioc_del_hdr_proc_ctx *hdls,
-	bool by_user)
-{
-	int i;
-	int result;
-
-	if (hdls == NULL || hdls->num_hdls == 0) {
-		IPAERR_RL("bad parm\n");
-		return -EINVAL;
-	}
-
-	mutex_lock(&ipa3_ctx->lock);
-	for (i = 0; i < hdls->num_hdls; i++) {
-		if (__ipa3_del_hdr_proc_ctx(hdls->hdl[i].hdl, true, by_user)) {
-			IPAERR_RL("failed to del hdr %i\n", i);
-			hdls->hdl[i].status = -1;
-		} else {
-			hdls->hdl[i].status = 0;
-		}
-	}
-
-	if (hdls->commit) {
-		if (ipa3_ctx->ctrl->ipa3_commit_hdr()) {
-			result = -EPERM;
-			goto bail;
-		}
-	}
-	result = 0;
-bail:
-	mutex_unlock(&ipa3_ctx->lock);
-	return result;
-}
-
-/**
- * ipa3_del_hdr_proc_ctx() -
- * Remove the specified processing context headers from SW and
- * optionally commit them to IPA HW.
- * @hdls:	[inout] set of processing context headers to delete
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_del_hdr_proc_ctx(struct ipa_ioc_del_hdr_proc_ctx *hdls)
-{
-	return ipa3_del_hdr_proc_ctx_by_user(hdls, false);
-}
-
-/**
- * ipa3_commit_hdr() - commit to IPA HW the current header table in SW
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_commit_hdr(void)
-{
-	int result = -EFAULT;
-
-	/*
-	 * issue a commit on the routing module since routing rules point to
-	 * header table entries
-	 */
-	if (ipa3_commit_rt(IPA_IP_v4))
-		return -EPERM;
-	if (ipa3_commit_rt(IPA_IP_v6))
-		return -EPERM;
-
-	mutex_lock(&ipa3_ctx->lock);
-	if (ipa3_ctx->ctrl->ipa3_commit_hdr()) {
-		result = -EPERM;
-		goto bail;
-	}
-	result = 0;
-bail:
-	mutex_unlock(&ipa3_ctx->lock);
-	return result;
-}
-
-/**
- * ipa3_reset_hdr() - reset the current header table in SW (does not commit to
- * HW)
- *
- * @user_only:	[in] indicate delete rules installed by userspace
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_reset_hdr(bool user_only)
-{
-	struct ipa3_hdr_entry *entry;
-	struct ipa3_hdr_entry *next;
-	struct ipa3_hdr_proc_ctx_entry *ctx_entry;
-	struct ipa3_hdr_proc_ctx_entry *ctx_next;
-	struct ipa_hdr_offset_entry *off_entry;
-	struct ipa_hdr_offset_entry *off_next;
-	struct ipa3_hdr_proc_ctx_offset_entry *ctx_off_entry;
-	struct ipa3_hdr_proc_ctx_offset_entry *ctx_off_next;
-	struct ipa3_hdr_tbl *htbl = &ipa3_ctx->hdr_tbl;
-	struct ipa3_hdr_proc_ctx_tbl *htbl_proc = &ipa3_ctx->hdr_proc_ctx_tbl;
-	int i;
-
-	/*
-	 * issue a reset on the routing module since routing rules point to
-	 * header table entries
-	 */
-	if (ipa3_reset_rt(IPA_IP_v4, user_only))
-		IPAERR_RL("fail to reset v4 rt\n");
-	if (ipa3_reset_rt(IPA_IP_v6, user_only))
-		IPAERR_RL("fail to reset v6 rt\n");
-
-	mutex_lock(&ipa3_ctx->lock);
-	IPADBG("reset hdr\n");
-	list_for_each_entry_safe(entry, next,
-			&ipa3_ctx->hdr_tbl.head_hdr_entry_list, link) {
-
-		/* do not remove the default header */
-		if (!strcmp(entry->name, IPA_LAN_RX_HDR_NAME)) {
-			IPADBG("Trying to remove hdr %s offset=%u\n",
-				entry->name, entry->offset_entry->offset);
-			if (!entry->offset_entry->offset) {
-				if (entry->is_hdr_proc_ctx) {
-					IPAERR("default header is proc ctx\n");
-					mutex_unlock(&ipa3_ctx->lock);
-					WARN_ON_RATELIMIT_IPA(1);
-					return -EFAULT;
-				}
-				IPADBG("skip default header\n");
-				continue;
-			}
-		}
-
-		if (ipa3_id_find(entry->id) == NULL) {
-			mutex_unlock(&ipa3_ctx->lock);
-			WARN_ON_RATELIMIT_IPA(1);
-			return -EFAULT;
-		}
-
-		if (!user_only || entry->ipacm_installed) {
-			if (entry->is_hdr_proc_ctx) {
-				dma_unmap_single(ipa3_ctx->pdev,
-					entry->phys_base,
-					entry->hdr_len,
-					DMA_TO_DEVICE);
-				entry->proc_ctx = NULL;
-			} else {
-				/* move the offset entry to free list */
-				entry->offset_entry->ipacm_installed = false;
-				list_move(&entry->offset_entry->link,
-				&htbl->head_free_offset_list[
-					entry->offset_entry->bin]);
-			}
-			list_del(&entry->link);
-			htbl->hdr_cnt--;
-			entry->ref_cnt = 0;
-			entry->cookie = 0;
-
-			/* remove the handle from the database */
-			ipa3_id_remove(entry->id);
-			kmem_cache_free(ipa3_ctx->hdr_cache, entry);
-		}
-	}
-
-	/* only clean up offset_list and free_offset_list on global reset */
-	if (!user_only) {
-		for (i = 0; i < IPA_HDR_BIN_MAX; i++) {
-			list_for_each_entry_safe(off_entry, off_next,
-					 &ipa3_ctx->hdr_tbl.head_offset_list[i],
-					 link) {
-				/**
-				 * do not remove the default exception
-				 * header which is at offset 0
-				 */
-				if (off_entry->offset == 0)
-					continue;
-				list_del(&off_entry->link);
-				kmem_cache_free(ipa3_ctx->hdr_offset_cache,
-					off_entry);
-			}
-			list_for_each_entry_safe(off_entry, off_next,
-				&ipa3_ctx->hdr_tbl.head_free_offset_list[i],
-				link) {
-				list_del(&off_entry->link);
-				kmem_cache_free(ipa3_ctx->hdr_offset_cache,
-					off_entry);
-			}
-		}
-		/* there is one header of size 8 */
-		ipa3_ctx->hdr_tbl.end = 8;
-		ipa3_ctx->hdr_tbl.hdr_cnt = 1;
-	}
-
-	IPADBG("reset hdr proc ctx\n");
-	list_for_each_entry_safe(
-		ctx_entry,
-		ctx_next,
-		&(htbl_proc->head_proc_ctx_entry_list),
-		link) {
-
-		if (ipa3_id_find(ctx_entry->id) == NULL) {
-			mutex_unlock(&ipa3_ctx->lock);
-			WARN_ON_RATELIMIT_IPA(1);
-			return -EFAULT;
-		}
-
-		if (!user_only ||
-				ctx_entry->ipacm_installed) {
-			/* move the offset entry to appropriate free list */
-			list_move(&ctx_entry->offset_entry->link,
-				&htbl_proc->head_free_offset_list[
-					ctx_entry->offset_entry->bin]);
-			list_del(&ctx_entry->link);
-			htbl_proc->proc_ctx_cnt--;
-			ctx_entry->ref_cnt = 0;
-			ctx_entry->cookie = 0;
-
-			/* remove the handle from the database */
-			ipa3_id_remove(ctx_entry->id);
-			kmem_cache_free(ipa3_ctx->hdr_proc_ctx_cache,
-				ctx_entry);
-		}
-	}
-	/* only clean up offset_list and free_offset_list on global reset */
-	if (!user_only) {
-		for (i = 0; i < IPA_HDR_PROC_CTX_BIN_MAX; i++) {
-			list_for_each_entry_safe(ctx_off_entry, ctx_off_next,
-				&(htbl_proc->head_offset_list[i]), link) {
-				list_del(&ctx_off_entry->link);
-				kmem_cache_free(
-					ipa3_ctx->hdr_proc_ctx_offset_cache,
-					ctx_off_entry);
-			}
-			list_for_each_entry_safe(ctx_off_entry, ctx_off_next,
-				&(htbl_proc->head_free_offset_list[i]), link) {
-				list_del(&ctx_off_entry->link);
-				kmem_cache_free(
-					ipa3_ctx->hdr_proc_ctx_offset_cache,
-					ctx_off_entry);
-			}
-		}
-		htbl_proc->end = 0;
-		htbl_proc->proc_ctx_cnt = 0;
-	}
-
-	/* commit the change to IPA-HW */
-	if (ipa3_ctx->ctrl->ipa3_commit_hdr()) {
-		IPAERR("fail to commit hdr\n");
-		WARN_ON_RATELIMIT_IPA(1);
-		mutex_unlock(&ipa3_ctx->lock);
-		return -EFAULT;
-	}
-
-	mutex_unlock(&ipa3_ctx->lock);
-	return 0;
-}
-
-static struct ipa3_hdr_entry *__ipa_find_hdr(const char *name)
-{
-	struct ipa3_hdr_entry *entry;
-
-	if (strnlen(name, IPA_RESOURCE_NAME_MAX) == IPA_RESOURCE_NAME_MAX) {
-		IPAERR_RL("Header name too long: %s\n", name);
-		return NULL;
-	}
-
-	list_for_each_entry(entry, &ipa3_ctx->hdr_tbl.head_hdr_entry_list,
-			link) {
-		if (!strcmp(name, entry->name))
-			return entry;
-	}
-
-	return NULL;
-}
-
-/**
- * ipa3_get_hdr() - Lookup the specified header resource
- * @lookup:	[inout] header to lookup and its handle
- *
- * lookup the specified header resource and return handle if it exists
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- *		Caller should call ipa3_put_hdr later if this function succeeds
- */
-int ipa3_get_hdr(struct ipa_ioc_get_hdr *lookup)
-{
-	struct ipa3_hdr_entry *entry;
-	int result = -1;
-
-	if (lookup == NULL) {
-		IPAERR_RL("bad parm\n");
-		return -EINVAL;
-	}
-	mutex_lock(&ipa3_ctx->lock);
-	lookup->name[IPA_RESOURCE_NAME_MAX-1] = '\0';
-	entry = __ipa_find_hdr(lookup->name);
-	if (entry) {
-		lookup->hdl = entry->id;
-		result = 0;
-	}
-	mutex_unlock(&ipa3_ctx->lock);
-
-	return result;
-}
-
-/**
- * __ipa3_release_hdr() - drop reference to header and cause
- * deletion if reference count permits
- * @hdr_hdl:	[in] handle of header to be released
- *
- * Returns:	0 on success, negative on failure
- */
-int __ipa3_release_hdr(u32 hdr_hdl)
-{
-	int result = 0;
-
-	if (__ipa3_del_hdr(hdr_hdl, false)) {
-		IPADBG("fail to del hdr %x\n", hdr_hdl);
-		result = -EFAULT;
-		goto bail;
-	}
-
-	/* commit for put */
-	if (ipa3_ctx->ctrl->ipa3_commit_hdr()) {
-		IPAERR("fail to commit hdr\n");
-		result = -EFAULT;
-		goto bail;
-	}
-
-bail:
-	return result;
-}
-
-/**
- * __ipa3_release_hdr_proc_ctx() - drop reference to processing context
- *  and cause deletion if reference count permits
- * @proc_ctx_hdl:	[in] handle of processing context to be released
- *
- * Returns:	0 on success, negative on failure
- */
-int __ipa3_release_hdr_proc_ctx(u32 proc_ctx_hdl)
-{
-	int result = 0;
-
-	if (__ipa3_del_hdr_proc_ctx(proc_ctx_hdl, true, false)) {
-		IPADBG("fail to del hdr %x\n", proc_ctx_hdl);
-		result = -EFAULT;
-		goto bail;
-	}
-
-	/* commit for put */
-	if (ipa3_ctx->ctrl->ipa3_commit_hdr()) {
-		IPAERR("fail to commit hdr\n");
-		result = -EFAULT;
-		goto bail;
-	}
-
-bail:
-	return result;
-}
-
-/**
- * ipa3_put_hdr() - Release the specified header handle
- * @hdr_hdl:	[in] the header handle to release
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_put_hdr(u32 hdr_hdl)
-{
-	struct ipa3_hdr_entry *entry;
-	int result = -EFAULT;
-
-	mutex_lock(&ipa3_ctx->lock);
-
-	entry = ipa3_id_find(hdr_hdl);
-	if (entry == NULL) {
-		IPAERR_RL("lookup failed\n");
-		result = -EINVAL;
-		goto bail;
-	}
-
-	if (entry->cookie != IPA_HDR_COOKIE) {
-		IPAERR_RL("invalid header entry\n");
-		result = -EINVAL;
-		goto bail;
-	}
-
-	result = 0;
-bail:
-	mutex_unlock(&ipa3_ctx->lock);
-	return result;
-}
-
-/**
- * ipa3_copy_hdr() - Lookup the specified header resource and return a copy of
- * it
- * @copy:	[inout] header to lookup and its copy
- *
- * lookup the specified header resource and return a copy of it (along with its
- * attributes) if it exists, this would be called for partial headers
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_copy_hdr(struct ipa_ioc_copy_hdr *copy)
-{
-	struct ipa3_hdr_entry *entry;
-	int result = -EFAULT;
-
-	if (copy == NULL) {
-		IPAERR_RL("bad parm\n");
-		return -EINVAL;
-	}
-	mutex_lock(&ipa3_ctx->lock);
-	copy->name[IPA_RESOURCE_NAME_MAX-1] = '\0';
-	entry = __ipa_find_hdr(copy->name);
-	if (entry) {
-		memcpy(copy->hdr, entry->hdr, entry->hdr_len);
-		copy->hdr_len = entry->hdr_len;
-		copy->type = entry->type;
-		copy->is_partial = entry->is_partial;
-		copy->is_eth2_ofst_valid = entry->is_eth2_ofst_valid;
-		copy->eth2_ofst = entry->eth2_ofst;
-		result = 0;
-	}
-	mutex_unlock(&ipa3_ctx->lock);
-
-	return result;
-}

ipa/ipa_v3/ipa_hw_stats.c

@@ -1,2027 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2017-2019, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/debugfs.h>
-#include <linux/kernel.h>
-#include <linux/delay.h>
-#include "ipa_i.h"
-#include "ipahal/ipahal.h"
-#include "ipahal/ipahal_hw_stats.h"
-
-#define IPA_CLIENT_BIT_32(client) \
-	((ipa3_get_ep_mapping(client) >= 0 && \
-		ipa3_get_ep_mapping(client) < IPA_STATS_MAX_PIPE_BIT) ? \
-		(1 << ipa3_get_ep_mapping(client)) : 0)
-
-int ipa_hw_stats_init(void)
-{
-	int ret = 0, ep_index;
-	struct ipa_teth_stats_endpoints *teth_stats_init;
-
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_0)
-		return 0;
-
-	/* initialize stats here */
-	ipa3_ctx->hw_stats.enabled = true;
-
-	teth_stats_init = kzalloc(sizeof(*teth_stats_init), GFP_KERNEL);
-	if (!teth_stats_init) {
-		IPAERR("mem allocated failed!\n");
-		return -ENOMEM;
-	}
-	/* enable prod mask */
-	if (ipa3_ctx->platform_type == IPA_PLAT_TYPE_APQ) {
-		teth_stats_init->prod_mask = (
-			IPA_CLIENT_BIT_32(IPA_CLIENT_MHI_PRIME_TETH_PROD) |
-			IPA_CLIENT_BIT_32(IPA_CLIENT_USB_PROD));
-		if (ipa3_ctx->ipa_hw_type == IPA_HW_v4_5)
-			teth_stats_init->prod_mask |=
-			IPA_CLIENT_BIT_32(IPA_CLIENT_WLAN2_PROD);
-		else
-			teth_stats_init->prod_mask |=
-			IPA_CLIENT_BIT_32(IPA_CLIENT_WLAN1_PROD);
-
-		if (IPA_CLIENT_BIT_32(IPA_CLIENT_MHI_PRIME_TETH_PROD)) {
-			ep_index = ipa3_get_ep_mapping(
-				IPA_CLIENT_MHI_PRIME_TETH_PROD);
-			if (ep_index == -1) {
-				IPAERR("Invalid client.\n");
-				kfree(teth_stats_init);
-				return -EINVAL;
-			}
-			teth_stats_init->dst_ep_mask[ep_index] =
-				IPA_CLIENT_BIT_32(IPA_CLIENT_USB_CONS);
-
-			if (ipa3_ctx->ipa_hw_type == IPA_HW_v4_5)
-				teth_stats_init->dst_ep_mask[ep_index] |=
-				IPA_CLIENT_BIT_32(IPA_CLIENT_WLAN2_CONS);
-			else
-				teth_stats_init->dst_ep_mask[ep_index] |=
-				IPA_CLIENT_BIT_32(IPA_CLIENT_WLAN1_CONS);
-		}
-	} else {
-		teth_stats_init->prod_mask = (
-			IPA_CLIENT_BIT_32(IPA_CLIENT_Q6_WAN_PROD) |
-			IPA_CLIENT_BIT_32(IPA_CLIENT_USB_PROD));
-
-		if (ipa3_ctx->ipa_hw_type == IPA_HW_v4_5)
-			teth_stats_init->prod_mask |=
-			IPA_CLIENT_BIT_32(IPA_CLIENT_WLAN2_PROD);
-		else
-			teth_stats_init->prod_mask |=
-			IPA_CLIENT_BIT_32(IPA_CLIENT_WLAN1_PROD);
-
-		if (IPA_CLIENT_BIT_32(IPA_CLIENT_Q6_WAN_PROD)) {
-			ep_index = ipa3_get_ep_mapping(IPA_CLIENT_Q6_WAN_PROD);
-			if (ep_index == -1) {
-				IPAERR("Invalid client.\n");
-				kfree(teth_stats_init);
-				return -EINVAL;
-			}
-			teth_stats_init->dst_ep_mask[ep_index] =
-			IPA_CLIENT_BIT_32(IPA_CLIENT_USB_CONS);
-
-			if (ipa3_ctx->ipa_hw_type == IPA_HW_v4_5)
-				teth_stats_init->dst_ep_mask[ep_index] |=
-				IPA_CLIENT_BIT_32(IPA_CLIENT_WLAN2_CONS);
-			else
-				teth_stats_init->dst_ep_mask[ep_index] |=
-				IPA_CLIENT_BIT_32(IPA_CLIENT_WLAN1_CONS);
-		}
-	}
-
-	if (IPA_CLIENT_BIT_32(IPA_CLIENT_USB_PROD)) {
-		ep_index = ipa3_get_ep_mapping(IPA_CLIENT_USB_PROD);
-		if (ep_index == -1) {
-			IPAERR("Invalid client.\n");
-			kfree(teth_stats_init);
-			return -EINVAL;
-		}
-		/* enable addtional pipe monitoring for pcie modem */
-		if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_1)
-			teth_stats_init->dst_ep_mask[ep_index] =
-				(IPA_CLIENT_BIT_32(
-					IPA_CLIENT_Q6_WAN_CONS) |
-				IPA_CLIENT_BIT_32(
-					IPA_CLIENT_MHI_PRIME_TETH_CONS));
-		else
-			teth_stats_init->dst_ep_mask[ep_index] =
-				IPA_CLIENT_BIT_32(IPA_CLIENT_Q6_WAN_CONS);
-	}
-
-	if (IPA_CLIENT_BIT_32(IPA_CLIENT_WLAN1_PROD)) {
-		ep_index = ipa3_get_ep_mapping(IPA_CLIENT_WLAN1_PROD);
-		if (ep_index == -1) {
-			IPAERR("Invalid client.\n");
-			kfree(teth_stats_init);
-			return -EINVAL;
-		}
-		/* enable addtional pipe monitoring for pcie modem*/
-		if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_1)
-			teth_stats_init->dst_ep_mask[ep_index] =
-				(IPA_CLIENT_BIT_32(IPA_CLIENT_Q6_WAN_CONS) |
-				IPA_CLIENT_BIT_32(
-					IPA_CLIENT_MHI_PRIME_TETH_CONS));
-		else
-			teth_stats_init->dst_ep_mask[ep_index] =
-				IPA_CLIENT_BIT_32(IPA_CLIENT_Q6_WAN_CONS);
-	}
-
-	if (IPA_CLIENT_BIT_32(IPA_CLIENT_WLAN2_PROD)) {
-		ep_index = ipa3_get_ep_mapping(IPA_CLIENT_WLAN2_PROD);
-		if (ep_index == -1) {
-			IPAERR("Invalid client.\n");
-			kfree(teth_stats_init);
-			return -EINVAL;
-		}
-		/* enable addtional pipe monitoring for pcie modem*/
-		if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_1)
-			teth_stats_init->dst_ep_mask[ep_index] =
-				(IPA_CLIENT_BIT_32(IPA_CLIENT_Q6_WAN_CONS) |
-				IPA_CLIENT_BIT_32(
-					IPA_CLIENT_MHI_PRIME_TETH_CONS));
-		else
-			teth_stats_init->dst_ep_mask[ep_index] =
-				IPA_CLIENT_BIT_32(IPA_CLIENT_Q6_WAN_CONS);
-	}
-
-
-	ret = ipa_init_teth_stats(teth_stats_init);
-	if (ret != 0)
-		IPAERR("init teth stats fails\n");
-	kfree(teth_stats_init);
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_5) {
-		ret = ipa_init_flt_rt_stats();
-		if (ret != 0)
-			IPAERR("init flt rt stats fails\n");
-	}
-	return ret;
-}
-
-int ipa_init_quota_stats(u32 pipe_bitmask)
-{
-	struct ipahal_stats_init_pyld *pyld;
-	struct ipahal_imm_cmd_dma_shared_mem cmd = { 0 };
-	struct ipahal_imm_cmd_pyld *cmd_pyld;
-	struct ipahal_imm_cmd_register_write quota_base = {0};
-	struct ipahal_imm_cmd_pyld *quota_base_pyld;
-	struct ipahal_imm_cmd_register_write quota_mask = {0};
-	struct ipahal_imm_cmd_pyld *quota_mask_pyld;
-	struct ipa3_desc desc[3] = { {0} };
-	dma_addr_t dma_address;
-	int ret;
-
-	if (!ipa3_ctx->hw_stats.enabled)
-		return 0;
-
-	/* reset driver's cache */
-	memset(&ipa3_ctx->hw_stats.quota, 0, sizeof(ipa3_ctx->hw_stats.quota));
-	ipa3_ctx->hw_stats.quota.init.enabled_bitmask = pipe_bitmask;
-	IPADBG_LOW("pipe_bitmask=0x%x\n", pipe_bitmask);
-
-	pyld = ipahal_stats_generate_init_pyld(IPAHAL_HW_STATS_QUOTA,
-		&ipa3_ctx->hw_stats.quota.init, false);
-	if (!pyld) {
-		IPAERR("failed to generate pyld\n");
-		return -EPERM;
-	}
-
-	if (pyld->len > IPA_MEM_PART(stats_quota_size)) {
-		IPAERR("SRAM partition too small: %d needed %d\n",
-			IPA_MEM_PART(stats_quota_size), pyld->len);
-		ret = -EPERM;
-		goto destroy_init_pyld;
-	}
-
-	dma_address = dma_map_single(ipa3_ctx->pdev,
-		pyld->data,
-		pyld->len,
-		DMA_TO_DEVICE);
-	if (dma_mapping_error(ipa3_ctx->pdev, dma_address)) {
-		IPAERR("failed to DMA map\n");
-		ret = -EPERM;
-		goto destroy_init_pyld;
-	}
-
-	/* setting the registers and init the stats pyld are done atomically */
-	quota_mask.skip_pipeline_clear = false;
-	quota_mask.pipeline_clear_options = IPAHAL_FULL_PIPELINE_CLEAR;
-	quota_mask.offset = ipahal_get_reg_n_ofst(IPA_STAT_QUOTA_MASK_n,
-		ipa3_ctx->ee);
-	quota_mask.value = pipe_bitmask;
-	quota_mask.value_mask = ~0;
-	quota_mask_pyld = ipahal_construct_imm_cmd(IPA_IMM_CMD_REGISTER_WRITE,
-		&quota_mask, false);
-	if (!quota_mask_pyld) {
-		IPAERR("failed to construct register_write imm cmd\n");
-		ret = -ENOMEM;
-		goto unmap;
-	}
-	desc[0].opcode = quota_mask_pyld->opcode;
-	desc[0].pyld = quota_mask_pyld->data;
-	desc[0].len = quota_mask_pyld->len;
-	desc[0].type = IPA_IMM_CMD_DESC;
-
-	quota_base.skip_pipeline_clear = false;
-	quota_base.pipeline_clear_options = IPAHAL_FULL_PIPELINE_CLEAR;
-	quota_base.offset = ipahal_get_reg_n_ofst(IPA_STAT_QUOTA_BASE_n,
-		ipa3_ctx->ee);
-	quota_base.value = ipa3_ctx->smem_restricted_bytes +
-		IPA_MEM_PART(stats_quota_ofst);
-	quota_base.value_mask = ~0;
-	quota_base_pyld = ipahal_construct_imm_cmd(IPA_IMM_CMD_REGISTER_WRITE,
-		&quota_base, false);
-	if (!quota_base_pyld) {
-		IPAERR("failed to construct register_write imm cmd\n");
-		ret = -ENOMEM;
-		goto destroy_quota_mask;
-	}
-	desc[1].opcode = quota_base_pyld->opcode;
-	desc[1].pyld = quota_base_pyld->data;
-	desc[1].len = quota_base_pyld->len;
-	desc[1].type = IPA_IMM_CMD_DESC;
-
-	cmd.is_read = false;
-	cmd.skip_pipeline_clear = false;
-	cmd.pipeline_clear_options = IPAHAL_FULL_PIPELINE_CLEAR;
-	cmd.size = pyld->len;
-	cmd.system_addr = dma_address;
-	cmd.local_addr = ipa3_ctx->smem_restricted_bytes +
-			IPA_MEM_PART(stats_quota_ofst);
-	cmd_pyld = ipahal_construct_imm_cmd(
-		IPA_IMM_CMD_DMA_SHARED_MEM, &cmd, false);
-	if (!cmd_pyld) {
-		IPAERR("failed to construct dma_shared_mem imm cmd\n");
-		ret = -ENOMEM;
-		goto destroy_quota_base;
-	}
-	desc[2].opcode = cmd_pyld->opcode;
-	desc[2].pyld = cmd_pyld->data;
-	desc[2].len = cmd_pyld->len;
-	desc[2].type = IPA_IMM_CMD_DESC;
-
-	ret = ipa3_send_cmd(3, desc);
-	if (ret) {
-		IPAERR("failed to send immediate command (error %d)\n", ret);
-		goto destroy_imm;
-	}
-
-	ret = 0;
-
-destroy_imm:
-	ipahal_destroy_imm_cmd(cmd_pyld);
-destroy_quota_base:
-	ipahal_destroy_imm_cmd(quota_base_pyld);
-destroy_quota_mask:
-	ipahal_destroy_imm_cmd(quota_mask_pyld);
-unmap:
-	dma_unmap_single(ipa3_ctx->pdev, dma_address, pyld->len, DMA_TO_DEVICE);
-destroy_init_pyld:
-	ipahal_destroy_stats_init_pyld(pyld);
-	return ret;
-}
-
-int ipa_get_quota_stats(struct ipa_quota_stats_all *out)
-{
-	int i;
-	int ret;
-	struct ipahal_stats_get_offset_quota get_offset = { { 0 } };
-	struct ipahal_stats_offset offset = { 0 };
-	struct ipahal_imm_cmd_dma_shared_mem cmd = { 0 };
-	struct ipahal_imm_cmd_pyld *cmd_pyld;
-	struct ipa_mem_buffer mem;
-	struct ipa3_desc desc = { 0 };
-	struct ipahal_stats_quota_all *stats;
-
-	if (!ipa3_ctx->hw_stats.enabled)
-		return 0;
-
-	get_offset.init = ipa3_ctx->hw_stats.quota.init;
-	ret = ipahal_stats_get_offset(IPAHAL_HW_STATS_QUOTA, &get_offset,
-		&offset);
-	if (ret) {
-		IPAERR("failed to get offset from hal %d\n", ret);
-		return ret;
-	}
-
-	IPADBG_LOW("offset = %d size = %d\n", offset.offset, offset.size);
-
-	if (offset.size == 0)
-		return 0;
-
-	mem.size = offset.size;
-	mem.base = dma_alloc_coherent(ipa3_ctx->pdev,
-		mem.size,
-		&mem.phys_base,
-		GFP_KERNEL);
-	if (!mem.base) {
-		IPAERR("fail to alloc DMA memory");
-		return ret;
-	}
-
-	cmd.is_read = true;
-	cmd.clear_after_read = true;
-	cmd.skip_pipeline_clear = false;
-	cmd.pipeline_clear_options = IPAHAL_HPS_CLEAR;
-	cmd.size = mem.size;
-	cmd.system_addr = mem.phys_base;
-	cmd.local_addr = ipa3_ctx->smem_restricted_bytes +
-		IPA_MEM_PART(stats_quota_ofst) + offset.offset;
-	cmd_pyld = ipahal_construct_imm_cmd(
-		IPA_IMM_CMD_DMA_SHARED_MEM, &cmd, false);
-	if (!cmd_pyld) {
-		IPAERR("failed to construct dma_shared_mem imm cmd\n");
-		ret = -ENOMEM;
-		goto free_dma_mem;
-	}
-	desc.opcode = cmd_pyld->opcode;
-	desc.pyld = cmd_pyld->data;
-	desc.len = cmd_pyld->len;
-	desc.type = IPA_IMM_CMD_DESC;
-
-	ret = ipa3_send_cmd(1, &desc);
-	if (ret) {
-		IPAERR("failed to send immediate command (error %d)\n", ret);
-		goto destroy_imm;
-	}
-
-	stats = kzalloc(sizeof(*stats), GFP_KERNEL);
-	if (!stats) {
-		ret = -ENOMEM;
-		goto destroy_imm;
-	}
-
-	ret = ipahal_parse_stats(IPAHAL_HW_STATS_QUOTA,
-		&ipa3_ctx->hw_stats.quota.init, mem.base, stats);
-	if (ret) {
-		IPAERR("failed to parse stats (error %d)\n", ret);
-		goto free_stats;
-	}
-
-	/*
-	 * update driver cache.
-	 * the stats were read from hardware with clear_after_read meaning
-	 * hardware stats are 0 now
-	 */
-	for (i = 0; i < IPA_CLIENT_MAX; i++) {
-		int ep_idx = ipa3_get_ep_mapping(i);
-
-		if (ep_idx == -1 || ep_idx >= IPA3_MAX_NUM_PIPES)
-			continue;
-
-		if (ipa3_ctx->ep[ep_idx].client != i)
-			continue;
-
-		ipa3_ctx->hw_stats.quota.stats.client[i].num_ipv4_bytes +=
-			stats->stats[ep_idx].num_ipv4_bytes;
-		ipa3_ctx->hw_stats.quota.stats.client[i].num_ipv4_pkts +=
-			stats->stats[ep_idx].num_ipv4_pkts;
-		ipa3_ctx->hw_stats.quota.stats.client[i].num_ipv6_bytes +=
-			stats->stats[ep_idx].num_ipv6_bytes;
-		ipa3_ctx->hw_stats.quota.stats.client[i].num_ipv6_pkts +=
-			stats->stats[ep_idx].num_ipv6_pkts;
-	}
-
-	/* copy results to out parameter */
-	if (out)
-		*out = ipa3_ctx->hw_stats.quota.stats;
-	ret = 0;
-free_stats:
-	kfree(stats);
-destroy_imm:
-	ipahal_destroy_imm_cmd(cmd_pyld);
-free_dma_mem:
-	dma_free_coherent(ipa3_ctx->pdev, mem.size, mem.base, mem.phys_base);
-	return ret;
-
-}
-
-int ipa_reset_quota_stats(enum ipa_client_type client)
-{
-	int ret;
-	struct ipa_quota_stats *stats;
-
-	if (!ipa3_ctx->hw_stats.enabled)
-		return 0;
-
-	if (client >= IPA_CLIENT_MAX) {
-		IPAERR("invalid client %d\n", client);
-		return -EINVAL;
-	}
-
-	/* reading stats will reset them in hardware */
-	ret = ipa_get_quota_stats(NULL);
-	if (ret) {
-		IPAERR("ipa_get_quota_stats failed %d\n", ret);
-		return ret;
-	}
-
-	/* reset driver's cache */
-	stats = &ipa3_ctx->hw_stats.quota.stats.client[client];
-	memset(stats, 0, sizeof(*stats));
-	return 0;
-}
-
-int ipa_reset_all_quota_stats(void)
-{
-	int ret;
-	struct ipa_quota_stats_all *stats;
-
-	if (!ipa3_ctx->hw_stats.enabled)
-		return 0;
-
-	/* reading stats will reset them in hardware */
-	ret = ipa_get_quota_stats(NULL);
-	if (ret) {
-		IPAERR("ipa_get_quota_stats failed %d\n", ret);
-		return ret;
-	}
-
-	/* reset driver's cache */
-	stats = &ipa3_ctx->hw_stats.quota.stats;
-	memset(stats, 0, sizeof(*stats));
-	return 0;
-}
-
-int ipa_init_teth_stats(struct ipa_teth_stats_endpoints *in)
-{
-	struct ipahal_stats_init_pyld *pyld;
-	struct ipahal_imm_cmd_dma_shared_mem cmd = { 0 };
-	struct ipahal_imm_cmd_pyld *cmd_pyld;
-	struct ipahal_imm_cmd_register_write teth_base = {0};
-	struct ipahal_imm_cmd_pyld *teth_base_pyld;
-	struct ipahal_imm_cmd_register_write teth_mask = { 0 };
-	struct ipahal_imm_cmd_pyld *teth_mask_pyld;
-	struct ipa3_desc desc[3] = { {0} };
-	dma_addr_t dma_address;
-	int ret;
-	int i;
-
-	if (!ipa3_ctx->hw_stats.enabled)
-		return 0;
-
-	if (!in || !in->prod_mask) {
-		IPAERR("invalid params\n");
-		return -EINVAL;
-	}
-
-	for (i = 0; i < IPA_STATS_MAX_PIPE_BIT; i++) {
-		if ((in->prod_mask & (1 << i)) && !in->dst_ep_mask[i]) {
-			IPAERR("prod %d doesn't have cons\n", i);
-			return -EINVAL;
-		}
-	}
-	IPADBG_LOW("prod_mask=0x%x\n", in->prod_mask);
-
-	/* reset driver's cache */
-	memset(&ipa3_ctx->hw_stats.teth.init, 0,
-		sizeof(ipa3_ctx->hw_stats.teth.init));
-	for (i = 0; i < IPA_CLIENT_MAX; i++) {
-		memset(&ipa3_ctx->hw_stats.teth.prod_stats_sum[i], 0,
-			sizeof(ipa3_ctx->hw_stats.teth.prod_stats_sum[i]));
-		memset(&ipa3_ctx->hw_stats.teth.prod_stats[i], 0,
-			sizeof(ipa3_ctx->hw_stats.teth.prod_stats[i]));
-	}
-	ipa3_ctx->hw_stats.teth.init.prod_bitmask = in->prod_mask;
-	memcpy(ipa3_ctx->hw_stats.teth.init.cons_bitmask, in->dst_ep_mask,
-		sizeof(ipa3_ctx->hw_stats.teth.init.cons_bitmask));
-
-
-	pyld = ipahal_stats_generate_init_pyld(IPAHAL_HW_STATS_TETHERING,
-		&ipa3_ctx->hw_stats.teth.init, false);
-	if (!pyld) {
-		IPAERR("failed to generate pyld\n");
-		return -EPERM;
-	}
-
-	if (pyld->len > IPA_MEM_PART(stats_tethering_size)) {
-		IPAERR("SRAM partition too small: %d needed %d\n",
-			IPA_MEM_PART(stats_tethering_size), pyld->len);
-		ret = -EPERM;
-		goto destroy_init_pyld;
-	}
-
-	dma_address = dma_map_single(ipa3_ctx->pdev,
-		pyld->data,
-		pyld->len,
-		DMA_TO_DEVICE);
-	if (dma_mapping_error(ipa3_ctx->pdev, dma_address)) {
-		IPAERR("failed to DMA map\n");
-		ret = -EPERM;
-		goto destroy_init_pyld;
-	}
-
-	/* setting the registers and init the stats pyld are done atomically */
-	teth_mask.skip_pipeline_clear = false;
-	teth_mask.pipeline_clear_options = IPAHAL_FULL_PIPELINE_CLEAR;
-	teth_mask.offset = ipahal_get_reg_n_ofst(IPA_STAT_TETHERING_MASK_n,
-		ipa3_ctx->ee);
-	teth_mask.value = in->prod_mask;
-	teth_mask.value_mask = ~0;
-	teth_mask_pyld = ipahal_construct_imm_cmd(IPA_IMM_CMD_REGISTER_WRITE,
-		&teth_mask, false);
-	if (!teth_mask_pyld) {
-		IPAERR("failed to construct register_write imm cmd\n");
-		ret = -ENOMEM;
-		goto unmap;
-	}
-	desc[0].opcode = teth_mask_pyld->opcode;
-	desc[0].pyld = teth_mask_pyld->data;
-	desc[0].len = teth_mask_pyld->len;
-	desc[0].type = IPA_IMM_CMD_DESC;
-
-	teth_base.skip_pipeline_clear = false;
-	teth_base.pipeline_clear_options = IPAHAL_FULL_PIPELINE_CLEAR;
-	teth_base.offset = ipahal_get_reg_n_ofst(IPA_STAT_TETHERING_BASE_n,
-		ipa3_ctx->ee);
-	teth_base.value = ipa3_ctx->smem_restricted_bytes +
-		IPA_MEM_PART(stats_tethering_ofst);
-	teth_base.value_mask = ~0;
-	teth_base_pyld = ipahal_construct_imm_cmd(IPA_IMM_CMD_REGISTER_WRITE,
-		&teth_base, false);
-	if (!teth_base_pyld) {
-		IPAERR("failed to construct register_write imm cmd\n");
-		ret = -ENOMEM;
-		goto destroy_teth_mask;
-	}
-	desc[1].opcode = teth_base_pyld->opcode;
-	desc[1].pyld = teth_base_pyld->data;
-	desc[1].len = teth_base_pyld->len;
-	desc[1].type = IPA_IMM_CMD_DESC;
-
-	cmd.is_read = false;
-	cmd.skip_pipeline_clear = false;
-	cmd.pipeline_clear_options = IPAHAL_FULL_PIPELINE_CLEAR;
-	cmd.size = pyld->len;
-	cmd.system_addr = dma_address;
-	cmd.local_addr = ipa3_ctx->smem_restricted_bytes +
-			IPA_MEM_PART(stats_tethering_ofst);
-	cmd_pyld = ipahal_construct_imm_cmd(
-		IPA_IMM_CMD_DMA_SHARED_MEM, &cmd, false);
-	if (!cmd_pyld) {
-		IPAERR("failed to construct dma_shared_mem imm cmd\n");
-		ret = -ENOMEM;
-		goto destroy_teth_base;
-	}
-	desc[2].opcode = cmd_pyld->opcode;
-	desc[2].pyld = cmd_pyld->data;
-	desc[2].len = cmd_pyld->len;
-	desc[2].type = IPA_IMM_CMD_DESC;
-
-	ret = ipa3_send_cmd(3, desc);
-	if (ret) {
-		IPAERR("failed to send immediate command (error %d)\n", ret);
-		goto destroy_imm;
-	}
-
-	ret = 0;
-
-destroy_imm:
-	ipahal_destroy_imm_cmd(cmd_pyld);
-destroy_teth_base:
-	ipahal_destroy_imm_cmd(teth_base_pyld);
-destroy_teth_mask:
-	ipahal_destroy_imm_cmd(teth_mask_pyld);
-unmap:
-	dma_unmap_single(ipa3_ctx->pdev, dma_address, pyld->len, DMA_TO_DEVICE);
-destroy_init_pyld:
-	ipahal_destroy_stats_init_pyld(pyld);
-	return ret;
-}
-
-int ipa_get_teth_stats(void)
-{
-	int i, j;
-	int ret;
-	struct ipahal_stats_get_offset_tethering get_offset = { { 0 } };
-	struct ipahal_stats_offset offset = {0};
-	struct ipahal_imm_cmd_dma_shared_mem cmd = { 0 };
-	struct ipahal_imm_cmd_pyld *cmd_pyld;
-	struct ipa_mem_buffer mem;
-	struct ipa3_desc desc = { 0 };
-	struct ipahal_stats_tethering_all *stats_all;
-	struct ipa_hw_stats_teth *sw_stats = &ipa3_ctx->hw_stats.teth;
-	struct ipahal_stats_tethering *stats;
-	struct ipa_quota_stats *quota_stats;
-	struct ipahal_stats_init_tethering *init =
-		(struct ipahal_stats_init_tethering *)
-			&ipa3_ctx->hw_stats.teth.init;
-
-	if (!ipa3_ctx->hw_stats.enabled)
-		return 0;
-
-	get_offset.init = ipa3_ctx->hw_stats.teth.init;
-	ret = ipahal_stats_get_offset(IPAHAL_HW_STATS_TETHERING, &get_offset,
-		&offset);
-	if (ret) {
-		IPAERR("failed to get offset from hal %d\n", ret);
-		return ret;
-	}
-
-	IPADBG_LOW("offset = %d size = %d\n", offset.offset, offset.size);
-
-	if (offset.size == 0)
-		return 0;
-
-	mem.size = offset.size;
-	mem.base = dma_alloc_coherent(ipa3_ctx->pdev,
-		mem.size,
-		&mem.phys_base,
-		GFP_KERNEL);
-	if (!mem.base) {
-		IPAERR("fail to alloc DMA memory\n");
-		return ret;
-	}
-
-	cmd.is_read = true;
-	cmd.clear_after_read = true;
-	cmd.skip_pipeline_clear = false;
-	cmd.pipeline_clear_options = IPAHAL_HPS_CLEAR;
-	cmd.size = mem.size;
-	cmd.system_addr = mem.phys_base;
-	cmd.local_addr = ipa3_ctx->smem_restricted_bytes +
-		IPA_MEM_PART(stats_tethering_ofst) + offset.offset;
-	cmd_pyld = ipahal_construct_imm_cmd(
-		IPA_IMM_CMD_DMA_SHARED_MEM, &cmd, false);
-	if (!cmd_pyld) {
-		IPAERR("failed to construct dma_shared_mem imm cmd\n");
-		ret = -ENOMEM;
-		goto free_dma_mem;
-	}
-	desc.opcode = cmd_pyld->opcode;
-	desc.pyld = cmd_pyld->data;
-	desc.len = cmd_pyld->len;
-	desc.type = IPA_IMM_CMD_DESC;
-
-	ret = ipa3_send_cmd(1, &desc);
-	if (ret) {
-		IPAERR("failed to send immediate command (error %d)\n", ret);
-		goto destroy_imm;
-	}
-
-	stats_all = kzalloc(sizeof(*stats_all), GFP_KERNEL);
-	if (!stats_all) {
-		IPADBG("failed to alloc memory\n");
-		ret = -ENOMEM;
-		goto destroy_imm;
-	}
-
-	ret = ipahal_parse_stats(IPAHAL_HW_STATS_TETHERING,
-		&ipa3_ctx->hw_stats.teth.init, mem.base, stats_all);
-	if (ret) {
-		IPAERR("failed to parse stats_all (error %d)\n", ret);
-		goto free_stats;
-	}
-
-	/* reset prod_stats cache */
-	for (i = 0; i < IPA_CLIENT_MAX; i++) {
-		memset(&ipa3_ctx->hw_stats.teth.prod_stats[i], 0,
-			sizeof(ipa3_ctx->hw_stats.teth.prod_stats[i]));
-	}
-
-	/*
-	 * update driver cache.
-	 * the stats were read from hardware with clear_after_read meaning
-	 * hardware stats are 0 now
-	 */
-	for (i = 0; i < IPA_CLIENT_MAX; i++) {
-		for (j = 0; j < IPA_CLIENT_MAX; j++) {
-			int prod_idx = ipa3_get_ep_mapping(i);
-			int cons_idx = ipa3_get_ep_mapping(j);
-
-			if (prod_idx == -1 || prod_idx >= IPA3_MAX_NUM_PIPES)
-				continue;
-
-			if (cons_idx == -1 || cons_idx >= IPA3_MAX_NUM_PIPES)
-				continue;
-
-			/* save hw-query result */
-			if ((init->prod_bitmask & (1 << prod_idx)) &&
-				(init->cons_bitmask[prod_idx]
-					& (1 << cons_idx))) {
-				IPADBG_LOW("prod %d cons %d\n",
-					prod_idx, cons_idx);
-				stats = &stats_all->stats[prod_idx][cons_idx];
-				IPADBG_LOW("num_ipv4_bytes %lld\n",
-					stats->num_ipv4_bytes);
-				IPADBG_LOW("num_ipv4_pkts %lld\n",
-					stats->num_ipv4_pkts);
-				IPADBG_LOW("num_ipv6_pkts %lld\n",
-					stats->num_ipv6_pkts);
-				IPADBG_LOW("num_ipv6_bytes %lld\n",
-					stats->num_ipv6_bytes);
-
-				/* update stats*/
-				quota_stats =
-					&sw_stats->prod_stats[i].client[j];
-				quota_stats->num_ipv4_bytes =
-					stats->num_ipv4_bytes;
-				quota_stats->num_ipv4_pkts =
-					stats->num_ipv4_pkts;
-				quota_stats->num_ipv6_bytes =
-					stats->num_ipv6_bytes;
-				quota_stats->num_ipv6_pkts =
-					stats->num_ipv6_pkts;
-
-				/* Accumulated stats */
-				quota_stats =
-					&sw_stats->prod_stats_sum[i].client[j];
-				quota_stats->num_ipv4_bytes +=
-					stats->num_ipv4_bytes;
-				quota_stats->num_ipv4_pkts +=
-					stats->num_ipv4_pkts;
-				quota_stats->num_ipv6_bytes +=
-					stats->num_ipv6_bytes;
-				quota_stats->num_ipv6_pkts +=
-					stats->num_ipv6_pkts;
-			}
-		}
-	}
-
-	ret = 0;
-free_stats:
-	kfree(stats_all);
-	stats = NULL;
-destroy_imm:
-	ipahal_destroy_imm_cmd(cmd_pyld);
-free_dma_mem:
-	dma_free_coherent(ipa3_ctx->pdev, mem.size, mem.base, mem.phys_base);
-	return ret;
-
-}
-
-int ipa_query_teth_stats(enum ipa_client_type prod,
-	struct ipa_quota_stats_all *out, bool reset)
-{
-	if (!IPA_CLIENT_IS_PROD(prod) || ipa3_get_ep_mapping(prod) == -1) {
-		IPAERR("invalid prod %d\n", prod);
-		return -EINVAL;
-	}
-
-	/* copy results to out parameter */
-	if (reset)
-		*out = ipa3_ctx->hw_stats.teth.prod_stats[prod];
-	else
-		*out = ipa3_ctx->hw_stats.teth.prod_stats_sum[prod];
-	return 0;
-}
-
-int ipa_reset_teth_stats(enum ipa_client_type prod, enum ipa_client_type cons)
-{
-	int ret;
-	struct ipa_quota_stats *stats;
-
-	if (!ipa3_ctx->hw_stats.enabled)
-		return 0;
-
-	if (!IPA_CLIENT_IS_PROD(prod) || !IPA_CLIENT_IS_CONS(cons)) {
-		IPAERR("invalid prod %d or cons %d\n", prod, cons);
-		return -EINVAL;
-	}
-
-	/* reading stats will reset them in hardware */
-	ret = ipa_get_teth_stats();
-	if (ret) {
-		IPAERR("ipa_get_teth_stats failed %d\n", ret);
-		return ret;
-	}
-
-	/* reset driver's cache */
-	stats = &ipa3_ctx->hw_stats.teth.prod_stats_sum[prod].client[cons];
-	memset(stats, 0, sizeof(*stats));
-	return 0;
-}
-
-int ipa_reset_all_cons_teth_stats(enum ipa_client_type prod)
-{
-	int ret;
-	int i;
-	struct ipa_quota_stats *stats;
-
-	if (!ipa3_ctx->hw_stats.enabled)
-		return 0;
-
-	if (!IPA_CLIENT_IS_PROD(prod)) {
-		IPAERR("invalid prod %d\n", prod);
-		return -EINVAL;
-	}
-
-	/* reading stats will reset them in hardware */
-	ret = ipa_get_teth_stats();
-	if (ret) {
-		IPAERR("ipa_get_teth_stats failed %d\n", ret);
-		return ret;
-	}
-
-	/* reset driver's cache */
-	for (i = 0; i < IPA_CLIENT_MAX; i++) {
-		stats = &ipa3_ctx->hw_stats.teth.prod_stats_sum[prod].client[i];
-		memset(stats, 0, sizeof(*stats));
-	}
-
-	return 0;
-}
-
-int ipa_reset_all_teth_stats(void)
-{
-	int i;
-	int ret;
-	struct ipa_quota_stats_all *stats;
-
-	if (!ipa3_ctx->hw_stats.enabled)
-		return 0;
-
-	/* reading stats will reset them in hardware */
-	for (i = 0; i < IPA_CLIENT_MAX; i++) {
-		if (IPA_CLIENT_IS_PROD(i) && ipa3_get_ep_mapping(i) != -1) {
-			ret = ipa_get_teth_stats();
-			if (ret) {
-				IPAERR("ipa_get_teth_stats failed %d\n", ret);
-				return ret;
-			}
-			/* a single iteration will reset all hardware stats */
-			break;
-		}
-	}
-
-	/* reset driver's cache */
-	for (i = 0; i < IPA_CLIENT_MAX; i++) {
-		stats = &ipa3_ctx->hw_stats.teth.prod_stats_sum[i];
-		memset(stats, 0, sizeof(*stats));
-	}
-
-	return 0;
-}
-
-int ipa_init_flt_rt_stats(void)
-{
-	struct ipahal_stats_init_pyld *pyld;
-	int smem_ofst, smem_size;
-	int stats_base_flt_v4, stats_base_flt_v6;
-	int stats_base_rt_v4, stats_base_rt_v6;
-	struct ipahal_imm_cmd_dma_shared_mem cmd = { 0 };
-	struct ipahal_imm_cmd_pyld *cmd_pyld;
-	struct ipahal_imm_cmd_register_write flt_v4_base = {0};
-	struct ipahal_imm_cmd_pyld *flt_v4_base_pyld;
-	struct ipahal_imm_cmd_register_write flt_v6_base = {0};
-	struct ipahal_imm_cmd_pyld *flt_v6_base_pyld;
-	struct ipahal_imm_cmd_register_write rt_v4_base = {0};
-	struct ipahal_imm_cmd_pyld *rt_v4_base_pyld;
-	struct ipahal_imm_cmd_register_write rt_v6_base = {0};
-	struct ipahal_imm_cmd_pyld *rt_v6_base_pyld;
-	struct ipa3_desc desc[5] = { {0} };
-	dma_addr_t dma_address;
-	int ret;
-
-	if (!ipa3_ctx->hw_stats.enabled)
-		return 0;
-
-	smem_ofst = IPA_MEM_PART(stats_fnr_ofst);
-	smem_size = IPA_MEM_PART(stats_fnr_size);
-
-	pyld = ipahal_stats_generate_init_pyld(IPAHAL_HW_STATS_FNR,
-		(void *)(uintptr_t)(IPA_MAX_FLT_RT_CNT_INDEX), false);
-	if (!pyld) {
-		IPAERR("failed to generate pyld\n");
-		return -EPERM;
-	}
-
-	if (pyld->len > smem_size) {
-		IPAERR("SRAM partition too small: %d needed %d\n",
-			smem_size, pyld->len);
-		ret = -EPERM;
-		goto destroy_init_pyld;
-	}
-
-	dma_address = dma_map_single(ipa3_ctx->pdev,
-		pyld->data,
-		pyld->len,
-		DMA_TO_DEVICE);
-	if (dma_mapping_error(ipa3_ctx->pdev, dma_address)) {
-		IPAERR("failed to DMA map\n");
-		ret = -EPERM;
-		goto destroy_init_pyld;
-	}
-
-	stats_base_flt_v4 = ipahal_get_reg_ofst(IPA_STAT_FILTER_IPV4_BASE);
-	stats_base_flt_v6 = ipahal_get_reg_ofst(IPA_STAT_FILTER_IPV6_BASE);
-	stats_base_rt_v4 = ipahal_get_reg_ofst(IPA_STAT_ROUTER_IPV4_BASE);
-	stats_base_rt_v6 = ipahal_get_reg_ofst(IPA_STAT_ROUTER_IPV6_BASE);
-
-	/* setting the registers and init the stats pyld are done atomically */
-	/* set IPA_STAT_FILTER_IPV4_BASE */
-	flt_v4_base.skip_pipeline_clear = false;
-	flt_v4_base.pipeline_clear_options = IPAHAL_FULL_PIPELINE_CLEAR;
-	flt_v4_base.offset = stats_base_flt_v4;
-	flt_v4_base.value = ipa3_ctx->smem_restricted_bytes +
-		smem_ofst;
-	flt_v4_base.value_mask = ~0;
-	flt_v4_base_pyld = ipahal_construct_imm_cmd(IPA_IMM_CMD_REGISTER_WRITE,
-		&flt_v4_base, false);
-	if (!flt_v4_base_pyld) {
-		IPAERR("failed to construct register_write imm cmd\n");
-		ret = -ENOMEM;
-		goto unmap;
-	}
-	desc[0].opcode = flt_v4_base_pyld->opcode;
-	desc[0].pyld = flt_v4_base_pyld->data;
-	desc[0].len = flt_v4_base_pyld->len;
-	desc[0].type = IPA_IMM_CMD_DESC;
-
-	/* set IPA_STAT_FILTER_IPV6_BASE */
-	flt_v6_base.skip_pipeline_clear = false;
-	flt_v6_base.pipeline_clear_options = IPAHAL_FULL_PIPELINE_CLEAR;
-	flt_v6_base.offset = stats_base_flt_v6;
-	flt_v6_base.value = ipa3_ctx->smem_restricted_bytes +
-		smem_ofst;
-	flt_v6_base.value_mask = ~0;
-	flt_v6_base_pyld = ipahal_construct_imm_cmd(IPA_IMM_CMD_REGISTER_WRITE,
-		&flt_v6_base, false);
-	if (!flt_v6_base_pyld) {
-		IPAERR("failed to construct register_write imm cmd\n");
-		ret = -ENOMEM;
-		goto destroy_flt_v4_base;
-	}
-	desc[1].opcode = flt_v6_base_pyld->opcode;
-	desc[1].pyld = flt_v6_base_pyld->data;
-	desc[1].len = flt_v6_base_pyld->len;
-	desc[1].type = IPA_IMM_CMD_DESC;
-
-
-	/* set IPA_STAT_ROUTER_IPV4_BASE */
-	rt_v4_base.skip_pipeline_clear = false;
-	rt_v4_base.pipeline_clear_options = IPAHAL_FULL_PIPELINE_CLEAR;
-	rt_v4_base.offset = stats_base_rt_v4;
-	rt_v4_base.value = ipa3_ctx->smem_restricted_bytes +
-		smem_ofst;
-	rt_v4_base.value_mask = ~0;
-	rt_v4_base_pyld = ipahal_construct_imm_cmd(IPA_IMM_CMD_REGISTER_WRITE,
-		&rt_v4_base, false);
-	if (!rt_v4_base_pyld) {
-		IPAERR("failed to construct register_write imm cmd\n");
-		ret = -ENOMEM;
-		goto destroy_flt_v6_base;
-	}
-	desc[2].opcode = rt_v4_base_pyld->opcode;
-	desc[2].pyld = rt_v4_base_pyld->data;
-	desc[2].len = rt_v4_base_pyld->len;
-	desc[2].type = IPA_IMM_CMD_DESC;
-
-	/* set IPA_STAT_ROUTER_IPV6_BASE */
-	rt_v6_base.skip_pipeline_clear = false;
-	rt_v6_base.pipeline_clear_options = IPAHAL_FULL_PIPELINE_CLEAR;
-	rt_v6_base.offset = stats_base_rt_v6;
-	rt_v6_base.value = ipa3_ctx->smem_restricted_bytes +
-		smem_ofst;
-	rt_v6_base.value_mask = ~0;
-	rt_v6_base_pyld = ipahal_construct_imm_cmd(IPA_IMM_CMD_REGISTER_WRITE,
-		&rt_v6_base, false);
-	if (!rt_v6_base_pyld) {
-		IPAERR("failed to construct register_write imm cmd\n");
-		ret = -ENOMEM;
-		goto destroy_rt_v4_base;
-	}
-	desc[3].opcode = rt_v6_base_pyld->opcode;
-	desc[3].pyld = rt_v6_base_pyld->data;
-	desc[3].len = rt_v6_base_pyld->len;
-	desc[3].type = IPA_IMM_CMD_DESC;
-
-	cmd.is_read = false;
-	cmd.skip_pipeline_clear = false;
-	cmd.pipeline_clear_options = IPAHAL_FULL_PIPELINE_CLEAR;
-	cmd.size = pyld->len;
-	cmd.system_addr = dma_address;
-	cmd.local_addr = ipa3_ctx->smem_restricted_bytes +
-			smem_ofst;
-	cmd_pyld = ipahal_construct_imm_cmd(
-		IPA_IMM_CMD_DMA_SHARED_MEM, &cmd, false);
-	if (!cmd_pyld) {
-		IPAERR("failed to construct dma_shared_mem imm cmd\n");
-		ret = -ENOMEM;
-		goto destroy_rt_v6_base;
-	}
-	desc[4].opcode = cmd_pyld->opcode;
-	desc[4].pyld = cmd_pyld->data;
-	desc[4].len = cmd_pyld->len;
-	desc[4].type = IPA_IMM_CMD_DESC;
-
-	ret = ipa3_send_cmd(5, desc);
-	if (ret) {
-		IPAERR("failed to send immediate command (error %d)\n", ret);
-		goto destroy_imm;
-	}
-
-	ret = 0;
-
-destroy_imm:
-	ipahal_destroy_imm_cmd(cmd_pyld);
-destroy_rt_v6_base:
-	ipahal_destroy_imm_cmd(rt_v6_base_pyld);
-destroy_rt_v4_base:
-	ipahal_destroy_imm_cmd(rt_v4_base_pyld);
-destroy_flt_v6_base:
-	ipahal_destroy_imm_cmd(flt_v6_base_pyld);
-destroy_flt_v4_base:
-	ipahal_destroy_imm_cmd(flt_v4_base_pyld);
-unmap:
-	dma_unmap_single(ipa3_ctx->pdev, dma_address, pyld->len, DMA_TO_DEVICE);
-destroy_init_pyld:
-	ipahal_destroy_stats_init_pyld(pyld);
-	return ret;
-}
-
-static int __ipa_get_flt_rt_stats(struct ipa_ioc_flt_rt_query *query)
-{
-	int ret;
-	int smem_ofst;
-	bool clear = query->reset;
-	struct ipahal_stats_get_offset_flt_rt_v4_5 *get_offset;
-	struct ipahal_stats_offset offset = { 0 };
-	struct ipahal_imm_cmd_dma_shared_mem cmd = { 0 };
-	struct ipahal_imm_cmd_pyld *cmd_pyld;
-	struct ipa_mem_buffer mem;
-	struct ipa3_desc desc = { 0 };
-
-	get_offset = kzalloc(sizeof(*get_offset), GFP_KERNEL);
-	if (!get_offset) {
-		IPADBG("no mem\n");
-		return -ENOMEM;
-	}
-
-	smem_ofst = IPA_MEM_PART(stats_fnr_ofst);
-
-	get_offset->start_id = query->start_id;
-	get_offset->end_id = query->end_id;
-
-	ret = ipahal_stats_get_offset(IPAHAL_HW_STATS_FNR, get_offset,
-		&offset);
-	if (ret) {
-		IPAERR("failed to get offset from hal %d\n", ret);
-		goto free_offset;
-	}
-
-	IPADBG("offset = %d size = %d\n", offset.offset, offset.size);
-
-	if (offset.size == 0) {
-		ret = 0;
-		goto free_offset;
-	}
-
-	mem.size = offset.size;
-	mem.base = dma_alloc_coherent(ipa3_ctx->pdev,
-		mem.size,
-		&mem.phys_base,
-		GFP_KERNEL);
-	if (!mem.base) {
-		IPAERR("fail to alloc DMA memory\n");
-		goto free_offset;
-	}
-
-	cmd.is_read = true;
-	cmd.clear_after_read = clear;
-	cmd.skip_pipeline_clear = false;
-	cmd.pipeline_clear_options = IPAHAL_HPS_CLEAR;
-	cmd.size = mem.size;
-	cmd.system_addr = mem.phys_base;
-	cmd.local_addr = ipa3_ctx->smem_restricted_bytes +
-		smem_ofst + offset.offset;
-	cmd_pyld = ipahal_construct_imm_cmd(
-		IPA_IMM_CMD_DMA_SHARED_MEM, &cmd, false);
-	if (!cmd_pyld) {
-		IPAERR("failed to construct dma_shared_mem imm cmd\n");
-		ret = -ENOMEM;
-		goto free_dma_mem;
-	}
-	desc.opcode = cmd_pyld->opcode;
-	desc.pyld = cmd_pyld->data;
-	desc.len = cmd_pyld->len;
-	desc.type = IPA_IMM_CMD_DESC;
-
-	ret = ipa3_send_cmd(1, &desc);
-	if (ret) {
-		IPAERR("failed to send immediate command (error %d)\n", ret);
-		goto destroy_imm;
-	}
-
-	ret = ipahal_parse_stats(IPAHAL_HW_STATS_FNR,
-		NULL, mem.base, query);
-	if (ret) {
-		IPAERR("failed to parse stats (error %d)\n", ret);
-		goto destroy_imm;
-	}
-	ret = 0;
-
-destroy_imm:
-	ipahal_destroy_imm_cmd(cmd_pyld);
-free_dma_mem:
-	dma_free_coherent(ipa3_ctx->pdev, mem.size, mem.base, mem.phys_base);
-free_offset:
-	kfree(get_offset);
-	return ret;
-}
-
-int ipa_get_flt_rt_stats(struct ipa_ioc_flt_rt_query *query)
-{
-	if (!ipa3_ctx->hw_stats.enabled) {
-		IPAERR("hw_stats is not enabled\n");
-		return 0;
-	}
-
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_5) {
-		IPAERR("FnR stats not supported in %d hw_type\n",
-			ipa3_ctx->ipa_hw_type);
-		return 0;
-	}
-
-	if (query->start_id == 0 || query->end_id == 0) {
-		IPAERR("Invalid start_id/end_id, must be not 0\n");
-		IPAERR("start_id %d, end_id %d\n",
-			query->start_id, query->end_id);
-		return -EINVAL;
-	}
-
-	if (query->start_id > IPA_MAX_FLT_RT_CNT_INDEX) {
-		IPAERR("start_cnt_id %d out of range\n", query->start_id);
-		return -EINVAL;
-	}
-
-	if (query->end_id > IPA_MAX_FLT_RT_CNT_INDEX) {
-		IPAERR("end_cnt_id %d out of range\n", query->end_id);
-		return -EINVAL;
-	}
-
-	if (query->end_id < query->start_id) {
-		IPAERR("end_id %d < start_id %d\n",
-			query->end_id, query->start_id);
-		return -EINVAL;
-	}
-
-	if (query->stats_size > sizeof(struct ipa_flt_rt_stats)) {
-		IPAERR("stats_size %d > ipa_flt_rt_stats %d\n",
-			query->stats_size, sizeof(struct ipa_flt_rt_stats));
-		return -EINVAL;
-	}
-
-	return __ipa_get_flt_rt_stats(query);
-}
-
-
-static int __ipa_set_flt_rt_stats(int index, struct ipa_flt_rt_stats stats)
-{
-	int ret;
-	int smem_ofst;
-	struct ipahal_stats_get_offset_flt_rt_v4_5 *get_offset;
-	struct ipahal_stats_offset offset = { 0 };
-	struct ipahal_imm_cmd_dma_shared_mem cmd = { 0 };
-	struct ipahal_imm_cmd_pyld *cmd_pyld;
-	struct ipa_mem_buffer mem;
-	struct ipa3_desc desc = { 0 };
-
-	get_offset = kzalloc(sizeof(*get_offset), GFP_KERNEL);
-	if (!get_offset) {
-		IPADBG("no mem\n");
-		return -ENOMEM;
-	}
-
-	smem_ofst = IPA_MEM_PART(stats_fnr_ofst);
-
-	get_offset->start_id = index;
-	get_offset->end_id = index;
-
-	ret = ipahal_stats_get_offset(IPAHAL_HW_STATS_FNR, get_offset,
-		&offset);
-	if (ret) {
-		IPAERR("failed to get offset from hal %d\n", ret);
-		goto free_offset;
-	}
-
-	IPADBG("offset = %d size = %d\n", offset.offset, offset.size);
-
-	if (offset.size == 0) {
-		ret = 0;
-		goto free_offset;
-	}
-
-	mem.size = offset.size;
-	mem.base = dma_alloc_coherent(ipa3_ctx->pdev,
-		mem.size,
-		&mem.phys_base,
-		GFP_KERNEL);
-	if (!mem.base) {
-		IPAERR("fail to alloc DMA memory\n");
-		goto free_offset;
-	}
-	ipahal_set_flt_rt_sw_stats(mem.base, stats);
-
-	cmd.is_read = false;
-	cmd.skip_pipeline_clear = false;
-	cmd.pipeline_clear_options = IPAHAL_HPS_CLEAR;
-	cmd.size = mem.size;
-	cmd.system_addr = mem.phys_base;
-	cmd.local_addr = ipa3_ctx->smem_restricted_bytes +
-		smem_ofst + offset.offset;
-	cmd_pyld = ipahal_construct_imm_cmd(
-		IPA_IMM_CMD_DMA_SHARED_MEM, &cmd, false);
-	if (!cmd_pyld) {
-		IPAERR("failed to construct dma_shared_mem imm cmd\n");
-		ret = -ENOMEM;
-		goto free_dma_mem;
-	}
-	desc.opcode = cmd_pyld->opcode;
-	desc.pyld = cmd_pyld->data;
-	desc.len = cmd_pyld->len;
-	desc.type = IPA_IMM_CMD_DESC;
-
-	ret = ipa3_send_cmd(1, &desc);
-	if (ret) {
-		IPAERR("failed to send immediate command (error %d)\n", ret);
-		goto destroy_imm;
-	}
-
-	ret = 0;
-
-destroy_imm:
-	ipahal_destroy_imm_cmd(cmd_pyld);
-free_dma_mem:
-	dma_free_coherent(ipa3_ctx->pdev, mem.size, mem.base, mem.phys_base);
-free_offset:
-	kfree(get_offset);
-	return ret;
-}
-
-int ipa_set_flt_rt_stats(int index, struct ipa_flt_rt_stats stats)
-{
-	if (!ipa3_ctx->hw_stats.enabled) {
-		IPAERR("hw_stats is not enabled\n");
-		return 0;
-	}
-
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_5) {
-		IPAERR("FnR stats not supported in %d hw_type\n",
-			ipa3_ctx->ipa_hw_type);
-		return 0;
-	}
-
-	if (index > IPA_MAX_FLT_RT_CNT_INDEX) {
-		IPAERR("index %d out of range\n", index);
-		return -EINVAL;
-	}
-
-	if (index <= IPA_FLT_RT_HW_COUNTER) {
-		IPAERR("index %d invalid, only support sw counter set\n",
-			index);
-		return -EINVAL;
-	}
-
-	return __ipa_set_flt_rt_stats(index, stats);
-}
-
-int ipa_init_drop_stats(u32 pipe_bitmask)
-{
-	struct ipahal_stats_init_pyld *pyld;
-	struct ipahal_imm_cmd_dma_shared_mem cmd = { 0 };
-	struct ipahal_imm_cmd_pyld *cmd_pyld;
-	struct ipahal_imm_cmd_register_write drop_base = {0};
-	struct ipahal_imm_cmd_pyld *drop_base_pyld;
-	struct ipahal_imm_cmd_register_write drop_mask = {0};
-	struct ipahal_imm_cmd_pyld *drop_mask_pyld;
-	struct ipa3_desc desc[3] = { {0} };
-	dma_addr_t dma_address;
-	int ret;
-
-	if (!ipa3_ctx->hw_stats.enabled)
-		return 0;
-
-	/* reset driver's cache */
-	memset(&ipa3_ctx->hw_stats.drop, 0, sizeof(ipa3_ctx->hw_stats.drop));
-	ipa3_ctx->hw_stats.drop.init.enabled_bitmask = pipe_bitmask;
-	IPADBG_LOW("pipe_bitmask=0x%x\n", pipe_bitmask);
-
-	pyld = ipahal_stats_generate_init_pyld(IPAHAL_HW_STATS_DROP,
-		&ipa3_ctx->hw_stats.drop.init, false);
-	if (!pyld) {
-		IPAERR("failed to generate pyld\n");
-		return -EPERM;
-	}
-
-	if (pyld->len > IPA_MEM_PART(stats_drop_size)) {
-		IPAERR("SRAM partition too small: %d needed %d\n",
-			IPA_MEM_PART(stats_drop_size), pyld->len);
-		ret = -EPERM;
-		goto destroy_init_pyld;
-	}
-
-	dma_address = dma_map_single(ipa3_ctx->pdev,
-		pyld->data,
-		pyld->len,
-		DMA_TO_DEVICE);
-	if (dma_mapping_error(ipa3_ctx->pdev, dma_address)) {
-		IPAERR("failed to DMA map\n");
-		ret = -EPERM;
-		goto destroy_init_pyld;
-	}
-
-	/* setting the registers and init the stats pyld are done atomically */
-	drop_mask.skip_pipeline_clear = false;
-	drop_mask.pipeline_clear_options = IPAHAL_FULL_PIPELINE_CLEAR;
-	drop_mask.offset = ipahal_get_reg_n_ofst(IPA_STAT_DROP_CNT_MASK_n,
-		ipa3_ctx->ee);
-	drop_mask.value = pipe_bitmask;
-	drop_mask.value_mask = ~0;
-	drop_mask_pyld = ipahal_construct_imm_cmd(IPA_IMM_CMD_REGISTER_WRITE,
-		&drop_mask, false);
-	if (!drop_mask_pyld) {
-		IPAERR("failed to construct register_write imm cmd\n");
-		ret = -ENOMEM;
-		goto unmap;
-	}
-	desc[0].opcode = drop_mask_pyld->opcode;
-	desc[0].pyld = drop_mask_pyld->data;
-	desc[0].len = drop_mask_pyld->len;
-	desc[0].type = IPA_IMM_CMD_DESC;
-
-	drop_base.skip_pipeline_clear = false;
-	drop_base.pipeline_clear_options = IPAHAL_FULL_PIPELINE_CLEAR;
-	drop_base.offset = ipahal_get_reg_n_ofst(IPA_STAT_DROP_CNT_BASE_n,
-		ipa3_ctx->ee);
-	drop_base.value = ipa3_ctx->smem_restricted_bytes +
-		IPA_MEM_PART(stats_drop_ofst);
-	drop_base.value_mask = ~0;
-	drop_base_pyld = ipahal_construct_imm_cmd(IPA_IMM_CMD_REGISTER_WRITE,
-		&drop_base, false);
-	if (!drop_base_pyld) {
-		IPAERR("failed to construct register_write imm cmd\n");
-		ret = -ENOMEM;
-		goto destroy_drop_mask;
-	}
-	desc[1].opcode = drop_base_pyld->opcode;
-	desc[1].pyld = drop_base_pyld->data;
-	desc[1].len = drop_base_pyld->len;
-	desc[1].type = IPA_IMM_CMD_DESC;
-
-	cmd.is_read = false;
-	cmd.skip_pipeline_clear = false;
-	cmd.pipeline_clear_options = IPAHAL_FULL_PIPELINE_CLEAR;
-	cmd.size = pyld->len;
-	cmd.system_addr = dma_address;
-	cmd.local_addr = ipa3_ctx->smem_restricted_bytes +
-			IPA_MEM_PART(stats_drop_ofst);
-	cmd_pyld = ipahal_construct_imm_cmd(
-		IPA_IMM_CMD_DMA_SHARED_MEM, &cmd, false);
-	if (!cmd_pyld) {
-		IPAERR("failed to construct dma_shared_mem imm cmd\n");
-		ret = -ENOMEM;
-		goto destroy_drop_base;
-	}
-	desc[2].opcode = cmd_pyld->opcode;
-	desc[2].pyld = cmd_pyld->data;
-	desc[2].len = cmd_pyld->len;
-	desc[2].type = IPA_IMM_CMD_DESC;
-
-	ret = ipa3_send_cmd(3, desc);
-	if (ret) {
-		IPAERR("failed to send immediate command (error %d)\n", ret);
-		goto destroy_imm;
-	}
-
-	ret = 0;
-
-destroy_imm:
-	ipahal_destroy_imm_cmd(cmd_pyld);
-destroy_drop_base:
-	ipahal_destroy_imm_cmd(drop_base_pyld);
-destroy_drop_mask:
-	ipahal_destroy_imm_cmd(drop_mask_pyld);
-unmap:
-	dma_unmap_single(ipa3_ctx->pdev, dma_address, pyld->len, DMA_TO_DEVICE);
-destroy_init_pyld:
-	ipahal_destroy_stats_init_pyld(pyld);
-	return ret;
-}
-
-int ipa_get_drop_stats(struct ipa_drop_stats_all *out)
-{
-	int i;
-	int ret;
-	struct ipahal_stats_get_offset_drop get_offset = { { 0 } };
-	struct ipahal_stats_offset offset = { 0 };
-	struct ipahal_imm_cmd_dma_shared_mem cmd = { 0 };
-	struct ipahal_imm_cmd_pyld *cmd_pyld;
-	struct ipa_mem_buffer mem;
-	struct ipa3_desc desc = { 0 };
-	struct ipahal_stats_drop_all *stats;
-
-	if (!ipa3_ctx->hw_stats.enabled)
-		return 0;
-
-	get_offset.init = ipa3_ctx->hw_stats.drop.init;
-	ret = ipahal_stats_get_offset(IPAHAL_HW_STATS_DROP, &get_offset,
-		&offset);
-	if (ret) {
-		IPAERR("failed to get offset from hal %d\n", ret);
-		return ret;
-	}
-
-	IPADBG_LOW("offset = %d size = %d\n", offset.offset, offset.size);
-
-	if (offset.size == 0)
-		return 0;
-
-	mem.size = offset.size;
-	mem.base = dma_alloc_coherent(ipa3_ctx->pdev,
-		mem.size,
-		&mem.phys_base,
-		GFP_KERNEL);
-	if (!mem.base) {
-		IPAERR("fail to alloc DMA memory\n");
-		return ret;
-	}
-
-	cmd.is_read = true;
-	cmd.clear_after_read = true;
-	cmd.skip_pipeline_clear = false;
-	cmd.pipeline_clear_options = IPAHAL_HPS_CLEAR;
-	cmd.size = mem.size;
-	cmd.system_addr = mem.phys_base;
-	cmd.local_addr = ipa3_ctx->smem_restricted_bytes +
-		IPA_MEM_PART(stats_drop_ofst) + offset.offset;
-	cmd_pyld = ipahal_construct_imm_cmd(
-		IPA_IMM_CMD_DMA_SHARED_MEM, &cmd, false);
-	if (!cmd_pyld) {
-		IPAERR("failed to construct dma_shared_mem imm cmd\n");
-		ret = -ENOMEM;
-		goto free_dma_mem;
-	}
-	desc.opcode = cmd_pyld->opcode;
-	desc.pyld = cmd_pyld->data;
-	desc.len = cmd_pyld->len;
-	desc.type = IPA_IMM_CMD_DESC;
-
-	ret = ipa3_send_cmd(1, &desc);
-	if (ret) {
-		IPAERR("failed to send immediate command (error %d)\n", ret);
-		goto destroy_imm;
-	}
-
-	stats = kzalloc(sizeof(*stats), GFP_KERNEL);
-	if (!stats) {
-		ret = -ENOMEM;
-		goto destroy_imm;
-	}
-
-	ret = ipahal_parse_stats(IPAHAL_HW_STATS_DROP,
-		&ipa3_ctx->hw_stats.drop.init, mem.base, stats);
-	if (ret) {
-		IPAERR("failed to parse stats (error %d)\n", ret);
-		goto free_stats;
-	}
-
-	/*
-	 * update driver cache.
-	 * the stats were read from hardware with clear_after_read meaning
-	 * hardware stats are 0 now
-	 */
-	for (i = 0; i < IPA_CLIENT_MAX; i++) {
-		int ep_idx = ipa3_get_ep_mapping(i);
-
-		if (ep_idx == -1 || ep_idx >= IPA3_MAX_NUM_PIPES)
-			continue;
-
-		if (ipa3_ctx->ep[ep_idx].client != i)
-			continue;
-
-		ipa3_ctx->hw_stats.drop.stats.client[i].drop_byte_cnt +=
-			stats->stats[ep_idx].drop_byte_cnt;
-		ipa3_ctx->hw_stats.drop.stats.client[i].drop_packet_cnt +=
-			stats->stats[ep_idx].drop_packet_cnt;
-	}
-
-
-	if (!out) {
-		ret = 0;
-		goto free_stats;
-	}
-
-	/* copy results to out parameter */
-	*out = ipa3_ctx->hw_stats.drop.stats;
-
-	ret = 0;
-free_stats:
-	kfree(stats);
-destroy_imm:
-	ipahal_destroy_imm_cmd(cmd_pyld);
-free_dma_mem:
-	dma_free_coherent(ipa3_ctx->pdev, mem.size, mem.base, mem.phys_base);
-	return ret;
-
-}
-
-int ipa_reset_drop_stats(enum ipa_client_type client)
-{
-	int ret;
-	struct ipa_drop_stats *stats;
-
-	if (client >= IPA_CLIENT_MAX) {
-		IPAERR("invalid client %d\n", client);
-		return -EINVAL;
-	}
-
-	/* reading stats will reset them in hardware */
-	ret = ipa_get_drop_stats(NULL);
-	if (ret) {
-		IPAERR("ipa_get_drop_stats failed %d\n", ret);
-		return ret;
-	}
-
-	/* reset driver's cache */
-	stats = &ipa3_ctx->hw_stats.drop.stats.client[client];
-	memset(stats, 0, sizeof(*stats));
-	return 0;
-}
-
-int ipa_reset_all_drop_stats(void)
-{
-	int ret;
-	struct ipa_drop_stats_all *stats;
-
-	if (!ipa3_ctx->hw_stats.enabled)
-		return 0;
-
-	/* reading stats will reset them in hardware */
-	ret = ipa_get_drop_stats(NULL);
-	if (ret) {
-		IPAERR("ipa_get_drop_stats failed %d\n", ret);
-		return ret;
-	}
-
-	/* reset driver's cache */
-	stats = &ipa3_ctx->hw_stats.drop.stats;
-	memset(stats, 0, sizeof(*stats));
-	return 0;
-}
-
-
-#ifndef CONFIG_DEBUG_FS
-int ipa_debugfs_init_stats(struct dentry *parent) { return 0; }
-#else
-#define IPA_MAX_MSG_LEN 4096
-static char dbg_buff[IPA_MAX_MSG_LEN];
-
-static ssize_t ipa_debugfs_reset_quota_stats(struct file *file,
-	const char __user *ubuf, size_t count, loff_t *ppos)
-{
-	s8 client = 0;
-	int ret;
-
-	mutex_lock(&ipa3_ctx->lock);
-
-	ret = kstrtos8_from_user(ubuf, count, 0, &client);
-	if (ret)
-		goto bail;
-
-	if (client == -1)
-		ipa_reset_all_quota_stats();
-	else
-		ipa_reset_quota_stats(client);
-
-	ret = count;
-bail:
-	mutex_unlock(&ipa3_ctx->lock);
-	return ret;
-}
-
-static ssize_t ipa_debugfs_print_quota_stats(struct file *file,
-	char __user *ubuf, size_t count, loff_t *ppos)
-{
-	int nbytes = 0;
-	struct ipa_quota_stats_all *out;
-	int i;
-	int res;
-
-	out = kzalloc(sizeof(*out), GFP_KERNEL);
-	if (!out)
-		return -ENOMEM;
-
-	mutex_lock(&ipa3_ctx->lock);
-	res = ipa_get_quota_stats(out);
-	if (res) {
-		mutex_unlock(&ipa3_ctx->lock);
-		kfree(out);
-		return res;
-	}
-	for (i = 0; i < IPA_CLIENT_MAX; i++) {
-		int ep_idx = ipa3_get_ep_mapping(i);
-
-		if (ep_idx == -1)
-			continue;
-
-		if (IPA_CLIENT_IS_TEST(i))
-			continue;
-
-		if (!(ipa3_ctx->hw_stats.quota.init.enabled_bitmask &
-			(1 << ep_idx)))
-			continue;
-
-		nbytes += scnprintf(dbg_buff + nbytes,
-			IPA_MAX_MSG_LEN - nbytes,
-			"%s:\n",
-			ipa_clients_strings[i]);
-		nbytes += scnprintf(dbg_buff + nbytes,
-			IPA_MAX_MSG_LEN - nbytes,
-			"num_ipv4_bytes=%llu\n",
-			out->client[i].num_ipv4_bytes);
-		nbytes += scnprintf(dbg_buff + nbytes,
-			IPA_MAX_MSG_LEN - nbytes,
-			"num_ipv6_bytes=%llu\n",
-			out->client[i].num_ipv6_bytes);
-		nbytes += scnprintf(dbg_buff + nbytes,
-			IPA_MAX_MSG_LEN - nbytes,
-			"num_ipv4_pkts=%u\n",
-			out->client[i].num_ipv4_pkts);
-		nbytes += scnprintf(dbg_buff + nbytes,
-			IPA_MAX_MSG_LEN - nbytes,
-			"num_ipv6_pkts=%u\n",
-			out->client[i].num_ipv6_pkts);
-		nbytes += scnprintf(dbg_buff + nbytes,
-			IPA_MAX_MSG_LEN - nbytes,
-			"\n");
-
-	}
-	mutex_unlock(&ipa3_ctx->lock);
-	kfree(out);
-
-	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, nbytes);
-}
-
-static ssize_t ipa_debugfs_reset_tethering_stats(struct file *file,
-	const char __user *ubuf, size_t count, loff_t *ppos)
-{
-	s8 client = 0;
-	int ret;
-
-	mutex_lock(&ipa3_ctx->lock);
-
-	ret = kstrtos8_from_user(ubuf, count, 0, &client);
-	if (ret)
-		goto bail;
-
-	if (client == -1)
-		ipa_reset_all_teth_stats();
-	else
-		ipa_reset_all_cons_teth_stats(client);
-
-	ret = count;
-bail:
-	mutex_unlock(&ipa3_ctx->lock);
-	return ret;
-}
-
-static ssize_t ipa_debugfs_print_tethering_stats(struct file *file,
-	char __user *ubuf, size_t count, loff_t *ppos)
-{
-	int nbytes = 0;
-	struct ipa_quota_stats_all *out;
-	int i, j;
-	int res;
-
-	out = kzalloc(sizeof(*out), GFP_KERNEL);
-	if (!out)
-		return -ENOMEM;
-
-	mutex_lock(&ipa3_ctx->lock);
-	for (i = 0; i < IPA_CLIENT_MAX; i++) {
-		int ep_idx = ipa3_get_ep_mapping(i);
-
-		if (ep_idx == -1)
-			continue;
-
-		if (!IPA_CLIENT_IS_PROD(i))
-			continue;
-
-		if (IPA_CLIENT_IS_TEST(i))
-			continue;
-
-		if (!(ipa3_ctx->hw_stats.teth.init.prod_bitmask &
-			(1 << ep_idx)))
-			continue;
-
-		res = ipa_get_teth_stats();
-		if (res) {
-			mutex_unlock(&ipa3_ctx->lock);
-			kfree(out);
-			return res;
-		}
-
-		for (j = 0; j < IPA_CLIENT_MAX; j++) {
-			int cons_idx = ipa3_get_ep_mapping(j);
-
-			if (cons_idx == -1)
-				continue;
-
-			if (IPA_CLIENT_IS_TEST(j))
-				continue;
-
-			if (!(ipa3_ctx->hw_stats.teth.init.cons_bitmask[ep_idx]
-				& (1 << cons_idx)))
-				continue;
-
-			nbytes += scnprintf(dbg_buff + nbytes,
-				IPA_MAX_MSG_LEN - nbytes,
-				"%s->%s:\n",
-				ipa_clients_strings[i],
-				ipa_clients_strings[j]);
-			nbytes += scnprintf(dbg_buff + nbytes,
-				IPA_MAX_MSG_LEN - nbytes,
-				"num_ipv4_bytes=%llu\n",
-				out->client[j].num_ipv4_bytes);
-			nbytes += scnprintf(dbg_buff + nbytes,
-				IPA_MAX_MSG_LEN - nbytes,
-				"num_ipv6_bytes=%llu\n",
-				out->client[j].num_ipv6_bytes);
-			nbytes += scnprintf(dbg_buff + nbytes,
-				IPA_MAX_MSG_LEN - nbytes,
-				"num_ipv4_pkts=%u\n",
-				out->client[j].num_ipv4_pkts);
-			nbytes += scnprintf(dbg_buff + nbytes,
-				IPA_MAX_MSG_LEN - nbytes,
-				"num_ipv6_pkts=%u\n",
-				out->client[j].num_ipv6_pkts);
-			nbytes += scnprintf(dbg_buff + nbytes,
-				IPA_MAX_MSG_LEN - nbytes,
-				"\n");
-		}
-	}
-	mutex_unlock(&ipa3_ctx->lock);
-	kfree(out);
-
-	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, nbytes);
-}
-
-static ssize_t ipa_debugfs_control_flt_rt_stats(struct file *file,
-	const char __user *ubuf, size_t count, loff_t *ppos)
-{
-	struct ipa_ioc_flt_rt_query *query;
-	unsigned long missing;
-	int pyld_size = 0;
-	int ret;
-
-	query = kzalloc(sizeof(struct ipa_ioc_flt_rt_query),
-		GFP_KERNEL);
-	if (!query)
-		return -ENOMEM;
-	query->stats_size = sizeof(struct ipa_flt_rt_stats);
-	pyld_size = IPA_MAX_FLT_RT_CNT_INDEX *
-		sizeof(struct ipa_flt_rt_stats);
-	query->stats = (uint64_t)kzalloc(pyld_size, GFP_KERNEL);
-	if (!query->stats) {
-		kfree(query);
-		return -ENOMEM;
-	}
-
-	mutex_lock(&ipa3_ctx->lock);
-	if (count >= sizeof(dbg_buff)) {
-		ret = -EFAULT;
-		goto bail;
-	}
-
-	missing = copy_from_user(dbg_buff, ubuf, count);
-	if (missing) {
-		ret = -EFAULT;
-		goto bail;
-	}
-
-	dbg_buff[count] = '\0';
-	if (strcmp(dbg_buff, "reset\n") == 0) {
-		query->reset = 1;
-		query->start_id = 1;
-		query->end_id = IPA_MAX_FLT_RT_CNT_INDEX;
-		ipa_get_flt_rt_stats(query);
-	} else {
-		IPAERR("unsupport flt_rt command\n");
-	}
-
-	ret = count;
-bail:
-	kfree((void *)(uintptr_t)(query->stats));
-	kfree(query);
-	mutex_unlock(&ipa3_ctx->lock);
-	return ret;
-}
-
-static ssize_t ipa_debugfs_print_flt_rt_stats(struct file *file,
-	char __user *ubuf, size_t count, loff_t *ppos)
-{
-	int nbytes = 0;
-	int i;
-	int res;
-	int pyld_size = 0;
-	struct ipa_ioc_flt_rt_query *query;
-
-	query = kzalloc(sizeof(struct ipa_ioc_flt_rt_query),
-		GFP_KERNEL);
-	if (!query)
-		return -ENOMEM;
-	query->start_id = 1;
-	query->end_id = IPA_MAX_FLT_RT_CNT_INDEX;
-	query->reset = true;
-	query->stats_size = sizeof(struct ipa_flt_rt_stats);
-	pyld_size = IPA_MAX_FLT_RT_CNT_INDEX *
-		sizeof(struct ipa_flt_rt_stats);
-	query->stats = (uint64_t)kzalloc(pyld_size, GFP_KERNEL);
-	if (!query->stats) {
-		kfree(query);
-		return -ENOMEM;
-	}
-	mutex_lock(&ipa3_ctx->lock);
-	res = ipa_get_flt_rt_stats(query);
-	if (res) {
-		mutex_unlock(&ipa3_ctx->lock);
-		kfree((void *)(uintptr_t)(query->stats));
-		kfree(query);
-		return res;
-	}
-	for (i = 0; i < IPA_MAX_FLT_RT_CNT_INDEX; i++) {
-		nbytes += scnprintf(dbg_buff + nbytes,
-			IPA_MAX_MSG_LEN - nbytes,
-			"cnt_id: %d\n", i + 1);
-		nbytes += scnprintf(dbg_buff + nbytes,
-			IPA_MAX_MSG_LEN - nbytes,
-			"num_pkts: %d\n",
-			((struct ipa_flt_rt_stats *)
-			query->stats)[i].num_pkts);
-		nbytes += scnprintf(dbg_buff + nbytes,
-			IPA_MAX_MSG_LEN - nbytes,
-			"num_pkts_hash: %d\n",
-			((struct ipa_flt_rt_stats *)
-			query->stats)[i].num_pkts_hash);
-		nbytes += scnprintf(dbg_buff + nbytes,
-			IPA_MAX_MSG_LEN - nbytes,
-			"num_bytes: %lld\n",
-			((struct ipa_flt_rt_stats *)
-			query->stats)[i].num_bytes);
-		nbytes += scnprintf(dbg_buff + nbytes,
-			IPA_MAX_MSG_LEN - nbytes,
-			"\n");
-	}
-	mutex_unlock(&ipa3_ctx->lock);
-	kfree((void *)(uintptr_t)(query->stats));
-	kfree(query);
-	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, nbytes);
-}
-
-static ssize_t ipa_debugfs_reset_drop_stats(struct file *file,
-	const char __user *ubuf, size_t count, loff_t *ppos)
-{
-	s8 client = 0;
-	int ret;
-
-	mutex_lock(&ipa3_ctx->lock);
-
-	ret = kstrtos8_from_user(ubuf, count, 0, &client);
-	if (ret)
-		goto bail;
-
-	if (client == -1)
-		ipa_reset_all_drop_stats();
-	else
-		ipa_reset_drop_stats(client);
-
-	ret = count;
-bail:
-	mutex_unlock(&ipa3_ctx->lock);
-	return count;
-}
-
-static ssize_t ipa_debugfs_print_drop_stats(struct file *file,
-	char __user *ubuf, size_t count, loff_t *ppos)
-{
-	int nbytes = 0;
-	struct ipa_drop_stats_all *out;
-	int i;
-	int res;
-
-	out = kzalloc(sizeof(*out), GFP_KERNEL);
-	if (!out)
-		return -ENOMEM;
-
-	mutex_lock(&ipa3_ctx->lock);
-	res = ipa_get_drop_stats(out);
-	if (res) {
-		mutex_unlock(&ipa3_ctx->lock);
-		kfree(out);
-		return res;
-	}
-
-	for (i = 0; i < IPA_CLIENT_MAX; i++) {
-		int ep_idx = ipa3_get_ep_mapping(i);
-
-		if (ep_idx == -1)
-			continue;
-
-		if (!IPA_CLIENT_IS_CONS(i))
-			continue;
-
-		if (IPA_CLIENT_IS_TEST(i))
-			continue;
-
-		if (!(ipa3_ctx->hw_stats.drop.init.enabled_bitmask &
-			(1 << ep_idx)))
-			continue;
-
-
-		nbytes += scnprintf(dbg_buff + nbytes,
-			IPA_MAX_MSG_LEN - nbytes,
-			"%s:\n",
-			ipa_clients_strings[i]);
-
-		nbytes += scnprintf(dbg_buff + nbytes,
-			IPA_MAX_MSG_LEN - nbytes,
-			"drop_byte_cnt=%u\n",
-			out->client[i].drop_byte_cnt);
-
-		nbytes += scnprintf(dbg_buff + nbytes,
-			IPA_MAX_MSG_LEN - nbytes,
-			"drop_packet_cnt=%u\n",
-			out->client[i].drop_packet_cnt);
-		nbytes += scnprintf(dbg_buff + nbytes,
-			IPA_MAX_MSG_LEN - nbytes,
-			"\n");
-	}
-	mutex_unlock(&ipa3_ctx->lock);
-	kfree(out);
-
-	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, nbytes);
-}
-
-static const struct file_operations ipa3_quota_ops = {
-	.read = ipa_debugfs_print_quota_stats,
-	.write = ipa_debugfs_reset_quota_stats,
-};
-
-static const struct file_operations ipa3_tethering_ops = {
-	.read = ipa_debugfs_print_tethering_stats,
-	.write = ipa_debugfs_reset_tethering_stats,
-};
-
-static const struct file_operations ipa3_flt_rt_ops = {
-	.read = ipa_debugfs_print_flt_rt_stats,
-	.write = ipa_debugfs_control_flt_rt_stats,
-};
-
-static const struct file_operations ipa3_drop_ops = {
-	.read = ipa_debugfs_print_drop_stats,
-	.write = ipa_debugfs_reset_drop_stats,
-};
-
-
-int ipa_debugfs_init_stats(struct dentry *parent)
-{
-	const mode_t read_write_mode = 0664;
-	struct dentry *file;
-	struct dentry *dent;
-
-	if (!ipa3_ctx->hw_stats.enabled)
-		return 0;
-
-	dent = debugfs_create_dir("hw_stats", parent);
-	if (IS_ERR_OR_NULL(dent)) {
-		IPAERR("fail to create folder in debug_fs\n");
-		return -EFAULT;
-	}
-
-	file = debugfs_create_file("quota", read_write_mode, dent, NULL,
-		&ipa3_quota_ops);
-	if (IS_ERR_OR_NULL(file)) {
-		IPAERR("fail to create file %s\n", "quota");
-		goto fail;
-	}
-
-	file = debugfs_create_file("drop", read_write_mode, dent, NULL,
-		&ipa3_drop_ops);
-	if (IS_ERR_OR_NULL(file)) {
-		IPAERR("fail to create file %s\n", "drop");
-		goto fail;
-	}
-
-	file = debugfs_create_file("tethering", read_write_mode, dent, NULL,
-		&ipa3_tethering_ops);
-	if (IS_ERR_OR_NULL(file)) {
-		IPAERR("fail to create file %s\n", "tethering");
-		goto fail;
-	}
-
-	file = debugfs_create_file("flt_rt", read_write_mode, dent, NULL,
-		&ipa3_flt_rt_ops);
-	if (IS_ERR_OR_NULL(file)) {
-		IPAERR("fail to create file flt_rt\n");
-		goto fail;
-	}
-
-	return 0;
-fail:
-	debugfs_remove_recursive(dent);
-	return -EFAULT;
-}
-#endif

ipa/ipa_v3/ipa_i.h

@@ -1,3042 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2012-2019, The Linux Foundation. All rights reserved.
- */
-
-#ifndef _IPA3_I_H_
-#define _IPA3_I_H_
-
-#include <linux/bitops.h>
-#include <linux/cdev.h>
-#include <linux/export.h>
-#include <linux/idr.h>
-#include <linux/list.h>
-#include <linux/mutex.h>
-#include <linux/skbuff.h>
-#include <linux/slab.h>
-#include <linux/notifier.h>
-#include <linux/interrupt.h>
-#include <linux/ipa.h>
-#include <linux/ipa_usb.h>
-#include <asm/dma-iommu.h>
-#include <linux/iommu.h>
-#include <linux/platform_device.h>
-#include <linux/firmware.h>
-#include "ipa_qmi_service.h"
-#include "../ipa_api.h"
-#include "ipahal/ipahal_reg.h"
-#include "ipahal/ipahal.h"
-#include "ipahal/ipahal_fltrt.h"
-#include "ipahal/ipahal_hw_stats.h"
-#include "../ipa_common_i.h"
-#include "ipa_uc_offload_i.h"
-#include "ipa_pm.h"
-#include "ipa_defs.h"
-#include <linux/mailbox_client.h>
-#include <linux/mailbox/qmp.h>
-#include <linux/rmnet_ipa_fd_ioctl.h>
-
-#define IPA_DEV_NAME_MAX_LEN 15
-#define DRV_NAME "ipa"
-
-#define IPA_COOKIE 0x57831603
-#define IPA_RT_RULE_COOKIE 0x57831604
-#define IPA_RT_TBL_COOKIE 0x57831605
-#define IPA_FLT_COOKIE 0x57831606
-#define IPA_HDR_COOKIE 0x57831607
-#define IPA_PROC_HDR_COOKIE 0x57831608
-
-#define MTU_BYTE 1500
-
-#define IPA_EP_NOT_ALLOCATED (-1)
-#define IPA3_MAX_NUM_PIPES 31
-#define IPA_SYS_DESC_FIFO_SZ 0x800
-#define IPA_SYS_TX_DATA_DESC_FIFO_SZ 0x1000
-#define IPA_COMMON_EVENT_RING_SIZE 0x7C00
-#define IPA_LAN_RX_HEADER_LENGTH (2)
-#define IPA_QMAP_HEADER_LENGTH (4)
-#define IPA_DL_CHECKSUM_LENGTH (8)
-#define IPA_NUM_DESC_PER_SW_TX (3)
-#define IPA_GENERIC_RX_POOL_SZ 192
-#define IPA_UC_FINISH_MAX 6
-#define IPA_UC_WAIT_MIN_SLEEP 1000
-#define IPA_UC_WAII_MAX_SLEEP 1200
-/*
- * The transport descriptor size was changed to GSI_CHAN_RE_SIZE_16B, but
- * IPA users still use sps_iovec size as FIFO element size.
- */
-#define IPA_FIFO_ELEMENT_SIZE 8
-
-#define IPA_MAX_STATUS_STAT_NUM 30
-
-#define IPA_IPC_LOG_PAGES 50
-
-#define IPA_MAX_NUM_REQ_CACHE 10
-
-#define NAPI_WEIGHT 60
-
-#define IPADBG(fmt, args...) \
-	do { \
-		pr_debug(DRV_NAME " %s:%d " fmt, __func__, __LINE__, ## args);\
-		if (ipa3_ctx) { \
-			IPA_IPC_LOGGING(ipa3_ctx->logbuf, \
-				DRV_NAME " %s:%d " fmt, ## args); \
-			IPA_IPC_LOGGING(ipa3_ctx->logbuf_low, \
-				DRV_NAME " %s:%d " fmt, ## args); \
-		} \
-	} while (0)
-
-#define IPADBG_LOW(fmt, args...) \
-	do { \
-		pr_debug(DRV_NAME " %s:%d " fmt, __func__, __LINE__, ## args);\
-		if (ipa3_ctx) \
-			IPA_IPC_LOGGING(ipa3_ctx->logbuf_low, \
-				DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-#define IPAERR(fmt, args...) \
-	do { \
-		pr_err(DRV_NAME " %s:%d " fmt, __func__, __LINE__, ## args);\
-		if (ipa3_ctx) { \
-			IPA_IPC_LOGGING(ipa3_ctx->logbuf, \
-				DRV_NAME " %s:%d " fmt, ## args); \
-			IPA_IPC_LOGGING(ipa3_ctx->logbuf_low, \
-				DRV_NAME " %s:%d " fmt, ## args); \
-		} \
-	} while (0)
-
-#define IPAERR_RL(fmt, args...) \
-	do { \
-		pr_err_ratelimited_ipa(DRV_NAME " %s:%d " fmt, __func__,\
-		__LINE__, ## args);\
-		if (ipa3_ctx) { \
-			IPA_IPC_LOGGING(ipa3_ctx->logbuf, \
-				DRV_NAME " %s:%d " fmt, ## args); \
-			IPA_IPC_LOGGING(ipa3_ctx->logbuf_low, \
-				DRV_NAME " %s:%d " fmt, ## args); \
-		} \
-	} while (0)
-
-#define IPALOG_VnP_ADDRS(ptr) \
-	do { \
-		phys_addr_t b = (phys_addr_t) virt_to_phys(ptr); \
-		IPAERR("%s: VIRT: %pK PHYS: %pa\n", \
-			   #ptr, ptr, &b); \
-	} while (0)
-
-/* round addresses for closes page per SMMU requirements */
-#define IPA_SMMU_ROUND_TO_PAGE(iova, pa, size, iova_p, pa_p, size_p) \
-	do { \
-		(iova_p) = rounddown((iova), PAGE_SIZE); \
-		(pa_p) = rounddown((pa), PAGE_SIZE); \
-		(size_p) = roundup((size) + (pa) - (pa_p), PAGE_SIZE); \
-	} while (0)
-
-#define WLAN_AMPDU_TX_EP 15
-#define WLAN_PROD_TX_EP  19
-#define WLAN1_CONS_RX_EP  14
-#define WLAN2_CONS_RX_EP  16
-#define WLAN3_CONS_RX_EP  17
-#define WLAN4_CONS_RX_EP  18
-
-#define IPA_RAM_NAT_OFST    0
-#define IPA_RAM_NAT_SIZE    0
-#define IPA_RAM_IPV6CT_OFST 0
-#define IPA_RAM_IPV6CT_SIZE 0
-#define IPA_MEM_CANARY_VAL 0xdeadbeef
-
-#define IPA_STATS
-
-#ifdef IPA_STATS
-#define IPA_STATS_INC_CNT(val) (++val)
-#define IPA_STATS_DEC_CNT(val) (--val)
-#define IPA_STATS_EXCP_CNT(__excp, __base) do {				\
-	if (__excp < 0 || __excp >= IPAHAL_PKT_STATUS_EXCEPTION_MAX)	\
-		break;							\
-	++__base[__excp];						\
-	} while (0)
-#else
-#define IPA_STATS_INC_CNT(x) do { } while (0)
-#define IPA_STATS_DEC_CNT(x)
-#define IPA_STATS_EXCP_CNT(__excp, __base) do { } while (0)
-#endif
-
-#define IPA_HDR_BIN0 0
-#define IPA_HDR_BIN1 1
-#define IPA_HDR_BIN2 2
-#define IPA_HDR_BIN3 3
-#define IPA_HDR_BIN4 4
-#define IPA_HDR_BIN_MAX 5
-
-#define IPA_HDR_PROC_CTX_BIN0 0
-#define IPA_HDR_PROC_CTX_BIN1 1
-#define IPA_HDR_PROC_CTX_BIN_MAX 2
-
-#define IPA_RX_POOL_CEIL 32
-#define IPA_RX_SKB_SIZE 1792
-
-#define IPA_A5_MUX_HDR_NAME "ipa_excp_hdr"
-#define IPA_LAN_RX_HDR_NAME "ipa_lan_hdr"
-#define IPA_INVALID_L4_PROTOCOL 0xFF
-
-#define IPA_HDR_PROC_CTX_TABLE_ALIGNMENT_BYTE 8
-#define IPA_HDR_PROC_CTX_TABLE_ALIGNMENT(start_ofst) \
-	(((start_ofst) + IPA_HDR_PROC_CTX_TABLE_ALIGNMENT_BYTE - 1) & \
-	~(IPA_HDR_PROC_CTX_TABLE_ALIGNMENT_BYTE - 1))
-
-#define MAX_RESOURCE_TO_CLIENTS (IPA_CLIENT_MAX)
-#define IPA_MEM_PART(x_) (ipa3_ctx->ctrl->mem_partition->x_)
-
-#define IPA_GSI_CHANNEL_STOP_MAX_RETRY 10
-#define IPA_GSI_CHANNEL_STOP_PKT_SIZE 1
-
-#define IPA_GSI_CHANNEL_EMPTY_MAX_RETRY 15
-#define IPA_GSI_CHANNEL_EMPTY_SLEEP_MIN_USEC (1000)
-#define IPA_GSI_CHANNEL_EMPTY_SLEEP_MAX_USEC (2000)
-
-#define IPA_SLEEP_CLK_RATE_KHZ (32)
-
-#define IPA3_ACTIVE_CLIENTS_LOG_BUFFER_SIZE_LINES 120
-#define IPA3_ACTIVE_CLIENTS_LOG_LINE_LEN 96
-#define IPA3_ACTIVE_CLIENTS_LOG_HASHTABLE_SIZE 50
-#define IPA3_ACTIVE_CLIENTS_LOG_NAME_LEN 40
-#define SMEM_IPA_FILTER_TABLE 497
-
-enum {
-	SMEM_APPS,
-	SMEM_MODEM,
-	SMEM_Q6,
-	SMEM_DSPS,
-	SMEM_WCNSS,
-	SMEM_CDSP,
-	SMEM_RPM,
-	SMEM_TZ,
-	SMEM_SPSS,
-	SMEM_HYP,
-	NUM_SMEM_SUBSYSTEMS,
-};
-
-#define IPA_WDI_RX_RING_RES			0
-#define IPA_WDI_RX_RING_RP_RES		1
-#define IPA_WDI_RX_COMP_RING_RES	2
-#define IPA_WDI_RX_COMP_RING_WP_RES	3
-#define IPA_WDI_TX_RING_RES			4
-#define IPA_WDI_CE_RING_RES			5
-#define IPA_WDI_CE_DB_RES			6
-#define IPA_WDI_TX_DB_RES			7
-#define IPA_WDI_MAX_RES				8
-
-/* use QMAP header reserved bit to identify tethered traffic */
-#define IPA_QMAP_TETH_BIT (1 << 30)
-
-#ifdef CONFIG_ARM64
-/* Outer caches unsupported on ARM64 platforms */
-# define outer_flush_range(x, y)
-# define __cpuc_flush_dcache_area __flush_dcache_area
-#endif
-
-#define IPA_SMP2P_OUT_CLK_RSP_CMPLT_IDX 0
-#define IPA_SMP2P_OUT_CLK_VOTE_IDX 1
-#define IPA_SMP2P_SMEM_STATE_MASK 3
-
-
-#define IPA_SUMMING_THRESHOLD (0x10)
-#define IPA_PIPE_MEM_START_OFST (0x0)
-#define IPA_PIPE_MEM_SIZE (0x0)
-#define IPA_MOBILE_AP_MODE(x) (x == IPA_MODE_MOBILE_AP_ETH || \
-				   x == IPA_MODE_MOBILE_AP_WAN || \
-				   x == IPA_MODE_MOBILE_AP_WLAN)
-#define IPA_CNOC_CLK_RATE (75 * 1000 * 1000UL)
-#define IPA_A5_MUX_HEADER_LENGTH (8)
-
-#define IPA_AGGR_MAX_STR_LENGTH (10)
-
-#define CLEANUP_TAG_PROCESS_TIMEOUT 1000
-
-#define IPA_AGGR_STR_IN_BYTES(str) \
-	(strnlen((str), IPA_AGGR_MAX_STR_LENGTH - 1) + 1)
-
-#define IPA_ADJUST_AGGR_BYTE_HARD_LIMIT(X) (X/1000)
-
-#define IPA_TRANSPORT_PROD_TIMEOUT_MSEC 100
-
-#define IPA3_ACTIVE_CLIENTS_TABLE_BUF_SIZE 4096
-
-#define IPA3_ACTIVE_CLIENT_LOG_TYPE_EP 0
-#define IPA3_ACTIVE_CLIENT_LOG_TYPE_SIMPLE 1
-#define IPA3_ACTIVE_CLIENT_LOG_TYPE_RESOURCE 2
-#define IPA3_ACTIVE_CLIENT_LOG_TYPE_SPECIAL 3
-
-#define IPA_MHI_GSI_EVENT_RING_ID_START 10
-#define IPA_MHI_GSI_EVENT_RING_ID_END 12
-
-#define IPA_SMEM_SIZE (8 * 1024)
-
-#define IPA_GSI_CHANNEL_HALT_MIN_SLEEP 5000
-#define IPA_GSI_CHANNEL_HALT_MAX_SLEEP 10000
-#define IPA_GSI_CHANNEL_HALT_MAX_TRY 10
-
-/* round addresses for closes page per SMMU requirements */
-#define IPA_SMMU_ROUND_TO_PAGE(iova, pa, size, iova_p, pa_p, size_p) \
-	do { \
-		(iova_p) = rounddown((iova), PAGE_SIZE); \
-		(pa_p) = rounddown((pa), PAGE_SIZE); \
-		(size_p) = roundup((size) + (pa) - (pa_p), PAGE_SIZE); \
-	} while (0)
-
-
-/* The relative location in /lib/firmware where the FWs will reside */
-#define IPA_FWS_PATH "ipa/ipa_fws.elf"
-/*
- * The following paths below are used when building the system for the
- * emulation environment.
- *
- * As new hardware platforms are added into the emulation environment,
- * please add the appropriate paths here for their firmwares.
- */
-#define IPA_FWS_PATH_4_0     "ipa/4.0/ipa_fws.elf"
-#define IPA_FWS_PATH_3_5_1   "ipa/3.5.1/ipa_fws.elf"
-#define IPA_FWS_PATH_4_5     "ipa/4.5/ipa_fws.elf"
-
-/*
- * The following will be used for determining/using access control
- * policy.
- */
-#define USE_SCM            0 /* use scm call to determine policy */
-#define OVERRIDE_SCM_TRUE  1 /* override scm call with true */
-#define OVERRIDE_SCM_FALSE 2 /* override scm call with false */
-
-#define SD_ENABLED  0 /* secure debug enabled. */
-#define SD_DISABLED 1 /* secure debug disabled. */
-
-#define IPA_MEM_INIT_VAL 0xFFFFFFFF
-
-#ifdef CONFIG_COMPAT
-#define IPA_IOC_ADD_HDR32 _IOWR(IPA_IOC_MAGIC, \
-					IPA_IOCTL_ADD_HDR, \
-					compat_uptr_t)
-#define IPA_IOC_DEL_HDR32 _IOWR(IPA_IOC_MAGIC, \
-					IPA_IOCTL_DEL_HDR, \
-					compat_uptr_t)
-#define IPA_IOC_ADD_RT_RULE32 _IOWR(IPA_IOC_MAGIC, \
-					IPA_IOCTL_ADD_RT_RULE, \
-					compat_uptr_t)
-#define IPA_IOC_DEL_RT_RULE32 _IOWR(IPA_IOC_MAGIC, \
-					IPA_IOCTL_DEL_RT_RULE, \
-					compat_uptr_t)
-#define IPA_IOC_ADD_FLT_RULE32 _IOWR(IPA_IOC_MAGIC, \
-					IPA_IOCTL_ADD_FLT_RULE, \
-					compat_uptr_t)
-#define IPA_IOC_DEL_FLT_RULE32 _IOWR(IPA_IOC_MAGIC, \
-					IPA_IOCTL_DEL_FLT_RULE, \
-					compat_uptr_t)
-#define IPA_IOC_GET_RT_TBL32 _IOWR(IPA_IOC_MAGIC, \
-				IPA_IOCTL_GET_RT_TBL, \
-				compat_uptr_t)
-#define IPA_IOC_COPY_HDR32 _IOWR(IPA_IOC_MAGIC, \
-				IPA_IOCTL_COPY_HDR, \
-				compat_uptr_t)
-#define IPA_IOC_QUERY_INTF32 _IOWR(IPA_IOC_MAGIC, \
-				IPA_IOCTL_QUERY_INTF, \
-				compat_uptr_t)
-#define IPA_IOC_QUERY_INTF_TX_PROPS32 _IOWR(IPA_IOC_MAGIC, \
-				IPA_IOCTL_QUERY_INTF_TX_PROPS, \
-				compat_uptr_t)
-#define IPA_IOC_QUERY_INTF_RX_PROPS32 _IOWR(IPA_IOC_MAGIC, \
-					IPA_IOCTL_QUERY_INTF_RX_PROPS, \
-					compat_uptr_t)
-#define IPA_IOC_QUERY_INTF_EXT_PROPS32 _IOWR(IPA_IOC_MAGIC, \
-					IPA_IOCTL_QUERY_INTF_EXT_PROPS, \
-					compat_uptr_t)
-#define IPA_IOC_GET_HDR32 _IOWR(IPA_IOC_MAGIC, \
-				IPA_IOCTL_GET_HDR, \
-				compat_uptr_t)
-#define IPA_IOC_ALLOC_NAT_MEM32 _IOWR(IPA_IOC_MAGIC, \
-				IPA_IOCTL_ALLOC_NAT_MEM, \
-				compat_uptr_t)
-#define IPA_IOC_ALLOC_NAT_TABLE32 _IOWR(IPA_IOC_MAGIC, \
-				IPA_IOCTL_ALLOC_NAT_TABLE, \
-				compat_uptr_t)
-#define IPA_IOC_ALLOC_IPV6CT_TABLE32 _IOWR(IPA_IOC_MAGIC, \
-				IPA_IOCTL_ALLOC_IPV6CT_TABLE, \
-				compat_uptr_t)
-#define IPA_IOC_V4_INIT_NAT32 _IOWR(IPA_IOC_MAGIC, \
-				IPA_IOCTL_V4_INIT_NAT, \
-				compat_uptr_t)
-#define IPA_IOC_INIT_IPV6CT_TABLE32 _IOWR(IPA_IOC_MAGIC, \
-				IPA_IOCTL_INIT_IPV6CT_TABLE, \
-				compat_uptr_t)
-#define IPA_IOC_TABLE_DMA_CMD32 _IOWR(IPA_IOC_MAGIC, \
-				IPA_IOCTL_TABLE_DMA_CMD, \
-				compat_uptr_t)
-#define IPA_IOC_V4_DEL_NAT32 _IOWR(IPA_IOC_MAGIC, \
-				IPA_IOCTL_V4_DEL_NAT, \
-				compat_uptr_t)
-#define IPA_IOC_DEL_NAT_TABLE32 _IOWR(IPA_IOC_MAGIC, \
-				IPA_IOCTL_DEL_NAT_TABLE, \
-				compat_uptr_t)
-#define IPA_IOC_DEL_IPV6CT_TABLE32 _IOWR(IPA_IOC_MAGIC, \
-				IPA_IOCTL_DEL_IPV6CT_TABLE, \
-				compat_uptr_t)
-#define IPA_IOC_NAT_MODIFY_PDN32 _IOWR(IPA_IOC_MAGIC, \
-				IPA_IOCTL_NAT_MODIFY_PDN, \
-				compat_uptr_t)
-#define IPA_IOC_GET_NAT_OFFSET32 _IOWR(IPA_IOC_MAGIC, \
-				IPA_IOCTL_GET_NAT_OFFSET, \
-				compat_uptr_t)
-#define IPA_IOC_PULL_MSG32 _IOWR(IPA_IOC_MAGIC, \
-				IPA_IOCTL_PULL_MSG, \
-				compat_uptr_t)
-#define IPA_IOC_RM_ADD_DEPENDENCY32 _IOWR(IPA_IOC_MAGIC, \
-				IPA_IOCTL_RM_ADD_DEPENDENCY, \
-				compat_uptr_t)
-#define IPA_IOC_RM_DEL_DEPENDENCY32 _IOWR(IPA_IOC_MAGIC, \
-				IPA_IOCTL_RM_DEL_DEPENDENCY, \
-				compat_uptr_t)
-#define IPA_IOC_GENERATE_FLT_EQ32 _IOWR(IPA_IOC_MAGIC, \
-				IPA_IOCTL_GENERATE_FLT_EQ, \
-				compat_uptr_t)
-#define IPA_IOC_QUERY_RT_TBL_INDEX32 _IOWR(IPA_IOC_MAGIC, \
-				IPA_IOCTL_QUERY_RT_TBL_INDEX, \
-				compat_uptr_t)
-#define IPA_IOC_WRITE_QMAPID32  _IOWR(IPA_IOC_MAGIC, \
-				IPA_IOCTL_WRITE_QMAPID, \
-				compat_uptr_t)
-#define IPA_IOC_MDFY_FLT_RULE32 _IOWR(IPA_IOC_MAGIC, \
-				IPA_IOCTL_MDFY_FLT_RULE, \
-				compat_uptr_t)
-#define IPA_IOC_NOTIFY_WAN_UPSTREAM_ROUTE_ADD32 _IOWR(IPA_IOC_MAGIC, \
-				IPA_IOCTL_NOTIFY_WAN_UPSTREAM_ROUTE_ADD, \
-				compat_uptr_t)
-#define IPA_IOC_NOTIFY_WAN_UPSTREAM_ROUTE_DEL32 _IOWR(IPA_IOC_MAGIC, \
-				IPA_IOCTL_NOTIFY_WAN_UPSTREAM_ROUTE_DEL, \
-				compat_uptr_t)
-#define IPA_IOC_NOTIFY_WAN_EMBMS_CONNECTED32 _IOWR(IPA_IOC_MAGIC, \
-					IPA_IOCTL_NOTIFY_WAN_EMBMS_CONNECTED, \
-					compat_uptr_t)
-#define IPA_IOC_ADD_HDR_PROC_CTX32 _IOWR(IPA_IOC_MAGIC, \
-				IPA_IOCTL_ADD_HDR_PROC_CTX, \
-				compat_uptr_t)
-#define IPA_IOC_DEL_HDR_PROC_CTX32 _IOWR(IPA_IOC_MAGIC, \
-				IPA_IOCTL_DEL_HDR_PROC_CTX, \
-				compat_uptr_t)
-#define IPA_IOC_MDFY_RT_RULE32 _IOWR(IPA_IOC_MAGIC, \
-				IPA_IOCTL_MDFY_RT_RULE, \
-				compat_uptr_t)
-#endif /* #ifdef CONFIG_COMPAT */
-
-#define IPA_TZ_UNLOCK_ATTRIBUTE 0x0C0311
-#define TZ_MEM_PROTECT_REGION_ID 0x10
-
-#define MBOX_TOUT_MS 100
-
-struct ipa3_active_client_htable_entry {
-	struct hlist_node list;
-	char id_string[IPA3_ACTIVE_CLIENTS_LOG_NAME_LEN];
-	int count;
-	enum ipa_active_client_log_type type;
-};
-
-struct ipa3_active_clients_log_ctx {
-	spinlock_t lock;
-	char *log_buffer[IPA3_ACTIVE_CLIENTS_LOG_BUFFER_SIZE_LINES];
-	int log_head;
-	int log_tail;
-	bool log_rdy;
-	struct hlist_head htable[IPA3_ACTIVE_CLIENTS_LOG_HASHTABLE_SIZE];
-};
-
-struct ipa3_client_names {
-	enum ipa_client_type names[MAX_RESOURCE_TO_CLIENTS];
-	int length;
-};
-
-struct ipa_smmu_cb_ctx {
-	bool valid;
-	struct device *dev;
-	struct iommu_domain *iommu_domain;
-	unsigned long next_addr;
-	u32 va_start;
-	u32 va_size;
-	u32 va_end;
-	bool shared;
-	bool is_cache_coherent;
-};
-
-/**
- * struct ipa_flt_rule_add_i - filtering rule descriptor
- * includes in and out parameters
- * @rule: actual rule to be added
- * @at_rear: add at back of filtering table?
- * @flt_rule_hdl: out parameter, handle to rule, valid when status is 0
- * @status:	output parameter, status of filtering rule add   operation,
- *		0 for success,
- *		-1 for failure
- *
- */
-struct ipa_flt_rule_add_i {
-	u8 at_rear;
-	u32 flt_rule_hdl;
-	int status;
-	struct ipa_flt_rule_i rule;
-};
-
-/**
- * struct ipa_flt_rule_mdfy_i - filtering rule descriptor
- * includes in and out parameters
- * @rule: actual rule to be added
- * @flt_rule_hdl: handle to rule
- * @status:	output parameter, status of filtering rule modify  operation,
- *		0 for success,
- *		-1 for failure
- *
- */
-struct ipa_flt_rule_mdfy_i {
-	u32 rule_hdl;
-	int status;
-	struct ipa_flt_rule_i rule;
-};
-
-/**
- * struct ipa_rt_rule_add_i - routing rule descriptor includes
- * in and out parameters
- * @rule: actual rule to be added
- * @at_rear:	add at back of routing table, it is NOT possible to add rules at
- *		the rear of the "default" routing tables
- * @rt_rule_hdl: output parameter, handle to rule, valid when status is 0
- * @status:	output parameter, status of routing rule add operation,
- *		0 for success,
- *		-1 for failure
- */
-struct ipa_rt_rule_add_i {
-	u8 at_rear;
-	u32 rt_rule_hdl;
-	int status;
-	struct ipa_rt_rule_i rule;
-};
-
-/**
- * struct ipa_rt_rule_mdfy_i - routing rule descriptor includes
- * in and out parameters
- * @rule: actual rule to be added
- * @rt_rule_hdl: handle to rule which supposed to modify
- * @status:	output parameter, status of routing rule modify  operation,
- *		0 for success,
- *		-1 for failure
- *
- */
-struct ipa_rt_rule_mdfy_i {
-	u32 rt_rule_hdl;
-	int status;
-	struct ipa_rt_rule_i rule;
-};
-
-/**
- * struct ipa_rt_rule_add_ext_i - routing rule descriptor
- * includes in and out parameters
- * @rule: actual rule to be added
- * @at_rear:	add at back of routing table, it is NOT possible to add rules at
- *		the rear of the "default" routing tables
- * @rt_rule_hdl: output parameter, handle to rule, valid when status is 0
- * @status:	output parameter, status of routing rule add operation,
- * @rule_id: rule_id to be assigned to the routing rule. In case client
- *  specifies rule_id as 0 the driver will assign a new rule_id
- *		0 for success,
- *		-1 for failure
- */
-struct ipa_rt_rule_add_ext_i {
-	uint8_t at_rear;
-	uint32_t rt_rule_hdl;
-	int status;
-	uint16_t rule_id;
-	struct ipa_rt_rule_i rule;
-};
-
-/**
- * struct ipa3_flt_entry - IPA filtering table entry
- * @link: entry's link in global filtering enrties list
- * @rule: filter rule
- * @cookie: cookie used for validity check
- * @tbl: filter table
- * @rt_tbl: routing table
- * @hw_len: entry's size
- * @id: rule handle - globally unique
- * @prio: rule 10bit priority which defines the order of the rule
- *  among other rules at the same integrated table
- * @rule_id: rule 10bit ID to be returned in packet status
- * @cnt_idx: stats counter index
- * @ipacm_installed: indicate if installed by ipacm
- */
-struct ipa3_flt_entry {
-	struct list_head link;
-	u32 cookie;
-	struct ipa_flt_rule_i rule;
-	struct ipa3_flt_tbl *tbl;
-	struct ipa3_rt_tbl *rt_tbl;
-	u32 hw_len;
-	int id;
-	u16 prio;
-	u16 rule_id;
-	u8 cnt_idx;
-	bool ipacm_installed;
-};
-
-/**
- * struct ipa3_rt_tbl - IPA routing table
- * @link: table's link in global routing tables list
- * @head_rt_rule_list: head of routing rules list
- * @name: routing table name
- * @idx: routing table index
- * @rule_cnt: number of rules in routing table
- * @ref_cnt: reference counter of routing table
- * @set: collection of routing tables
- * @cookie: cookie used for validity check
- * @in_sys: flag indicating if the table is located in system memory
- * @sz: the size of the routing table
- * @curr_mem: current routing tables block in sys memory
- * @prev_mem: previous routing table block in sys memory
- * @id: routing table id
- * @rule_ids: common idr structure that holds the rule_id for each rule
- */
-struct ipa3_rt_tbl {
-	struct list_head link;
-	u32 cookie;
-	struct list_head head_rt_rule_list;
-	char name[IPA_RESOURCE_NAME_MAX];
-	u32 idx;
-	u32 rule_cnt;
-	u32 ref_cnt;
-	struct ipa3_rt_tbl_set *set;
-	bool in_sys[IPA_RULE_TYPE_MAX];
-	u32 sz[IPA_RULE_TYPE_MAX];
-	struct ipa_mem_buffer curr_mem[IPA_RULE_TYPE_MAX];
-	struct ipa_mem_buffer prev_mem[IPA_RULE_TYPE_MAX];
-	int id;
-	struct idr *rule_ids;
-};
-
-/**
- * struct ipa3_hdr_entry - IPA header table entry
- * @link: entry's link in global header table entries list
- * @hdr: the header
- * @hdr_len: header length
- * @name: name of header table entry
- * @type: l2 header type
- * @is_partial: flag indicating if header table entry is partial
- * @is_hdr_proc_ctx: false - hdr entry resides in hdr table,
- * true - hdr entry resides in DDR and pointed to by proc ctx
- * @phys_base: physical address of entry in DDR when is_hdr_proc_ctx is true,
- * else 0
- * @proc_ctx: processing context header
- * @offset_entry: entry's offset
- * @cookie: cookie used for validity check
- * @ref_cnt: reference counter of routing table
- * @id: header entry id
- * @is_eth2_ofst_valid: is eth2_ofst field valid?
- * @eth2_ofst: offset to start of Ethernet-II/802.3 header
- * @user_deleted: is the header deleted by the user?
- * @ipacm_installed: indicate if installed by ipacm
- */
-struct ipa3_hdr_entry {
-	struct list_head link;
-	u32 cookie;
-	u8 hdr[IPA_HDR_MAX_SIZE];
-	u32 hdr_len;
-	char name[IPA_RESOURCE_NAME_MAX];
-	enum ipa_hdr_l2_type type;
-	u8 is_partial;
-	bool is_hdr_proc_ctx;
-	dma_addr_t phys_base;
-	struct ipa3_hdr_proc_ctx_entry *proc_ctx;
-	struct ipa_hdr_offset_entry *offset_entry;
-	u32 ref_cnt;
-	int id;
-	u8 is_eth2_ofst_valid;
-	u16 eth2_ofst;
-	bool user_deleted;
-	bool ipacm_installed;
-};
-
-/**
- * struct ipa3_hdr_tbl - IPA header table
- * @head_hdr_entry_list: header entries list
- * @head_offset_list: header offset list
- * @head_free_offset_list: header free offset list
- * @hdr_cnt: number of headers
- * @end: the last header index
- */
-struct ipa3_hdr_tbl {
-	struct list_head head_hdr_entry_list;
-	struct list_head head_offset_list[IPA_HDR_BIN_MAX];
-	struct list_head head_free_offset_list[IPA_HDR_BIN_MAX];
-	u32 hdr_cnt;
-	u32 end;
-};
-
-/**
- * struct ipa3_hdr_offset_entry - IPA header offset entry
- * @link: entry's link in global processing context header offset entries list
- * @offset: the offset
- * @bin: bin
- * @ipacm_installed: indicate if installed by ipacm
- */
-struct ipa3_hdr_proc_ctx_offset_entry {
-	struct list_head link;
-	u32 offset;
-	u32 bin;
-	bool ipacm_installed;
-};
-
-/**
- * struct ipa3_hdr_proc_ctx_entry - IPA processing context header table entry
- * @link: entry's link in global header table entries list
- * @type: header processing context type
- * @l2tp_params: L2TP parameters
- * @offset_entry: entry's offset
- * @hdr: the header
- * @cookie: cookie used for validity check
- * @ref_cnt: reference counter of routing table
- * @id: processing context header entry id
- * @user_deleted: is the hdr processing context deleted by the user?
- * @ipacm_installed: indicate if installed by ipacm
- */
-struct ipa3_hdr_proc_ctx_entry {
-	struct list_head link;
-	u32 cookie;
-	enum ipa_hdr_proc_type type;
-	struct ipa_l2tp_hdr_proc_ctx_params l2tp_params;
-	struct ipa3_hdr_proc_ctx_offset_entry *offset_entry;
-	struct ipa3_hdr_entry *hdr;
-	u32 ref_cnt;
-	int id;
-	bool user_deleted;
-	bool ipacm_installed;
-};
-
-/**
- * struct ipa3_hdr_proc_ctx_tbl - IPA processing context header table
- * @head_proc_ctx_entry_list: header entries list
- * @head_offset_list: header offset list
- * @head_free_offset_list: header free offset list
- * @proc_ctx_cnt: number of processing context headers
- * @end: the last processing context header index
- * @start_offset: offset in words of processing context header table
- */
-struct ipa3_hdr_proc_ctx_tbl {
-	struct list_head head_proc_ctx_entry_list;
-	struct list_head head_offset_list[IPA_HDR_PROC_CTX_BIN_MAX];
-	struct list_head head_free_offset_list[IPA_HDR_PROC_CTX_BIN_MAX];
-	u32 proc_ctx_cnt;
-	u32 end;
-	u32 start_offset;
-};
-
-/**
- * struct ipa3_flt_tbl - IPA filter table
- * @head_flt_rule_list: filter rules list
- * @rule_cnt: number of filter rules
- * @in_sys: flag indicating if filter table is located in system memory
- * @sz: the size of the filter tables
- * @end: the last header index
- * @curr_mem: current filter tables block in sys memory
- * @prev_mem: previous filter table block in sys memory
- * @rule_ids: common idr structure that holds the rule_id for each rule
- */
-struct ipa3_flt_tbl {
-	struct list_head head_flt_rule_list;
-	u32 rule_cnt;
-	bool in_sys[IPA_RULE_TYPE_MAX];
-	u32 sz[IPA_RULE_TYPE_MAX];
-	struct ipa_mem_buffer curr_mem[IPA_RULE_TYPE_MAX];
-	struct ipa_mem_buffer prev_mem[IPA_RULE_TYPE_MAX];
-	bool sticky_rear;
-	struct idr *rule_ids;
-};
-
-/**
- * struct ipa3_rt_entry - IPA routing table entry
- * @link: entry's link in global routing table entries list
- * @rule: routing rule
- * @cookie: cookie used for validity check
- * @tbl: routing table
- * @hdr: header table
- * @proc_ctx: processing context table
- * @hw_len: the length of the table
- * @id: rule handle - globaly unique
- * @prio: rule 10bit priority which defines the order of the rule
- *  among other rules at the integrated same table
- * @rule_id: rule 10bit ID to be returned in packet status
- * @rule_id_valid: indicate if rule_id_valid valid or not?
- * @cnt_idx: stats counter index
- * @ipacm_installed: indicate if installed by ipacm
- */
-struct ipa3_rt_entry {
-	struct list_head link;
-	u32 cookie;
-	struct ipa_rt_rule_i rule;
-	struct ipa3_rt_tbl *tbl;
-	struct ipa3_hdr_entry *hdr;
-	struct ipa3_hdr_proc_ctx_entry *proc_ctx;
-	u32 hw_len;
-	int id;
-	u16 prio;
-	u16 rule_id;
-	u16 rule_id_valid;
-	u8 cnt_idx;
-	bool ipacm_installed;
-};
-
-/**
- * struct ipa3_rt_tbl_set - collection of routing tables
- * @head_rt_tbl_list: collection of routing tables
- * @tbl_cnt: number of routing tables
- * @rule_ids: idr structure that holds the rule_id for each rule
- */
-struct ipa3_rt_tbl_set {
-	struct list_head head_rt_tbl_list;
-	u32 tbl_cnt;
-	struct idr rule_ids;
-};
-
-/**
- * struct ipa3_wlan_stats - Wlan stats for each wlan endpoint
- * @rx_pkts_rcvd: Packets sent by wlan driver
- * @rx_pkts_status_rcvd: Status packets received from ipa hw
- * @rx_hd_processed: Data Descriptors processed by IPA Driver
- * @rx_hd_reply: Data Descriptors recycled by wlan driver
- * @rx_hd_rcvd: Data Descriptors sent by wlan driver
- * @rx_pkt_leak: Packet count that are not recycled
- * @rx_dp_fail: Packets failed to transfer to IPA HW
- * @tx_pkts_rcvd: SKB Buffers received from ipa hw
- * @tx_pkts_sent: SKB Buffers sent to wlan driver
- * @tx_pkts_dropped: Dropped packets count
- */
-struct ipa3_wlan_stats {
-	u32 rx_pkts_rcvd;
-	u32 rx_pkts_status_rcvd;
-	u32 rx_hd_processed;
-	u32 rx_hd_reply;
-	u32 rx_hd_rcvd;
-	u32 rx_pkt_leak;
-	u32 rx_dp_fail;
-	u32 tx_pkts_rcvd;
-	u32 tx_pkts_sent;
-	u32 tx_pkts_dropped;
-};
-
-/**
- * struct ipa3_wlan_comm_memb - Wlan comm members
- * @wlan_spinlock: protects wlan comm buff list and its size
- * @ipa_tx_mul_spinlock: protects tx dp mul transfer
- * @wlan_comm_total_cnt: wlan common skb buffers allocated count
- * @wlan_comm_free_cnt: wlan common skb buffer free count
- * @total_tx_pkts_freed: Recycled Buffer count
- * @wlan_comm_desc_list: wlan common skb buffer list
- */
-struct ipa3_wlan_comm_memb {
-	spinlock_t wlan_spinlock;
-	spinlock_t ipa_tx_mul_spinlock;
-	u32 wlan_comm_total_cnt;
-	u32 wlan_comm_free_cnt;
-	u32 total_tx_pkts_freed;
-	struct list_head wlan_comm_desc_list;
-	atomic_t active_clnt_cnt;
-};
-
-struct ipa_gsi_ep_mem_info {
-	u16 evt_ring_len;
-	u64 evt_ring_base_addr;
-	void *evt_ring_base_vaddr;
-	u16 chan_ring_len;
-	u64 chan_ring_base_addr;
-	void *chan_ring_base_vaddr;
-};
-
-struct ipa3_status_stats {
-	struct ipahal_pkt_status status[IPA_MAX_STATUS_STAT_NUM];
-	unsigned int curr;
-};
-
-/**
- * struct ipa3_ep_context - IPA end point context
- * @valid: flag indicating id EP context is valid
- * @client: EP client type
- * @gsi_chan_hdl: EP's GSI channel handle
- * @gsi_evt_ring_hdl: EP's GSI channel event ring handle
- * @gsi_mem_info: EP's GSI channel rings info
- * @chan_scratch: EP's GSI channel scratch info
- * @cfg: EP cionfiguration
- * @dst_pipe_index: destination pipe index
- * @rt_tbl_idx: routing table index
- * @priv: user provided information which will forwarded once the user is
- *        notified for new data avail
- * @client_notify: user provided CB for EP events notification, the event is
- *                 data revived.
- * @skip_ep_cfg: boolean field that determines if EP should be configured
- *  by IPA driver
- * @keep_ipa_awake: when true, IPA will not be clock gated
- * @disconnect_in_progress: Indicates client disconnect in progress.
- * @qmi_request_sent: Indicates whether QMI request to enable clear data path
- *					request is sent or not.
- * @client_lock_unlock: callback function to take mutex lock/unlock for USB
- *				clients
- */
-struct ipa3_ep_context {
-	int valid;
-	enum ipa_client_type client;
-	unsigned long gsi_chan_hdl;
-	unsigned long gsi_evt_ring_hdl;
-	struct ipa_gsi_ep_mem_info gsi_mem_info;
-	union __packed gsi_channel_scratch chan_scratch;
-	struct gsi_chan_xfer_notify xfer_notify;
-	bool xfer_notify_valid;
-	struct ipa_ep_cfg cfg;
-	struct ipa_ep_cfg_holb holb;
-	struct ipahal_reg_ep_cfg_status status;
-	u32 dst_pipe_index;
-	u32 rt_tbl_idx;
-	void *priv;
-	void (*client_notify)(void *priv, enum ipa_dp_evt_type evt,
-		       unsigned long data);
-	atomic_t avail_fifo_desc;
-	u32 dflt_flt4_rule_hdl;
-	u32 dflt_flt6_rule_hdl;
-	bool skip_ep_cfg;
-	bool keep_ipa_awake;
-	struct ipa3_wlan_stats wstats;
-	u32 uc_offload_state;
-	u32 gsi_offload_state;
-	bool disconnect_in_progress;
-	u32 qmi_request_sent;
-	u32 eot_in_poll_err;
-	bool ep_delay_set;
-
-	/* sys MUST be the last element of this struct */
-	struct ipa3_sys_context *sys;
-};
-
-/**
- * ipa_usb_xdci_chan_params - xDCI channel related properties
- *
- * @ipa_ep_cfg:          IPA EP configuration
- * @client:              type of "client"
- * @priv:                callback cookie
- * @notify:              callback
- *           priv - callback cookie evt - type of event data - data relevant
- *           to event.  May not be valid. See event_type enum for valid
- *           cases.
- * @skip_ep_cfg:         boolean field that determines if EP should be
- *                       configured by IPA driver
- * @keep_ipa_awake:      when true, IPA will not be clock gated
- * @evt_ring_params:     parameters for the channel's event ring
- * @evt_scratch:         parameters for the channel's event ring scratch
- * @chan_params:         parameters for the channel
- * @chan_scratch:        parameters for the channel's scratch
- *
- */
-struct ipa_request_gsi_channel_params {
-	struct ipa_ep_cfg ipa_ep_cfg;
-	enum ipa_client_type client;
-	void *priv;
-	ipa_notify_cb notify;
-	bool skip_ep_cfg;
-	bool keep_ipa_awake;
-	struct gsi_evt_ring_props evt_ring_params;
-	union __packed gsi_evt_scratch evt_scratch;
-	struct gsi_chan_props chan_params;
-	union __packed gsi_channel_scratch chan_scratch;
-};
-
-enum ipa3_sys_pipe_policy {
-	IPA_POLICY_INTR_MODE,
-	IPA_POLICY_NOINTR_MODE,
-	IPA_POLICY_INTR_POLL_MODE,
-};
-
-struct ipa3_repl_ctx {
-	struct ipa3_rx_pkt_wrapper **cache;
-	atomic_t head_idx;
-	atomic_t tail_idx;
-	u32 capacity;
-	atomic_t pending;
-};
-
-/**
- * struct ipa3_sys_context - IPA GPI pipes context
- * @head_desc_list: header descriptors list
- * @len: the size of the above list
- * @spinlock: protects the list and its size
- * @ep: IPA EP context
- * @xmit_eot_cnt: count of pending eot for tasklet to process
- * @tasklet: tasklet for eot write_done handle (tx_complete)
- *
- * IPA context specific to the GPI pipes a.k.a LAN IN/OUT and WAN
- */
-struct ipa3_sys_context {
-	u32 len;
-	atomic_t curr_polling_state;
-	struct delayed_work switch_to_intr_work;
-	enum ipa3_sys_pipe_policy policy;
-	bool use_comm_evt_ring;
-	bool nop_pending;
-	int (*pyld_hdlr)(struct sk_buff *skb, struct ipa3_sys_context *sys);
-	struct sk_buff * (*get_skb)(unsigned int len, gfp_t flags);
-	void (*free_skb)(struct sk_buff *skb);
-	void (*free_rx_wrapper)(struct ipa3_rx_pkt_wrapper *rk_pkt);
-	u32 rx_buff_sz;
-	u32 rx_pool_sz;
-	struct sk_buff *prev_skb;
-	unsigned int len_rem;
-	unsigned int len_pad;
-	unsigned int len_partial;
-	bool drop_packet;
-	struct work_struct work;
-	struct delayed_work replenish_rx_work;
-	struct work_struct repl_work;
-	void (*repl_hdlr)(struct ipa3_sys_context *sys);
-	struct ipa3_repl_ctx *repl;
-	u32 pkt_sent;
-	struct napi_struct *napi_obj;
-	struct list_head pending_pkts[GSI_VEID_MAX];
-	atomic_t xmit_eot_cnt;
-	struct tasklet_struct tasklet;
-
-	/* ordering is important - mutable fields go above */
-	struct ipa3_ep_context *ep;
-	struct list_head head_desc_list;
-	struct list_head rcycl_list;
-	spinlock_t spinlock;
-	struct hrtimer db_timer;
-	struct workqueue_struct *wq;
-	struct workqueue_struct *repl_wq;
-	struct ipa3_status_stats *status_stat;
-	u32 pm_hdl;
-	/* ordering is important - other immutable fields go below */
-};
-
-/**
- * enum ipa3_desc_type - IPA decriptors type
- *
- * IPA decriptors type, IPA supports DD and ICD but no CD
- */
-enum ipa3_desc_type {
-	IPA_DATA_DESC,
-	IPA_DATA_DESC_SKB,
-	IPA_DATA_DESC_SKB_PAGED,
-	IPA_IMM_CMD_DESC,
-};
-
-/**
- * struct ipa3_tx_pkt_wrapper - IPA Tx packet wrapper
- * @type: specify if this packet is for the skb or immediate command
- * @mem: memory buffer used by this Tx packet
- * @link: linked to the wrappers on that pipe
- * @callback: IPA client provided callback
- * @user1: cookie1 for above callback
- * @user2: cookie2 for above callback
- * @sys: corresponding IPA sys context
- * @cnt: 1 for single transfers,
- * >1 and <0xFFFF for first of a "multiple" transfer,
- * 0xFFFF for last desc, 0 for rest of "multiple' transfer
- * @bounce: va of bounce buffer
- * @unmap_dma: in case this is true, the buffer will not be dma unmapped
- * @xmit_done: flag to indicate the last desc got tx complete on each ieob
- *
- * This struct can wrap both data packet and immediate command packet.
- */
-struct ipa3_tx_pkt_wrapper {
-	enum ipa3_desc_type type;
-	struct ipa_mem_buffer mem;
-	struct list_head link;
-	void (*callback)(void *user1, int user2);
-	void *user1;
-	int user2;
-	struct ipa3_sys_context *sys;
-	u32 cnt;
-	void *bounce;
-	bool no_unmap_dma;
-	bool xmit_done;
-};
-
-/**
- * struct ipa3_dma_xfer_wrapper - IPADMA transfer descr wrapper
- * @phys_addr_src: physical address of the source data to copy
- * @phys_addr_dest: physical address to store the copied data
- * @len: len in bytes to copy
- * @link: linked to the wrappers list on the proper(sync/async) cons pipe
- * @xfer_done: completion object for sync_memcpy completion
- * @callback: IPADMA client provided completion callback
- * @user1: cookie1 for above callback
- *
- * This struct can wrap both sync and async memcpy transfers descriptors.
- */
-struct ipa3_dma_xfer_wrapper {
-	u64 phys_addr_src;
-	u64 phys_addr_dest;
-	u16 len;
-	struct list_head link;
-	struct completion xfer_done;
-	void (*callback)(void *user1);
-	void *user1;
-};
-
-/**
- * struct ipa3_desc - IPA descriptor
- * @type: skb or immediate command or plain old data
- * @pyld: points to skb
- * @frag: points to paged fragment
- * or kmalloc'ed immediate command parameters/plain old data
- * @dma_address: dma mapped address of pyld
- * @dma_address_valid: valid field for dma_address
- * @is_tag_status: flag for IP_PACKET_TAG_STATUS imd cmd
- * @len: length of the pyld
- * @opcode: for immediate commands
- * @callback: IPA client provided completion callback
- * @user1: cookie1 for above callback
- * @user2: cookie2 for above callback
- * @xfer_done: completion object for sync completion
- * @skip_db_ring: specifies whether GSI doorbell should not be rang
- */
-struct ipa3_desc {
-	enum ipa3_desc_type type;
-	void *pyld;
-	skb_frag_t *frag;
-	dma_addr_t dma_address;
-	bool dma_address_valid;
-	bool is_tag_status;
-	u16 len;
-	u16 opcode;
-	void (*callback)(void *user1, int user2);
-	void *user1;
-	int user2;
-	struct completion xfer_done;
-	bool skip_db_ring;
-};
-
-/**
- * struct ipa3_rx_pkt_wrapper - IPA Rx packet wrapper
- * @skb: skb
- * @dma_address: DMA address of this Rx packet
- * @link: linked to the Rx packets on that pipe
- * @len: how many bytes are copied into skb's flat buffer
- */
-struct ipa3_rx_pkt_wrapper {
-	struct list_head link;
-	struct ipa_rx_data data;
-	u32 len;
-	struct work_struct work;
-	struct ipa3_sys_context *sys;
-};
-
-/**
- * struct ipa3_nat_ipv6ct_tmp_mem - NAT/IPv6CT temporary memory
- *
- * In case NAT/IPv6CT table are destroyed the HW is provided with the
- * temporary memory
- *
- * @vaddr: the address of the temporary memory
- * @dma_handle: the handle of the temporary memory
- */
-struct ipa3_nat_ipv6ct_tmp_mem {
-	void *vaddr;
-	dma_addr_t dma_handle;
-};
-
-/**
- * struct ipa3_nat_ipv6ct_common_mem - IPA NAT/IPv6CT memory device
- * @class: pointer to the struct class
- * @dev: the dev_t of the device
- * @cdev: cdev of the device
- * @dev_num: device number
- * @vaddr: the virtual address in the system memory
- * @dma_handle: the system memory DMA handle
- * @phys_mem_size: the physical size in the shared memory
- * @smem_offset: the offset in the shared memory
- * @size: memory size
- * @is_mapped: flag indicating if memory is mapped
- * @is_sys_mem: flag indicating if memory is sys memory
- * @is_mem_allocated: flag indicating if the memory is allocated
- * @is_hw_init: flag indicating if the corresponding HW is initialized
- * @is_dev_init: flag indicating if device is initialized
- * @lock: memory mutex
- * @base_address: table virtual address
- * @base_table_addr: base table address
- * @expansion_table_addr: expansion table address
- * @table_entries: num of entries in the base table
- * @expn_table_entries: num of entries in the expansion table
- * @tmp_mem: temporary memory used to always provide HW with a legal memory
- * @name: the device name
- */
-struct ipa3_nat_ipv6ct_common_mem {
-	struct class *class;
-	struct device *dev;
-	struct cdev cdev;
-	dev_t dev_num;
-
-	/* system memory */
-	void *vaddr;
-	dma_addr_t dma_handle;
-
-	/* shared memory */
-	u32 phys_mem_size;
-	u32 smem_offset;
-
-	size_t size;
-	bool is_mapped;
-	bool is_sys_mem;
-	bool is_mem_allocated;
-	bool is_hw_init;
-	bool is_dev_init;
-	struct mutex lock;
-	void *base_address;
-	char *base_table_addr;
-	char *expansion_table_addr;
-	u32 table_entries;
-	u32 expn_table_entries;
-	struct ipa3_nat_ipv6ct_tmp_mem *tmp_mem;
-	char name[IPA_DEV_NAME_MAX_LEN];
-};
-
-/**
- * struct ipa3_nat_mem - IPA NAT memory description
- * @dev: the memory device structure
- * @index_table_addr: index table address
- * @index_table_expansion_addr: index expansion table address
- * @public_ip_addr: ip address of nat table
- * @pdn_mem: pdn config table SW cache memory structure
- * @is_tmp_mem_allocated: indicate if tmp mem has been allocated
- */
-struct ipa3_nat_mem {
-	struct ipa3_nat_ipv6ct_common_mem dev;
-	char *index_table_addr;
-	char *index_table_expansion_addr;
-	u32 public_ip_addr;
-	struct ipa_mem_buffer pdn_mem;
-	bool is_tmp_mem_allocated;
-};
-
-/**
- * struct ipa3_ipv6ct_mem - IPA IPv6 connection tracking memory description
- * @dev: the memory device structure
- */
-struct ipa3_ipv6ct_mem {
-	struct ipa3_nat_ipv6ct_common_mem dev;
-};
-
-/**
- * enum ipa3_hw_mode - IPA hardware mode
- * @IPA_HW_Normal: Regular IPA hardware
- * @IPA_HW_Virtual: IPA hardware supporting virtual memory allocation
- * @IPA_HW_PCIE: IPA hardware supporting memory allocation over PCIE Bridge
- * @IPA_HW_Emulation: IPA emulation hardware
- */
-enum ipa3_hw_mode {
-	IPA_HW_MODE_NORMAL    = 0,
-	IPA_HW_MODE_VIRTUAL   = 1,
-	IPA_HW_MODE_PCIE      = 2,
-	IPA_HW_MODE_EMULATION = 3,
-};
-
-/*
- * enum ipa3_platform_type - Platform type
- * @IPA_PLAT_TYPE_MDM: MDM platform (usually 32bit single core CPU platform)
- * @IPA_PLAT_TYPE_MSM: MSM SOC platform (usually 64bit multi-core platform)
- * @IPA_PLAT_TYPE_APQ: Similar to MSM but without modem
- */
-enum ipa3_platform_type {
-	IPA_PLAT_TYPE_MDM	= 0,
-	IPA_PLAT_TYPE_MSM	= 1,
-	IPA_PLAT_TYPE_APQ	= 2,
-};
-
-enum ipa3_config_this_ep {
-	IPA_CONFIGURE_THIS_EP,
-	IPA_DO_NOT_CONFIGURE_THIS_EP,
-};
-
-struct ipa3_stats {
-	u32 tx_sw_pkts;
-	u32 tx_hw_pkts;
-	u32 rx_pkts;
-	u32 rx_excp_pkts[IPAHAL_PKT_STATUS_EXCEPTION_MAX];
-	u32 rx_repl_repost;
-	u32 tx_pkts_compl;
-	u32 rx_q_len;
-	u32 msg_w[IPA_EVENT_MAX_NUM];
-	u32 msg_r[IPA_EVENT_MAX_NUM];
-	u32 stat_compl;
-	u32 aggr_close;
-	u32 wan_aggr_close;
-	u32 wan_rx_empty;
-	u32 wan_repl_rx_empty;
-	u32 lan_rx_empty;
-	u32 lan_repl_rx_empty;
-	u32 flow_enable;
-	u32 flow_disable;
-	u32 tx_non_linear;
-};
-
-/* offset for each stats */
-#define IPA3_UC_DEBUG_STATS_RINGFULL_OFF (0)
-#define IPA3_UC_DEBUG_STATS_RINGEMPTY_OFF (4)
-#define IPA3_UC_DEBUG_STATS_RINGUSAGEHIGH_OFF (8)
-#define IPA3_UC_DEBUG_STATS_RINGUSAGELOW_OFF (12)
-#define IPA3_UC_DEBUG_STATS_RINGUTILCOUNT_OFF (16)
-#define IPA3_UC_DEBUG_STATS_OFF (20)
-
-/**
- * struct ipa3_uc_dbg_gsi_stats - uC dbg stats info for each
- * offloading protocol
- * @ring: ring stats for each channel
- */
-struct ipa3_uc_dbg_ring_stats {
-	struct IpaHwRingStats_t ring[MAX_CH_STATS_SUPPORTED];
-};
-
-/**
- * struct ipa3_uc_dbg_stats - uC dbg stats for offloading
- * protocols
- * @uc_dbg_stats_ofst: offset to SRAM base
- * @uc_dbg_stats_size: stats size for all channels
- * @uc_dbg_stats_mmio: mmio offset
- */
-struct ipa3_uc_dbg_stats {
-	u32 uc_dbg_stats_ofst;
-	u16 uc_dbg_stats_size;
-	void __iomem *uc_dbg_stats_mmio;
-};
-
-struct ipa3_active_clients {
-	struct mutex mutex;
-	atomic_t cnt;
-	int bus_vote_idx;
-};
-
-struct ipa3_wakelock_ref_cnt {
-	spinlock_t spinlock;
-	int cnt;
-};
-
-struct ipa3_tag_completion {
-	struct completion comp;
-	atomic_t cnt;
-};
-
-struct ipa3_controller;
-
-/**
- * struct ipa3_uc_hdlrs - IPA uC callback functions
- * @ipa_uc_loaded_hdlr: Function handler when uC is loaded
- * @ipa_uc_event_hdlr: Event handler function
- * @ipa3_uc_response_hdlr: Response handler function
- * @ipa_uc_event_log_info_hdlr: Log event handler function
- */
-struct ipa3_uc_hdlrs {
-	void (*ipa_uc_loaded_hdlr)(void);
-
-	void (*ipa_uc_event_hdlr)
-		(struct IpaHwSharedMemCommonMapping_t *uc_sram_mmio);
-
-	int (*ipa3_uc_response_hdlr)
-		(struct IpaHwSharedMemCommonMapping_t *uc_sram_mmio,
-		u32 *uc_status);
-
-	void (*ipa_uc_event_log_info_hdlr)
-		(struct IpaHwEventLogInfoData_t *uc_event_top_mmio);
-};
-
-/**
- * enum ipa3_hw_flags - flags which defines the behavior of HW
- *
- * @IPA_HW_FLAG_HALT_SYSTEM_ON_ASSERT_FAILURE: Halt system in case of assert
- *	failure.
- * @IPA_HW_FLAG_NO_REPORT_MHI_CHANNEL_ERORR: Channel error would be reported
- *	in the event ring only. No event to CPU.
- * @IPA_HW_FLAG_NO_REPORT_MHI_CHANNEL_WAKE_UP: No need to report event
- *	IPA_HW_2_CPU_EVENT_MHI_WAKE_UP_REQUEST
- * @IPA_HW_FLAG_WORK_OVER_DDR: Perform all transaction to external addresses by
- *	QMB (avoid memcpy)
- * @IPA_HW_FLAG_NO_REPORT_OOB: If set do not report that the device is OOB in
- *	IN Channel
- * @IPA_HW_FLAG_NO_REPORT_DB_MODE: If set, do not report that the device is
- *	entering a mode where it expects a doorbell to be rung for OUT Channel
- * @IPA_HW_FLAG_NO_START_OOB_TIMER
- */
-enum ipa3_hw_flags {
-	IPA_HW_FLAG_HALT_SYSTEM_ON_ASSERT_FAILURE	= 0x01,
-	IPA_HW_FLAG_NO_REPORT_MHI_CHANNEL_ERORR		= 0x02,
-	IPA_HW_FLAG_NO_REPORT_MHI_CHANNEL_WAKE_UP	= 0x04,
-	IPA_HW_FLAG_WORK_OVER_DDR			= 0x08,
-	IPA_HW_FLAG_NO_REPORT_OOB			= 0x10,
-	IPA_HW_FLAG_NO_REPORT_DB_MODE			= 0x20,
-	IPA_HW_FLAG_NO_START_OOB_TIMER			= 0x40
-};
-
-/**
- * struct ipa3_uc_ctx - IPA uC context
- * @uc_inited: Indicates if uC interface has been initialized
- * @uc_loaded: Indicates if uC has loaded
- * @uc_failed: Indicates if uC has failed / returned an error
- * @uc_lock: uC interface lock to allow only one uC interaction at a time
- * @uc_spinlock: same as uc_lock but for irq contexts
- * @uc_completation: Completion mechanism to wait for uC commands
- * @uc_sram_mmio: Pointer to uC mapped memory
- * @pending_cmd: The last command sent waiting to be ACKed
- * @uc_status: The last status provided by the uC
- * @uc_error_type: error type from uC error event
- * @uc_error_timestamp: tag timer sampled after uC crashed
- */
-struct ipa3_uc_ctx {
-	bool uc_inited;
-	bool uc_loaded;
-	bool uc_failed;
-	struct mutex uc_lock;
-	spinlock_t uc_spinlock;
-	struct completion uc_completion;
-	struct IpaHwSharedMemCommonMapping_t *uc_sram_mmio;
-	struct IpaHwEventLogInfoData_t *uc_event_top_mmio;
-	u32 uc_event_top_ofst;
-	u32 pending_cmd;
-	u32 uc_status;
-	u32 uc_error_type;
-	u32 uc_error_timestamp;
-	phys_addr_t rdy_ring_base_pa;
-	phys_addr_t rdy_ring_rp_pa;
-	u32 rdy_ring_size;
-	phys_addr_t rdy_comp_ring_base_pa;
-	phys_addr_t rdy_comp_ring_wp_pa;
-	u32 rdy_comp_ring_size;
-	u32 *rdy_ring_rp_va;
-	u32 *rdy_comp_ring_wp_va;
-};
-
-/**
- * struct ipa3_uc_wdi_ctx
- * @wdi_uc_top_ofst:
- * @wdi_uc_top_mmio:
- * @wdi_uc_stats_ofst:
- * @wdi_uc_stats_mmio:
- */
-struct ipa3_uc_wdi_ctx {
-	/* WDI specific fields */
-	u32 wdi_uc_stats_ofst;
-	struct IpaHwStatsWDIInfoData_t *wdi_uc_stats_mmio;
-	void *priv;
-	ipa_uc_ready_cb uc_ready_cb;
-	/* for AP+STA stats update */
-#ifdef IPA_WAN_MSG_IPv6_ADDR_GW_LEN
-	ipa_wdi_meter_notifier_cb stats_notify;
-#endif
-};
-
-/**
- * struct ipa3_uc_wigig_ctx
- * @priv: wigig driver private data
- * @uc_ready_cb: wigig driver uc ready callback
- * @int_notify: wigig driver misc interrupt callback
- */
-struct ipa3_uc_wigig_ctx {
-	void *priv;
-	ipa_uc_ready_cb uc_ready_cb;
-	ipa_wigig_misc_int_cb misc_notify_cb;
-};
-
-/**
- * struct ipa3_wdi2_ctx - IPA wdi2 context
- */
-struct ipa3_wdi2_ctx {
-	phys_addr_t rdy_ring_base_pa;
-	phys_addr_t rdy_ring_rp_pa;
-	u32 rdy_ring_size;
-	phys_addr_t rdy_comp_ring_base_pa;
-	phys_addr_t rdy_comp_ring_wp_pa;
-	u32 rdy_comp_ring_size;
-	u32 *rdy_ring_rp_va;
-	u32 *rdy_comp_ring_wp_va;
-	struct ipa3_uc_dbg_stats dbg_stats;
-};
-
-/**
- * struct ipa3_wdi3_ctx - IPA wdi3 context
- */
-struct ipa3_wdi3_ctx {
-	struct ipa3_uc_dbg_stats dbg_stats;
-};
-
-/**
- * struct ipa3_usb_ctx - IPA usb context
- */
-struct ipa3_usb_ctx {
-	struct ipa3_uc_dbg_stats dbg_stats;
-};
-
-/**
- * struct ipa3_mhip_ctx - IPA mhip context
- */
-struct ipa3_mhip_ctx {
-	struct ipa3_uc_dbg_stats dbg_stats;
-};
-
-/**
- * struct ipa3_transport_pm - transport power management related members
- * @transport_pm_mutex: Mutex to protect the transport_pm functionality.
- */
-struct ipa3_transport_pm {
-	atomic_t dec_clients;
-	atomic_t eot_activity;
-	struct mutex transport_pm_mutex;
-};
-
-/**
- * struct ipa3cm_client_info - the client-info indicated from IPACM
- * @ipacm_client_enum: the enum to indicate tether-client
- * @ipacm_client_uplink: the bool to indicate pipe for uplink
- */
-struct ipa3cm_client_info {
-	enum ipacm_client_enum client_enum;
-	bool uplink;
-};
-
-struct ipa3_smp2p_info {
-	u32 out_base_id;
-	u32 in_base_id;
-	bool ipa_clk_on;
-	bool res_sent;
-	unsigned int smem_bit;
-	struct qcom_smem_state *smem_state;
-};
-
-/**
- * struct ipa3_ready_cb_info - A list of all the registrations
- *  for an indication of IPA driver readiness
- *
- * @link: linked list link
- * @ready_cb: callback
- * @user_data: User data
- *
- */
-struct ipa3_ready_cb_info {
-	struct list_head link;
-	ipa_ready_cb ready_cb;
-	void *user_data;
-};
-
-struct ipa_dma_task_info {
-	struct ipa_mem_buffer mem;
-	struct ipahal_imm_cmd_pyld *cmd_pyld;
-};
-
-struct ipa_quota_stats {
-	u64 num_ipv4_bytes;
-	u64 num_ipv6_bytes;
-	u32 num_ipv4_pkts;
-	u32 num_ipv6_pkts;
-};
-
-struct ipa_quota_stats_all {
-	struct ipa_quota_stats client[IPA_CLIENT_MAX];
-};
-
-struct ipa_drop_stats {
-	u32 drop_packet_cnt;
-	u32 drop_byte_cnt;
-};
-
-struct ipa_drop_stats_all {
-	struct ipa_drop_stats client[IPA_CLIENT_MAX];
-};
-
-struct ipa_hw_stats_quota {
-	struct ipahal_stats_init_quota init;
-	struct ipa_quota_stats_all stats;
-};
-
-struct ipa_hw_stats_teth {
-	struct ipahal_stats_init_tethering init;
-	struct ipa_quota_stats_all prod_stats_sum[IPA_CLIENT_MAX];
-	struct ipa_quota_stats_all prod_stats[IPA_CLIENT_MAX];
-};
-
-struct ipa_hw_stats_flt_rt {
-	struct ipahal_stats_init_flt_rt flt_v4_init;
-	struct ipahal_stats_init_flt_rt flt_v6_init;
-	struct ipahal_stats_init_flt_rt rt_v4_init;
-	struct ipahal_stats_init_flt_rt rt_v6_init;
-};
-
-struct ipa_hw_stats_drop {
-	struct ipahal_stats_init_drop init;
-	struct ipa_drop_stats_all stats;
-};
-
-struct ipa_hw_stats {
-	bool enabled;
-	struct ipa_hw_stats_quota quota;
-	struct ipa_hw_stats_teth teth;
-	struct ipa_hw_stats_flt_rt flt_rt;
-	struct ipa_hw_stats_drop drop;
-};
-
-struct ipa_cne_evt {
-	struct ipa_wan_msg wan_msg;
-	struct ipa_msg_meta msg_meta;
-};
-
-enum ipa_smmu_cb_type {
-	IPA_SMMU_CB_AP,
-	IPA_SMMU_CB_WLAN,
-	IPA_SMMU_CB_UC,
-	IPA_SMMU_CB_11AD,
-	IPA_SMMU_CB_MAX
-};
-
-#define VALID_IPA_SMMU_CB_TYPE(t) \
-	((t) >= IPA_SMMU_CB_AP && (t) < IPA_SMMU_CB_MAX)
-
-enum ipa_client_cb_type {
-	IPA_USB_CLNT,
-	IPA_MHI_CLNT,
-	IPA_MAX_CLNT
-};
-
-/**
- * struct ipa_flt_rt_counter - IPA flt rt counters management
- * @hdl: idr structure to manage hdl per request
- * @used_hw: boolean array to track used hw counters
- * @used_sw: boolean array to track used sw counters
- * @hdl_lock: spinlock for flt_rt handle
- */
-struct ipa_flt_rt_counter {
-	struct idr hdl;
-	bool used_hw[IPA_FLT_RT_HW_COUNTER];
-	bool used_sw[IPA_FLT_RT_SW_COUNTER];
-	spinlock_t hdl_lock;
-};
-
-/**
- * struct ipa3_char_device_context - IPA character device
- * @class: pointer to the struct class
- * @dev_num: device number
- * @dev: the dev_t of the device
- * @cdev: cdev of the device
- */
-struct ipa3_char_device_context {
-	struct class *class;
-	dev_t dev_num;
-	struct device *dev;
-	struct cdev cdev;
-};
-
-struct ipa3_pc_mbox_data {
-	struct mbox_client mbox_client;
-	struct mbox_chan *mbox;
-};
-
-/**
- * struct ipa3_context - IPA context
- * @cdev: cdev context
- * @ep: list of all end points
- * @skip_ep_cfg_shadow: state to update filter table correctly across
-  power-save
- * @ep_flt_bitmap: End-points supporting filtering bitmap
- * @ep_flt_num: End-points supporting filtering number
- * @resume_on_connect: resume ep on ipa connect
- * @flt_tbl: list of all IPA filter tables
- * @flt_rule_ids: idr structure that holds the rule_id for each rule
- * @mode: IPA operating mode
- * @mmio: iomem
- * @ipa_wrapper_base: IPA wrapper base address
- * @ipa_wrapper_size: size of the memory pointed to by ipa_wrapper_base
- * @hdr_tbl: IPA header table
- * @hdr_proc_ctx_tbl: IPA processing context table
- * @rt_tbl_set: list of routing tables each of which is a list of rules
- * @reap_rt_tbl_set: list of sys mem routing tables waiting to be reaped
- * @flt_rule_cache: filter rule cache
- * @rt_rule_cache: routing rule cache
- * @hdr_cache: header cache
- * @hdr_offset_cache: header offset cache
- * @hdr_proc_ctx_cache: processing context cache
- * @hdr_proc_ctx_offset_cache: processing context offset cache
- * @rt_tbl_cache: routing table cache
- * @tx_pkt_wrapper_cache: Tx packets cache
- * @rx_pkt_wrapper_cache: Rx packets cache
- * @rt_idx_bitmap: routing table index bitmap
- * @lock: this does NOT protect the linked lists within ipa3_sys_context
- * @smem_sz: shared memory size available for SW use starting
- *  from non-restricted bytes
- * @smem_restricted_bytes: the bytes that SW should not use in the shared mem
- * @nat_mem: NAT memory
- * @ipv6ct_mem: IPv6CT memory
- * @excp_hdr_hdl: exception header handle
- * @dflt_v4_rt_rule_hdl: default v4 routing rule handle
- * @dflt_v6_rt_rule_hdl: default v6 routing rule handle
- * @aggregation_type: aggregation type used on USB client endpoint
- * @aggregation_byte_limit: aggregation byte limit used on USB client endpoint
- * @aggregation_time_limit: aggregation time limit used on USB client endpoint
- * @hdr_tbl_lcl: where hdr tbl resides 1-local, 0-system
- * @hdr_proc_ctx_tbl_lcl: where proc_ctx tbl resides true-local, false-system
- * @hdr_mem: header memory
- * @hdr_proc_ctx_mem: processing context memory
- * @ip4_rt_tbl_lcl: where ip4 rt tables reside 1-local; 0-system
- * @ip6_rt_tbl_lcl: where ip6 rt tables reside 1-local; 0-system
- * @ip4_flt_tbl_lcl: where ip4 flt tables reside 1-local; 0-system
- * @ip6_flt_tbl_lcl: where ip6 flt tables reside 1-local; 0-system
- * @power_mgmt_wq: workqueue for power management
- * @transport_power_mgmt_wq: workqueue transport related power management
- * @tag_process_before_gating: indicates whether to start tag process before
- *  gating IPA clocks
- * @transport_pm: transport power management related information
- * @disconnect_lock: protects LAN_CONS packet receive notification CB
- * @ipa3_active_clients: structure for reference counting connected IPA clients
- * @ipa_hw_type: type of IPA HW type (e.g. IPA 1.0, IPA 1.1 etc')
- * @ipa3_hw_mode: mode of IPA HW mode (e.g. Normal, Virtual or over PCIe)
- * @use_ipa_teth_bridge: use tethering bridge driver
- * @modem_cfg_emb_pipe_flt: modem configure embedded pipe filtering rules
- * @logbuf: ipc log buffer for high priority messages
- * @logbuf_low: ipc log buffer for low priority messages
- * @ipa_wdi2: using wdi-2.0
- * @ipa_fltrt_not_hashable: filter/route rules not hashable
- * @use_64_bit_dma_mask: using 64bits dma mask
- * @ipa_bus_hdl: msm driver handle for the data path bus
- * @ctrl: holds the core specific operations based on
- *  core version (vtable like)
- * @pkt_init_imm_opcode: opcode for IP_PACKET_INIT imm cmd
- * @enable_clock_scaling: clock scaling is enabled ?
- * @curr_ipa_clk_rate: IPA current clock rate
- * @wcstats: wlan common buffer stats
- * @uc_ctx: uC interface context
- * @uc_wdi_ctx: WDI specific fields for uC interface
- * @uc_wigig_ctx: WIGIG specific fields for uC interface
- * @ipa_num_pipes: The number of pipes used by IPA HW
- * @skip_uc_pipe_reset: Indicates whether pipe reset via uC needs to be avoided
- * @ipa_client_apps_wan_cons_agg_gro: RMNET_IOCTL_INGRESS_FORMAT_AGG_DATA
- * @apply_rg10_wa: Indicates whether to use register group 10 workaround
- * @gsi_ch20_wa: Indicates whether to apply GSI physical channel 20 workaround
- * @w_lock: Indicates the wakeup source.
- * @wakelock_ref_cnt: Indicates the number of times wakelock is acquired
- * @ipa_initialization_complete: Indicates that IPA is fully initialized
- * @ipa_ready_cb_list: A list of all the clients who require a CB when IPA
- *  driver is ready/initialized.
- * @init_completion_obj: Completion object to be used in case IPA driver hasn't
- * @mhi_evid_limits: MHI event rings start and end ids
- *  finished initializing. Example of use - IOCTLs to /dev/ipa
- * @flt_rt_counters: the counters usage info for flt rt stats
- * @wdi3_ctx: IPA wdi3 context
- * @gsi_info: channel/protocol info for GSI offloading uC stats
- * IPA context - holds all relevant info about IPA driver and its state
- */
-struct ipa3_context {
-	struct ipa3_char_device_context cdev;
-	struct ipa3_ep_context ep[IPA3_MAX_NUM_PIPES];
-	bool skip_ep_cfg_shadow[IPA3_MAX_NUM_PIPES];
-	u32 ep_flt_bitmap;
-	u32 ep_flt_num;
-	bool resume_on_connect[IPA_CLIENT_MAX];
-	struct ipa3_flt_tbl flt_tbl[IPA3_MAX_NUM_PIPES][IPA_IP_MAX];
-	struct idr flt_rule_ids[IPA_IP_MAX];
-	void __iomem *mmio;
-	u32 ipa_wrapper_base;
-	u32 ipa_wrapper_size;
-	struct ipa3_hdr_tbl hdr_tbl;
-	struct ipa3_hdr_proc_ctx_tbl hdr_proc_ctx_tbl;
-	struct ipa3_rt_tbl_set rt_tbl_set[IPA_IP_MAX];
-	struct ipa3_rt_tbl_set reap_rt_tbl_set[IPA_IP_MAX];
-	struct kmem_cache *flt_rule_cache;
-	struct kmem_cache *rt_rule_cache;
-	struct kmem_cache *hdr_cache;
-	struct kmem_cache *hdr_offset_cache;
-	struct kmem_cache *hdr_proc_ctx_cache;
-	struct kmem_cache *hdr_proc_ctx_offset_cache;
-	struct kmem_cache *rt_tbl_cache;
-	struct kmem_cache *tx_pkt_wrapper_cache;
-	struct kmem_cache *rx_pkt_wrapper_cache;
-	unsigned long rt_idx_bitmap[IPA_IP_MAX];
-	struct mutex lock;
-	u16 smem_sz;
-	u16 smem_restricted_bytes;
-	u16 smem_reqd_sz;
-	struct ipa3_nat_mem nat_mem;
-	struct ipa3_ipv6ct_mem ipv6ct_mem;
-	u32 excp_hdr_hdl;
-	u32 dflt_v4_rt_rule_hdl;
-	u32 dflt_v6_rt_rule_hdl;
-	uint aggregation_type;
-	uint aggregation_byte_limit;
-	uint aggregation_time_limit;
-	bool hdr_tbl_lcl;
-	bool hdr_proc_ctx_tbl_lcl;
-	struct ipa_mem_buffer hdr_mem;
-	struct ipa_mem_buffer hdr_proc_ctx_mem;
-	bool ip4_rt_tbl_hash_lcl;
-	bool ip4_rt_tbl_nhash_lcl;
-	bool ip6_rt_tbl_hash_lcl;
-	bool ip6_rt_tbl_nhash_lcl;
-	bool ip4_flt_tbl_hash_lcl;
-	bool ip4_flt_tbl_nhash_lcl;
-	bool ip6_flt_tbl_hash_lcl;
-	bool ip6_flt_tbl_nhash_lcl;
-	struct ipa3_active_clients ipa3_active_clients;
-	struct ipa3_active_clients_log_ctx ipa3_active_clients_logging;
-	struct workqueue_struct *power_mgmt_wq;
-	struct workqueue_struct *transport_power_mgmt_wq;
-	bool tag_process_before_gating;
-	struct ipa3_transport_pm transport_pm;
-	unsigned long gsi_evt_comm_hdl;
-	u32 gsi_evt_comm_ring_rem;
-	u32 clnt_hdl_cmd;
-	u32 clnt_hdl_data_in;
-	u32 clnt_hdl_data_out;
-	spinlock_t disconnect_lock;
-	u8 a5_pipe_index;
-	struct list_head intf_list;
-	struct list_head msg_list;
-	struct list_head pull_msg_list;
-	struct mutex msg_lock;
-	struct list_head msg_wlan_client_list;
-	struct mutex msg_wlan_client_lock;
-	wait_queue_head_t msg_waitq;
-	enum ipa_hw_type ipa_hw_type;
-	enum ipa3_hw_mode ipa3_hw_mode;
-	enum ipa3_platform_type platform_type;
-	bool ipa_config_is_mhi;
-	bool use_ipa_teth_bridge;
-	bool modem_cfg_emb_pipe_flt;
-	bool ipa_wdi2;
-	bool ipa_wdi2_over_gsi;
-	bool ipa_wdi3_over_gsi;
-	bool ipa_endp_delay_wa;
-	bool ipa_fltrt_not_hashable;
-	bool use_64_bit_dma_mask;
-	/* featurize if memory footprint becomes a concern */
-	struct ipa3_stats stats;
-	void *smem_pipe_mem;
-	void *logbuf;
-	void *logbuf_low;
-	u32 ipa_bus_hdl;
-	struct ipa3_controller *ctrl;
-	struct idr ipa_idr;
-	struct platform_device *master_pdev;
-	struct device *pdev;
-	struct device *uc_pdev;
-	spinlock_t idr_lock;
-	u32 enable_clock_scaling;
-	u32 enable_napi_chain;
-	u32 curr_ipa_clk_rate;
-	bool q6_proxy_clk_vote_valid;
-	struct mutex q6_proxy_clk_vote_mutex;
-	u32 q6_proxy_clk_vote_cnt;
-	u32 ipa_num_pipes;
-	dma_addr_t pkt_init_imm[IPA3_MAX_NUM_PIPES];
-	u32 pkt_init_imm_opcode;
-
-	struct ipa3_wlan_comm_memb wc_memb;
-
-	struct ipa3_uc_ctx uc_ctx;
-
-	struct ipa3_uc_wdi_ctx uc_wdi_ctx;
-	struct ipa3_uc_ntn_ctx uc_ntn_ctx;
-	struct ipa3_uc_wigig_ctx uc_wigig_ctx;
-	u32 wan_rx_ring_size;
-	u32 lan_rx_ring_size;
-	bool skip_uc_pipe_reset;
-	unsigned long gsi_dev_hdl;
-	u32 ee;
-	bool apply_rg10_wa;
-	bool gsi_ch20_wa;
-	bool s1_bypass_arr[IPA_SMMU_CB_MAX];
-	u32 wdi_map_cnt;
-	struct wakeup_source w_lock;
-	struct ipa3_wakelock_ref_cnt wakelock_ref_cnt;
-	/* RMNET_IOCTL_INGRESS_FORMAT_AGG_DATA */
-	bool ipa_client_apps_wan_cons_agg_gro;
-	/* M-release support to know client pipes */
-	struct ipa3cm_client_info ipacm_client[IPA3_MAX_NUM_PIPES];
-	bool tethered_flow_control;
-	bool ipa_initialization_complete;
-	struct list_head ipa_ready_cb_list;
-	struct completion init_completion_obj;
-	struct completion uc_loaded_completion_obj;
-	struct ipa3_smp2p_info smp2p_info;
-	u32 mhi_evid_limits[2]; /* start and end values */
-	u32 ipa_tz_unlock_reg_num;
-	struct ipa_tz_unlock_reg_info *ipa_tz_unlock_reg;
-	struct ipa_dma_task_info dma_task_info;
-	struct ipa_hw_stats hw_stats;
-	struct ipa_flt_rt_counter flt_rt_counters;
-	struct ipa_cne_evt ipa_cne_evt_req_cache[IPA_MAX_NUM_REQ_CACHE];
-	int num_ipa_cne_evt_req;
-	struct mutex ipa_cne_evt_lock;
-	bool vlan_mode_iface[IPA_VLAN_IF_MAX];
-	bool wdi_over_pcie;
-	u32 entire_ipa_block_size;
-	bool do_register_collection_on_crash;
-	bool do_testbus_collection_on_crash;
-	bool do_non_tn_collection_on_crash;
-	bool do_ram_collection_on_crash;
-	u32 secure_debug_check_action;
-	u32 sd_state;
-	void __iomem *reg_collection_base;
-	struct ipa3_wdi2_ctx wdi2_ctx;
-	struct ipa3_pc_mbox_data pc_mbox;
-	struct ipa3_wdi3_ctx wdi3_ctx;
-	struct ipa3_usb_ctx usb_ctx;
-	struct ipa3_mhip_ctx mhip_ctx;
-	atomic_t ipa_clk_vote;
-	int (*client_lock_unlock[IPA_MAX_CLNT])(bool is_lock);
-	bool fw_loaded;
-	bool (*get_teth_port_state[IPA_MAX_CLNT])(void);
-	atomic_t is_ssr;
-	struct IpaHwOffloadStatsAllocCmdData_t
-		gsi_info[IPA_HW_PROTOCOL_MAX];
-};
-
-struct ipa3_plat_drv_res {
-	bool use_ipa_teth_bridge;
-	u32 ipa_mem_base;
-	u32 ipa_mem_size;
-	u32 transport_mem_base;
-	u32 transport_mem_size;
-	u32 emulator_intcntrlr_mem_base;
-	u32 emulator_intcntrlr_mem_size;
-	u32 emulator_irq;
-	u32 ipa_irq;
-	u32 transport_irq;
-	u32 ipa_pipe_mem_start_ofst;
-	u32 ipa_pipe_mem_size;
-	enum ipa_hw_type ipa_hw_type;
-	enum ipa3_hw_mode ipa3_hw_mode;
-	enum ipa3_platform_type platform_type;
-	u32 ee;
-	bool modem_cfg_emb_pipe_flt;
-	bool ipa_wdi2;
-	bool ipa_wdi2_over_gsi;
-	bool ipa_wdi3_over_gsi;
-	bool ipa_fltrt_not_hashable;
-	bool use_64_bit_dma_mask;
-	bool use_bw_vote;
-	u32 wan_rx_ring_size;
-	u32 lan_rx_ring_size;
-	bool skip_uc_pipe_reset;
-	bool apply_rg10_wa;
-	bool gsi_ch20_wa;
-	bool tethered_flow_control;
-	u32 mhi_evid_limits[2]; /* start and end values */
-	bool ipa_mhi_dynamic_config;
-	u32 ipa_tz_unlock_reg_num;
-	struct ipa_tz_unlock_reg_info *ipa_tz_unlock_reg;
-	struct ipa_pm_init_params pm_init;
-	bool wdi_over_pcie;
-	u32 entire_ipa_block_size;
-	bool do_register_collection_on_crash;
-	bool do_testbus_collection_on_crash;
-	bool do_non_tn_collection_on_crash;
-	bool do_ram_collection_on_crash;
-	u32 secure_debug_check_action;
-	bool ipa_endp_delay_wa;
-};
-
-/**
- * struct ipa3_mem_partition - represents IPA RAM Map as read from DTS
- * Order and type of members should not be changed without a suitable change
- * to DTS file or the code that reads it.
- *
- * IPA SRAM memory layout:
- * +-------------------------+
- * |    UC MEM               |
- * +-------------------------+
- * |    UC INFO              |
- * +-------------------------+
- * |    CANARY               |
- * +-------------------------+
- * |    CANARY               |
- * +-------------------------+
- * | V4 FLT HDR HASHABLE     |
- * +-------------------------+
- * |    CANARY               |
- * +-------------------------+
- * |    CANARY               |
- * +-------------------------+
- * | V4 FLT HDR NON-HASHABLE |
- * +-------------------------+
- * |    CANARY               |
- * +-------------------------+
- * |    CANARY               |
- * +-------------------------+
- * | V6 FLT HDR HASHABLE     |
- * +-------------------------+
- * |    CANARY               |
- * +-------------------------+
- * |    CANARY               |
- * +-------------------------+
- * | V6 FLT HDR NON-HASHABLE |
- * +-------------------------+
- * |    CANARY               |
- * +-------------------------+
- * |    CANARY               |
- * +-------------------------+
- * | V4 RT HDR HASHABLE      |
- * +-------------------------+
- * |    CANARY               |
- * +-------------------------+
- * |    CANARY               |
- * +-------------------------+
- * | V4 RT HDR NON-HASHABLE  |
- * +-------------------------+
- * |    CANARY               |
- * +-------------------------+
- * |    CANARY               |
- * +-------------------------+
- * | V6 RT HDR HASHABLE      |
- * +-------------------------+
- * |    CANARY               |
- * +-------------------------+
- * |    CANARY               |
- * +-------------------------+
- * | V6 RT HDR NON-HASHABLE  |
- * +-------------------------+
- * |    CANARY               |
- * +-------------------------+
- * |    CANARY               |
- * +-------------------------+
- * |  MODEM HDR              |
- * +-------------------------+
- * |  APPS HDR (IPA4.5)      |
- * +-------------------------+
- * |    CANARY               |
- * +-------------------------+
- * |    CANARY               |
- * +-------------------------+
- * | MODEM PROC CTX          |
- * +-------------------------+
- * | APPS PROC CTX           |
- * +-------------------------+
- * |    CANARY               |
- * +-------------------------+
- * |    CANARY               |
- * +-------------------------+
- * |    CANARY (IPA4.5)      |
- * +-------------------------+
- * |    CANARY (IPA4.5)      |
- * +-------------------------+
- * | NAT TABLE (IPA4.5)      |
- * +-------------------------+
- * | NAT IDX TABLE (IPA4.5)  |
- * +-------------------------+
- * | NAT EXP TABLE (IPA4.5)  |
- * +-------------------------+
- * |    CANARY (IPA4.5)      |
- * +-------------------------+
- * |    CANARY (IPA4.5)      |
- * +-------------------------+
- * | PDN CONFIG              |
- * +-------------------------+
- * |    CANARY               |
- * +-------------------------+
- * |    CANARY               |
- * +-------------------------+
- * | QUOTA STATS             |
- * +-------------------------+
- * | TETH STATS              |
- * +-------------------------+
- * | FnR STATS               |
- * +-------------------------+
- * | DROP STATS              |
- * +-------------------------+
- * |    CANARY (IPA4.5)      |
- * +-------------------------+
- * |    CANARY (IPA4.5)      |
- * +-------------------------+
- * | MODEM MEM               |
- * +-------------------------+
- * |    Dummy (IPA4.5)       |
- * +-------------------------+
- * |    CANARY (IPA4.5)      |
- * +-------------------------+
- * | UC DESC RAM (IPA3.5)    |
- * +-------------------------+
- */
-struct ipa3_mem_partition {
-	u32 ofst_start;
-	u32 v4_flt_hash_ofst;
-	u32 v4_flt_hash_size;
-	u32 v4_flt_hash_size_ddr;
-	u32 v4_flt_nhash_ofst;
-	u32 v4_flt_nhash_size;
-	u32 v4_flt_nhash_size_ddr;
-	u32 v6_flt_hash_ofst;
-	u32 v6_flt_hash_size;
-	u32 v6_flt_hash_size_ddr;
-	u32 v6_flt_nhash_ofst;
-	u32 v6_flt_nhash_size;
-	u32 v6_flt_nhash_size_ddr;
-	u32 v4_rt_num_index;
-	u32 v4_modem_rt_index_lo;
-	u32 v4_modem_rt_index_hi;
-	u32 v4_apps_rt_index_lo;
-	u32 v4_apps_rt_index_hi;
-	u32 v4_rt_hash_ofst;
-	u32 v4_rt_hash_size;
-	u32 v4_rt_hash_size_ddr;
-	u32 v4_rt_nhash_ofst;
-	u32 v4_rt_nhash_size;
-	u32 v4_rt_nhash_size_ddr;
-	u32 v6_rt_num_index;
-	u32 v6_modem_rt_index_lo;
-	u32 v6_modem_rt_index_hi;
-	u32 v6_apps_rt_index_lo;
-	u32 v6_apps_rt_index_hi;
-	u32 v6_rt_hash_ofst;
-	u32 v6_rt_hash_size;
-	u32 v6_rt_hash_size_ddr;
-	u32 v6_rt_nhash_ofst;
-	u32 v6_rt_nhash_size;
-	u32 v6_rt_nhash_size_ddr;
-	u32 modem_hdr_ofst;
-	u32 modem_hdr_size;
-	u32 apps_hdr_ofst;
-	u32 apps_hdr_size;
-	u32 apps_hdr_size_ddr;
-	u32 modem_hdr_proc_ctx_ofst;
-	u32 modem_hdr_proc_ctx_size;
-	u32 apps_hdr_proc_ctx_ofst;
-	u32 apps_hdr_proc_ctx_size;
-	u32 apps_hdr_proc_ctx_size_ddr;
-	u32 nat_tbl_ofst;
-	u32 nat_tbl_size;
-	u32 nat_index_tbl_ofst;
-	u32 nat_index_tbl_size;
-	u32 nat_exp_tbl_ofst;
-	u32 nat_exp_tbl_size;
-	u32 modem_comp_decomp_ofst;
-	u32 modem_comp_decomp_size;
-	u32 modem_ofst;
-	u32 modem_size;
-	u32 apps_v4_flt_hash_ofst;
-	u32 apps_v4_flt_hash_size;
-	u32 apps_v4_flt_nhash_ofst;
-	u32 apps_v4_flt_nhash_size;
-	u32 apps_v6_flt_hash_ofst;
-	u32 apps_v6_flt_hash_size;
-	u32 apps_v6_flt_nhash_ofst;
-	u32 apps_v6_flt_nhash_size;
-	u32 uc_info_ofst;
-	u32 uc_info_size;
-	u32 end_ofst;
-	u32 apps_v4_rt_hash_ofst;
-	u32 apps_v4_rt_hash_size;
-	u32 apps_v4_rt_nhash_ofst;
-	u32 apps_v4_rt_nhash_size;
-	u32 apps_v6_rt_hash_ofst;
-	u32 apps_v6_rt_hash_size;
-	u32 apps_v6_rt_nhash_ofst;
-	u32 apps_v6_rt_nhash_size;
-	u32 uc_descriptor_ram_ofst;
-	u32 uc_descriptor_ram_size;
-	u32 pdn_config_ofst;
-	u32 pdn_config_size;
-	u32 stats_quota_ofst;
-	u32 stats_quota_size;
-	u32 stats_tethering_ofst;
-	u32 stats_tethering_size;
-	u32 stats_fnr_ofst;
-	u32 stats_fnr_size;
-
-	/* Irrelevant starting IPA4.5 */
-	u32 stats_flt_v4_ofst;
-	u32 stats_flt_v4_size;
-	u32 stats_flt_v6_ofst;
-	u32 stats_flt_v6_size;
-	u32 stats_rt_v4_ofst;
-	u32 stats_rt_v4_size;
-	u32 stats_rt_v6_ofst;
-	u32 stats_rt_v6_size;
-
-	u32 stats_drop_ofst;
-	u32 stats_drop_size;
-};
-
-struct ipa3_controller {
-	struct ipa3_mem_partition *mem_partition;
-	u32 ipa_clk_rate_turbo;
-	u32 ipa_clk_rate_nominal;
-	u32 ipa_clk_rate_svs;
-	u32 ipa_clk_rate_svs2;
-	u32 clock_scaling_bw_threshold_turbo;
-	u32 clock_scaling_bw_threshold_nominal;
-	u32 clock_scaling_bw_threshold_svs;
-	u32 ipa_reg_base_ofst;
-	u32 max_holb_tmr_val;
-	void (*ipa_sram_read_settings)(void);
-	int (*ipa_init_sram)(void);
-	int (*ipa_init_hdr)(void);
-	int (*ipa_init_rt4)(void);
-	int (*ipa_init_rt6)(void);
-	int (*ipa_init_flt4)(void);
-	int (*ipa_init_flt6)(void);
-	int (*ipa3_read_ep_reg)(char *buff, int max_len, int pipe);
-	int (*ipa3_commit_flt)(enum ipa_ip_type ip);
-	int (*ipa3_commit_rt)(enum ipa_ip_type ip);
-	int (*ipa3_commit_hdr)(void);
-	void (*ipa3_enable_clks)(void);
-	void (*ipa3_disable_clks)(void);
-	struct msm_bus_scale_pdata *msm_bus_data_ptr;
-};
-
-extern struct ipa3_context *ipa3_ctx;
-
-/* public APIs */
-/* Generic GSI channels functions */
-int ipa3_request_gsi_channel(struct ipa_request_gsi_channel_params *params,
-			     struct ipa_req_chan_out_params *out_params);
-
-int ipa3_release_gsi_channel(u32 clnt_hdl);
-
-int ipa3_start_gsi_channel(u32 clnt_hdl);
-
-int ipa3_stop_gsi_channel(u32 clnt_hdl);
-
-int ipa3_reset_gsi_channel(u32 clnt_hdl);
-
-int ipa3_reset_gsi_event_ring(u32 clnt_hdl);
-
-/* Specific xDCI channels functions */
-int ipa3_set_usb_max_packet_size(
-	enum ipa_usb_max_usb_packet_size usb_max_packet_size);
-
-int ipa3_xdci_start(u32 clnt_hdl, u8 xferrscidx, bool xferrscidx_valid);
-
-int ipa3_xdci_connect(u32 clnt_hdl);
-
-int ipa3_xdci_disconnect(u32 clnt_hdl, bool should_force_clear, u32 qmi_req_id);
-
-void ipa3_xdci_ep_delay_rm(u32 clnt_hdl);
-void ipa3_register_client_callback(int (*client_cb)(bool),
-		bool (*teth_port_state)(void),
-		enum ipa_client_type client_type);
-void ipa3_deregister_client_callback(enum ipa_client_type client_type);
-int ipa3_set_reset_client_prod_pipe_delay(bool set_reset,
-		enum ipa_client_type client);
-int ipa3_start_stop_client_prod_gsi_chnl(enum ipa_client_type client,
-		bool start_chnl);
-void ipa3_client_prod_post_shutdown_cleanup(void);
-
-
-int ipa3_set_reset_client_cons_pipe_sus_holb(bool set_reset,
-		enum ipa_client_type client);
-
-int ipa3_xdci_suspend(u32 ul_clnt_hdl, u32 dl_clnt_hdl,
-	bool should_force_clear, u32 qmi_req_id, bool is_dpl);
-
-int ipa3_xdci_resume(u32 ul_clnt_hdl, u32 dl_clnt_hdl, bool is_dpl);
-
-/*
- * Remove ep delay
- */
-int ipa3_clear_endpoint_delay(u32 clnt_hdl);
-
-/*
- * Configuration
- */
-int ipa3_cfg_ep(u32 clnt_hdl, const struct ipa_ep_cfg *ipa_ep_cfg);
-
-int ipa3_cfg_ep_nat(u32 clnt_hdl, const struct ipa_ep_cfg_nat *ipa_ep_cfg);
-
-int ipa3_cfg_ep_conn_track(u32 clnt_hdl,
-	const struct ipa_ep_cfg_conn_track *ep_conn_track);
-
-int ipa3_cfg_ep_hdr(u32 clnt_hdl, const struct ipa_ep_cfg_hdr *ipa_ep_cfg);
-
-int ipa3_cfg_ep_hdr_ext(u32 clnt_hdl,
-			const struct ipa_ep_cfg_hdr_ext *ipa_ep_cfg);
-
-int ipa3_cfg_ep_mode(u32 clnt_hdl, const struct ipa_ep_cfg_mode *ipa_ep_cfg);
-
-int ipa3_cfg_ep_aggr(u32 clnt_hdl, const struct ipa_ep_cfg_aggr *ipa_ep_cfg);
-
-int ipa3_cfg_ep_deaggr(u32 clnt_hdl,
-		      const struct ipa_ep_cfg_deaggr *ipa_ep_cfg);
-
-int ipa3_cfg_ep_route(u32 clnt_hdl, const struct ipa_ep_cfg_route *ipa_ep_cfg);
-
-int ipa3_cfg_ep_holb(u32 clnt_hdl, const struct ipa_ep_cfg_holb *ipa_ep_cfg);
-
-void ipa3_cal_ep_holb_scale_base_val(u32 tmr_val,
-				struct ipa_ep_cfg_holb *ep_holb);
-
-int ipa3_cfg_ep_cfg(u32 clnt_hdl, const struct ipa_ep_cfg_cfg *ipa_ep_cfg);
-
-int ipa3_cfg_ep_metadata_mask(u32 clnt_hdl,
-		const struct ipa_ep_cfg_metadata_mask *ipa_ep_cfg);
-
-int ipa3_cfg_ep_holb_by_client(enum ipa_client_type client,
-				const struct ipa_ep_cfg_holb *ipa_ep_cfg);
-
-int ipa3_cfg_ep_ctrl(u32 clnt_hdl, const struct ipa_ep_cfg_ctrl *ep_ctrl);
-
-/*
- * Header removal / addition
- */
-int ipa3_add_hdr(struct ipa_ioc_add_hdr *hdrs);
-
-int ipa3_add_hdr_usr(struct ipa_ioc_add_hdr *hdrs, bool by_user);
-
-int ipa3_del_hdr(struct ipa_ioc_del_hdr *hdls);
-
-int ipa3_del_hdr_by_user(struct ipa_ioc_del_hdr *hdls, bool by_user);
-
-int ipa3_commit_hdr(void);
-
-int ipa3_reset_hdr(bool user_only);
-
-int ipa3_get_hdr(struct ipa_ioc_get_hdr *lookup);
-
-int ipa3_put_hdr(u32 hdr_hdl);
-
-int ipa3_copy_hdr(struct ipa_ioc_copy_hdr *copy);
-
-/*
- * Header Processing Context
- */
-int ipa3_add_hdr_proc_ctx(struct ipa_ioc_add_hdr_proc_ctx *proc_ctxs,
-							bool user_only);
-
-int ipa3_del_hdr_proc_ctx(struct ipa_ioc_del_hdr_proc_ctx *hdls);
-
-int ipa3_del_hdr_proc_ctx_by_user(struct ipa_ioc_del_hdr_proc_ctx *hdls,
-	bool by_user);
-
-/*
- * Routing
- */
-int ipa3_add_rt_rule(struct ipa_ioc_add_rt_rule *rules);
-
-int ipa3_add_rt_rule_v2(struct ipa_ioc_add_rt_rule_v2 *rules);
-
-int ipa3_add_rt_rule_usr(struct ipa_ioc_add_rt_rule *rules,
-	bool user_only);
-
-int ipa3_add_rt_rule_usr_v2(struct ipa_ioc_add_rt_rule_v2 *rules,
-	bool user_only);
-
-int ipa3_add_rt_rule_ext(struct ipa_ioc_add_rt_rule_ext *rules);
-
-int ipa3_add_rt_rule_ext_v2(struct ipa_ioc_add_rt_rule_ext_v2 *rules);
-
-int ipa3_add_rt_rule_after(struct ipa_ioc_add_rt_rule_after *rules);
-
-int ipa3_add_rt_rule_after_v2(struct ipa_ioc_add_rt_rule_after_v2
-	*rules);
-
-int ipa3_del_rt_rule(struct ipa_ioc_del_rt_rule *hdls);
-
-int ipa3_commit_rt(enum ipa_ip_type ip);
-
-int ipa3_reset_rt(enum ipa_ip_type ip, bool user_only);
-
-int ipa3_get_rt_tbl(struct ipa_ioc_get_rt_tbl *lookup);
-
-int ipa3_put_rt_tbl(u32 rt_tbl_hdl);
-
-int ipa3_query_rt_index(struct ipa_ioc_get_rt_tbl_indx *in);
-
-int ipa3_mdfy_rt_rule(struct ipa_ioc_mdfy_rt_rule *rules);
-
-int ipa3_mdfy_rt_rule_v2(struct ipa_ioc_mdfy_rt_rule_v2 *rules);
-
-/*
- * Filtering
- */
-int ipa3_add_flt_rule(struct ipa_ioc_add_flt_rule *rules);
-
-int ipa3_add_flt_rule_v2(struct ipa_ioc_add_flt_rule_v2 *rules);
-
-int ipa3_add_flt_rule_usr(struct ipa_ioc_add_flt_rule *rules,
-	bool user_only);
-
-int ipa3_add_flt_rule_usr_v2(struct ipa_ioc_add_flt_rule_v2 *rules,
-	bool user_only);
-
-int ipa3_add_flt_rule_after(struct ipa_ioc_add_flt_rule_after *rules);
-
-int ipa3_add_flt_rule_after_v2(struct ipa_ioc_add_flt_rule_after_v2
-	*rules);
-
-int ipa3_del_flt_rule(struct ipa_ioc_del_flt_rule *hdls);
-
-int ipa3_mdfy_flt_rule(struct ipa_ioc_mdfy_flt_rule *rules);
-
-int ipa3_mdfy_flt_rule_v2(struct ipa_ioc_mdfy_flt_rule_v2 *rules);
-
-int ipa3_commit_flt(enum ipa_ip_type ip);
-
-int ipa3_reset_flt(enum ipa_ip_type ip, bool user_only);
-
-/*
- * NAT
- */
-int ipa3_nat_ipv6ct_init_devices(void);
-void ipa3_nat_ipv6ct_destroy_devices(void);
-
-int ipa3_allocate_nat_device(struct ipa_ioc_nat_alloc_mem *mem);
-int ipa3_allocate_nat_table(
-	struct ipa_ioc_nat_ipv6ct_table_alloc *table_alloc);
-int ipa3_allocate_ipv6ct_table(
-	struct ipa_ioc_nat_ipv6ct_table_alloc *table_alloc);
-
-int ipa3_nat_init_cmd(struct ipa_ioc_v4_nat_init *init);
-int ipa3_ipv6ct_init_cmd(struct ipa_ioc_ipv6ct_init *init);
-
-int ipa3_table_dma_cmd(struct ipa_ioc_nat_dma_cmd *dma);
-int ipa3_nat_dma_cmd(struct ipa_ioc_nat_dma_cmd *dma);
-
-int ipa3_nat_del_cmd(struct ipa_ioc_v4_nat_del *del);
-int ipa3_del_nat_table(struct ipa_ioc_nat_ipv6ct_table_del *del);
-int ipa3_del_ipv6ct_table(struct ipa_ioc_nat_ipv6ct_table_del *del);
-
-int ipa3_nat_mdfy_pdn(struct ipa_ioc_nat_pdn_entry *mdfy_pdn);
-
-/*
- * Messaging
- */
-int ipa3_send_msg(struct ipa_msg_meta *meta, void *buff,
-		  ipa_msg_free_fn callback);
-int ipa3_resend_wlan_msg(void);
-int ipa3_register_pull_msg(struct ipa_msg_meta *meta, ipa_msg_pull_fn callback);
-int ipa3_deregister_pull_msg(struct ipa_msg_meta *meta);
-
-/*
- * Interface
- */
-int ipa3_register_intf(const char *name, const struct ipa_tx_intf *tx,
-		       const struct ipa_rx_intf *rx);
-int ipa3_register_intf_ext(const char *name, const struct ipa_tx_intf *tx,
-		       const struct ipa_rx_intf *rx,
-		       const struct ipa_ext_intf *ext);
-int ipa3_deregister_intf(const char *name);
-
-/*
- * Aggregation
- */
-int ipa3_set_aggr_mode(enum ipa_aggr_mode mode);
-
-int ipa3_set_qcncm_ndp_sig(char sig[3]);
-
-int ipa3_set_single_ndp_per_mbim(bool enable);
-
-/*
- * Data path
- */
-int ipa3_tx_dp(enum ipa_client_type dst, struct sk_buff *skb,
-		struct ipa_tx_meta *metadata);
-
-/*
- * To transfer multiple data packets
- * While passing the data descriptor list, the anchor node
- * should be of type struct ipa_tx_data_desc not list_head
- */
-int ipa3_tx_dp_mul(enum ipa_client_type dst,
-			struct ipa_tx_data_desc *data_desc);
-
-void ipa3_free_skb(struct ipa_rx_data *data);
-
-/*
- * System pipes
- */
-int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl);
-
-int ipa3_teardown_sys_pipe(u32 clnt_hdl);
-
-int ipa3_sys_setup(struct ipa_sys_connect_params *sys_in,
-	unsigned long *ipa_transport_hdl,
-	u32 *ipa_pipe_num, u32 *clnt_hdl, bool en_status);
-
-int ipa3_sys_teardown(u32 clnt_hdl);
-
-int ipa3_sys_update_gsi_hdls(u32 clnt_hdl, unsigned long gsi_ch_hdl,
-	unsigned long gsi_ev_hdl);
-
-int ipa3_connect_wdi_pipe(struct ipa_wdi_in_params *in,
-		struct ipa_wdi_out_params *out);
-int ipa3_connect_gsi_wdi_pipe(struct ipa_wdi_in_params *in,
-		struct ipa_wdi_out_params *out);
-
-int ipa3_disconnect_wdi_pipe(u32 clnt_hdl);
-int ipa3_enable_wdi_pipe(u32 clnt_hdl);
-int ipa3_enable_gsi_wdi_pipe(u32 clnt_hdl);
-int ipa3_disable_wdi_pipe(u32 clnt_hdl);
-int ipa3_disable_gsi_wdi_pipe(u32 clnt_hdl);
-int ipa3_disconnect_gsi_wdi_pipe(u32 clnt_hdl);
-int ipa3_resume_wdi_pipe(u32 clnt_hdl);
-int ipa3_resume_gsi_wdi_pipe(u32 clnt_hdl);
-int ipa3_suspend_wdi_pipe(u32 clnt_hdl);
-int ipa3_get_wdi_gsi_stats(struct ipa3_uc_dbg_ring_stats *stats);
-int ipa3_get_wdi3_gsi_stats(struct ipa3_uc_dbg_ring_stats *stats);
-int ipa3_get_usb_gsi_stats(struct ipa3_uc_dbg_ring_stats *stats);
-int ipa3_get_wdi_stats(struct IpaHwStatsWDIInfoData_t *stats);
-u16 ipa3_get_smem_restr_bytes(void);
-int ipa3_broadcast_wdi_quota_reach_ind(uint32_t fid, uint64_t num_bytes);
-int ipa3_setup_uc_ntn_pipes(struct ipa_ntn_conn_in_params *in,
-		ipa_notify_cb notify, void *priv, u8 hdr_len,
-		struct ipa_ntn_conn_out_params *outp);
-int ipa3_tear_down_uc_offload_pipes(int ipa_ep_idx_ul, int ipa_ep_idx_dl,
-	struct ipa_ntn_conn_in_params *params);
-int ipa3_ntn_uc_reg_rdyCB(void (*ipauc_ready_cb)(void *), void *priv);
-void ipa3_ntn_uc_dereg_rdyCB(void);
-int ipa3_conn_wdi3_pipes(struct ipa_wdi_conn_in_params *in,
-	struct ipa_wdi_conn_out_params *out,
-	ipa_wdi_meter_notifier_cb wdi_notify);
-int ipa3_disconn_wdi3_pipes(int ipa_ep_idx_tx, int ipa_ep_idx_rx);
-int ipa3_enable_wdi3_pipes(int ipa_ep_idx_tx, int ipa_ep_idx_rx);
-int ipa3_disable_wdi3_pipes(int ipa_ep_idx_tx, int ipa_ep_idx_rx);
-
-int ipa3_conn_wigig_rx_pipe_i(void *in,
-	struct ipa_wigig_conn_out_params *out,
-	struct dentry **parent);
-
-int ipa3_conn_wigig_client_i(void *in,
-	struct ipa_wigig_conn_out_params *out,
-	ipa_notify_cb tx_notify,
-	void *priv);
-
-int ipa3_wigig_uc_msi_init(bool init,
-	phys_addr_t periph_baddr_pa,
-	phys_addr_t pseudo_cause_pa,
-	phys_addr_t int_gen_tx_pa,
-	phys_addr_t int_gen_rx_pa,
-	phys_addr_t dma_ep_misc_pa);
-
-int ipa3_disconn_wigig_pipe_i(enum ipa_client_type client,
-	struct ipa_wigig_pipe_setup_info_smmu *pipe_smmu,
-	void *dbuff);
-
-int ipa3_enable_wigig_pipe_i(enum ipa_client_type client);
-
-int ipa3_disable_wigig_pipe_i(enum ipa_client_type client);
-
-int ipa3_wigig_init_debugfs_i(struct dentry *dent);
-
-/*
- * To retrieve doorbell physical address of
- * wlan pipes
- */
-int ipa3_uc_wdi_get_dbpa(struct ipa_wdi_db_params *out);
-
-/*
- * To register uC ready callback if uC not ready
- * and also check uC readiness
- * if uC not ready only, register callback
- */
-int ipa3_uc_reg_rdyCB(struct ipa_wdi_uc_ready_params *param);
-/*
- * To de-register uC ready callback
- */
-int ipa3_uc_dereg_rdyCB(void);
-
-int ipa_create_uc_smmu_mapping(int res_idx, bool wlan_smmu_en,
-		phys_addr_t pa, struct sg_table *sgt, size_t len, bool device,
-		unsigned long *iova);
-
-int ipa_create_gsi_smmu_mapping(int res_idx, bool wlan_smmu_en,
-		phys_addr_t pa, struct sg_table *sgt, size_t len, bool device,
-		unsigned long *iova);
-
-void ipa3_release_wdi3_gsi_smmu_mappings(u8 dir);
-
-/*
- * Tethering bridge (Rmnet / MBIM)
- */
-int ipa3_teth_bridge_init(struct teth_bridge_init_params *params);
-
-int ipa3_teth_bridge_disconnect(enum ipa_client_type client);
-
-int ipa3_teth_bridge_connect(struct teth_bridge_connect_params *connect_params);
-
-int ipa3_teth_bridge_get_pm_hdl(void);
-
-/*
- * Tethering client info
- */
-void ipa3_set_client(int index, enum ipacm_client_enum client, bool uplink);
-
-enum ipacm_client_enum ipa3_get_client(int pipe_idx);
-
-bool ipa3_get_client_uplink(int pipe_idx);
-
-int ipa3_get_wlan_stats(struct ipa_get_wdi_sap_stats *wdi_sap_stats);
-
-int ipa3_set_wlan_quota(struct ipa_set_wifi_quota *wdi_quota);
-/*
- * IPADMA
- */
-int ipa3_dma_init(void);
-
-int ipa3_dma_enable(void);
-
-int ipa3_dma_disable(void);
-
-int ipa3_dma_sync_memcpy(u64 dest, u64 src, int len);
-
-int ipa3_dma_async_memcpy(u64 dest, u64 src, int len,
-			void (*user_cb)(void *user1), void *user_param);
-
-int ipa3_dma_uc_memcpy(phys_addr_t dest, phys_addr_t src, int len);
-
-void ipa3_dma_destroy(void);
-
-/*
- * MHI
- */
-
-int ipa3_mhi_init_engine(struct ipa_mhi_init_engine *params);
-
-int ipa3_connect_mhi_pipe(
-		struct ipa_mhi_connect_params_internal *in,
-		u32 *clnt_hdl);
-
-int ipa3_disconnect_mhi_pipe(u32 clnt_hdl);
-
-bool ipa3_mhi_stop_gsi_channel(enum ipa_client_type client);
-
-int ipa3_mhi_reset_channel_internal(enum ipa_client_type client);
-
-int ipa3_mhi_start_channel_internal(enum ipa_client_type client);
-
-bool ipa3_has_open_aggr_frame(enum ipa_client_type client);
-
-int ipa3_mhi_resume_channels_internal(enum ipa_client_type client,
-		bool LPTransitionRejected, bool brstmode_enabled,
-		union __packed gsi_channel_scratch ch_scratch, u8 index);
-
-int ipa3_mhi_destroy_channel(enum ipa_client_type client);
-
-/*
- * mux id
- */
-int ipa3_write_qmap_id(struct ipa_ioc_write_qmapid *param_in);
-
-/*
- * interrupts
- */
-int ipa3_add_interrupt_handler(enum ipa_irq_type interrupt,
-		ipa_irq_handler_t handler,
-		bool deferred_flag,
-		void *private_data);
-
-int ipa3_remove_interrupt_handler(enum ipa_irq_type interrupt);
-
-/*
- * Miscellaneous
- */
-int ipa3_get_ep_mapping(enum ipa_client_type client);
-
-bool ipa3_is_ready(void);
-
-void ipa3_proxy_clk_vote(void);
-void ipa3_proxy_clk_unvote(void);
-
-bool ipa3_is_client_handle_valid(u32 clnt_hdl);
-
-enum ipa_client_type ipa3_get_client_mapping(int pipe_idx);
-enum ipa_client_type ipa3_get_client_by_pipe(int pipe_idx);
-
-void ipa_init_ep_flt_bitmap(void);
-
-bool ipa_is_ep_support_flt(int pipe_idx);
-
-bool ipa3_get_modem_cfg_emb_pipe_flt(void);
-
-u8 ipa3_get_qmb_master_sel(enum ipa_client_type client);
-
-int ipa3_get_smmu_params(struct ipa_smmu_in_params *in,
-	struct ipa_smmu_out_params *out);
-
-/* internal functions */
-
-int ipa3_bind_api_controller(enum ipa_hw_type ipa_hw_type,
-	struct ipa_api_controller *api_ctrl);
-
-bool ipa_is_modem_pipe(int pipe_idx);
-
-int ipa3_send_one(struct ipa3_sys_context *sys, struct ipa3_desc *desc,
-		bool in_atomic);
-int ipa3_send(struct ipa3_sys_context *sys,
-		u32 num_desc,
-		struct ipa3_desc *desc,
-		bool in_atomic);
-int ipa3_get_ep_mapping(enum ipa_client_type client);
-int ipa_get_ep_group(enum ipa_client_type client);
-
-int ipa3_generate_hw_rule(enum ipa_ip_type ip,
-			 const struct ipa_rule_attrib *attrib,
-			 u8 **buf,
-			 u16 *en_rule);
-int ipa3_init_hw(void);
-struct ipa3_rt_tbl *__ipa3_find_rt_tbl(enum ipa_ip_type ip, const char *name);
-int ipa3_set_single_ndp_per_mbim(bool enable);
-void ipa3_debugfs_init(void);
-void ipa3_debugfs_remove(void);
-
-void ipa3_dump_buff_internal(void *base, dma_addr_t phy_base, u32 size);
-#ifdef IPA_DEBUG
-#define IPA_DUMP_BUFF(base, phy_base, size) \
-	ipa3_dump_buff_internal(base, phy_base, size)
-#else
-#define IPA_DUMP_BUFF(base, phy_base, size)
-#endif
-int ipa3_init_mem_partition(enum ipa_hw_type ipa_hw_type);
-int ipa3_controller_static_bind(struct ipa3_controller *controller,
-		enum ipa_hw_type ipa_hw_type);
-int ipa3_cfg_route(struct ipahal_reg_route *route);
-int ipa3_send_cmd_timeout(u16 num_desc, struct ipa3_desc *descr, u32 timeout);
-int ipa3_send_cmd(u16 num_desc, struct ipa3_desc *descr);
-int ipa3_cfg_filter(u32 disable);
-int ipa3_straddle_boundary(u32 start, u32 end, u32 boundary);
-struct ipa3_context *ipa3_get_ctx(void);
-void ipa3_enable_clks(void);
-void ipa3_disable_clks(void);
-void ipa3_inc_client_enable_clks(struct ipa_active_client_logging_info *id);
-int ipa3_inc_client_enable_clks_no_block(struct ipa_active_client_logging_info
-		*id);
-void ipa3_dec_client_disable_clks(struct ipa_active_client_logging_info *id);
-void ipa3_dec_client_disable_clks_no_block(
-	struct ipa_active_client_logging_info *id);
-void ipa3_active_clients_log_dec(struct ipa_active_client_logging_info *id,
-		bool int_ctx);
-void ipa3_active_clients_log_inc(struct ipa_active_client_logging_info *id,
-		bool int_ctx);
-int ipa3_active_clients_log_print_buffer(char *buf, int size);
-int ipa3_active_clients_log_print_table(char *buf, int size);
-void ipa3_active_clients_log_clear(void);
-int ipa3_interrupts_init(u32 ipa_irq, u32 ee, struct device *ipa_dev);
-void ipa3_interrupts_destroy(u32 ipa_irq, struct device *ipa_dev);
-int __ipa3_del_rt_rule(u32 rule_hdl);
-int __ipa3_del_hdr(u32 hdr_hdl, bool by_user);
-int __ipa3_release_hdr(u32 hdr_hdl);
-int __ipa3_release_hdr_proc_ctx(u32 proc_ctx_hdl);
-int _ipa_read_ep_reg_v3_0(char *buf, int max_len, int pipe);
-int _ipa_read_ep_reg_v4_0(char *buf, int max_len, int pipe);
-int _ipa_read_ipahal_regs(void);
-void _ipa_enable_clks_v3_0(void);
-void _ipa_disable_clks_v3_0(void);
-struct device *ipa3_get_dma_dev(void);
-void ipa3_suspend_active_aggr_wa(u32 clnt_hdl);
-void ipa3_suspend_handler(enum ipa_irq_type interrupt,
-				void *private_data,
-				void *interrupt_data);
-
-ssize_t ipa3_read(struct file *filp, char __user *buf, size_t count,
-		 loff_t *f_pos);
-int ipa3_pull_msg(struct ipa_msg_meta *meta, char *buff, size_t count);
-int ipa3_query_intf(struct ipa_ioc_query_intf *lookup);
-int ipa3_query_intf_tx_props(struct ipa_ioc_query_intf_tx_props *tx);
-int ipa3_query_intf_rx_props(struct ipa_ioc_query_intf_rx_props *rx);
-int ipa3_query_intf_ext_props(struct ipa_ioc_query_intf_ext_props *ext);
-
-void wwan_cleanup(void);
-
-int ipa3_teth_bridge_driver_init(void);
-void ipa3_lan_rx_cb(void *priv, enum ipa_dp_evt_type evt, unsigned long data);
-
-int _ipa_init_sram_v3(void);
-int _ipa_init_hdr_v3_0(void);
-int _ipa_init_rt4_v3(void);
-int _ipa_init_rt6_v3(void);
-int _ipa_init_flt4_v3(void);
-int _ipa_init_flt6_v3(void);
-
-int __ipa_commit_flt_v3(enum ipa_ip_type ip);
-int __ipa_commit_rt_v3(enum ipa_ip_type ip);
-
-int __ipa_commit_hdr_v3_0(void);
-void ipa3_skb_recycle(struct sk_buff *skb);
-void ipa3_install_dflt_flt_rules(u32 ipa_ep_idx);
-void ipa3_delete_dflt_flt_rules(u32 ipa_ep_idx);
-
-int ipa3_enable_data_path(u32 clnt_hdl);
-int ipa3_disable_data_path(u32 clnt_hdl);
-int ipa3_disable_gsi_data_path(u32 clnt_hdl);
-int ipa3_alloc_rule_id(struct idr *rule_ids);
-int ipa3_alloc_counter_id(struct ipa_ioc_flt_rt_counter_alloc *counter);
-void ipa3_counter_remove_hdl(int hdl);
-void ipa3_counter_id_remove_all(void);
-int ipa3_id_alloc(void *ptr);
-void *ipa3_id_find(u32 id);
-void ipa3_id_remove(u32 id);
-int ipa3_enable_force_clear(u32 request_id, bool throttle_source,
-	u32 source_pipe_bitmask);
-int ipa3_disable_force_clear(u32 request_id);
-
-int ipa3_set_required_perf_profile(enum ipa_voltage_level floor_voltage,
-				  u32 bandwidth_mbps);
-
-int ipa3_cfg_ep_status(u32 clnt_hdl,
-		const struct ipahal_reg_ep_cfg_status *ipa_ep_cfg);
-
-int ipa3_suspend_resource_no_block(enum ipa_rm_resource_name name);
-int ipa3_suspend_resource_sync(enum ipa_rm_resource_name name);
-int ipa3_resume_resource(enum ipa_rm_resource_name name);
-bool ipa3_should_pipe_be_suspended(enum ipa_client_type client);
-int ipa3_tag_aggr_force_close(int pipe_num);
-
-void ipa3_active_clients_unlock(void);
-int ipa3_wdi_init(void);
-int ipa3_write_qmapid_gsi_wdi_pipe(u32 clnt_hdl, u8 qmap_id);
-int ipa3_write_qmapid_wdi_pipe(u32 clnt_hdl, u8 qmap_id);
-int ipa3_write_qmapid_wdi3_gsi_pipe(u32 clnt_hdl, u8 qmap_id);
-int ipa3_tag_process(struct ipa3_desc *desc, int num_descs,
-		    unsigned long timeout);
-
-void ipa3_q6_pre_shutdown_cleanup(void);
-void ipa3_q6_post_shutdown_cleanup(void);
-void ipa3_q6_pre_powerup_cleanup(void);
-void ipa3_update_ssr_state(bool is_ssr);
-int ipa3_init_q6_smem(void);
-
-int ipa3_mhi_handle_ipa_config_req(struct ipa_config_req_msg_v01 *config_req);
-int ipa3_mhi_query_ch_info(enum ipa_client_type client,
-		struct gsi_chan_info *ch_info);
-
-int ipa3_uc_interface_init(void);
-int ipa3_uc_is_gsi_channel_empty(enum ipa_client_type ipa_client);
-int ipa3_uc_state_check(void);
-int ipa3_uc_loaded_check(void);
-int ipa3_uc_register_ready_cb(struct notifier_block *nb);
-int ipa3_uc_unregister_ready_cb(struct notifier_block *nb);
-int ipa3_uc_send_cmd(u32 cmd, u32 opcode, u32 expected_status,
-		    bool polling_mode, unsigned long timeout_jiffies);
-void ipa3_uc_register_handlers(enum ipa3_hw_features feature,
-			      struct ipa3_uc_hdlrs *hdlrs);
-int ipa3_uc_notify_clk_state(bool enabled);
-int ipa3_dma_setup(void);
-void ipa3_dma_shutdown(void);
-void ipa3_dma_async_memcpy_notify_cb(void *priv,
-		enum ipa_dp_evt_type evt, unsigned long data);
-
-int ipa3_uc_update_hw_flags(u32 flags);
-
-int ipa3_uc_mhi_init(void (*ready_cb)(void), void (*wakeup_request_cb)(void));
-void ipa3_uc_mhi_cleanup(void);
-int ipa3_uc_mhi_send_dl_ul_sync_info(union IpaHwMhiDlUlSyncCmdData_t *cmd);
-int ipa3_uc_mhi_init_engine(struct ipa_mhi_msi_info *msi, u32 mmio_addr,
-	u32 host_ctrl_addr, u32 host_data_addr, u32 first_ch_idx,
-	u32 first_evt_idx);
-int ipa3_uc_mhi_init_channel(int ipa_ep_idx, int channelHandle,
-	int contexArrayIndex, int channelDirection);
-int ipa3_uc_mhi_reset_channel(int channelHandle);
-int ipa3_uc_mhi_suspend_channel(int channelHandle);
-int ipa3_uc_mhi_resume_channel(int channelHandle, bool LPTransitionRejected);
-int ipa3_uc_mhi_stop_event_update_channel(int channelHandle);
-int ipa3_uc_mhi_print_stats(char *dbg_buff, int size);
-int ipa3_uc_memcpy(phys_addr_t dest, phys_addr_t src, int len);
-int ipa3_uc_send_remote_ipa_info(u32 remote_addr, uint32_t mbox_n);
-int ipa3_uc_debug_stats_alloc(
-	struct IpaHwOffloadStatsAllocCmdData_t cmdinfo);
-int ipa3_uc_debug_stats_dealloc(uint32_t protocol);
-void ipa3_tag_destroy_imm(void *user1, int user2);
-const struct ipa_gsi_ep_config *ipa3_get_gsi_ep_info
-	(enum ipa_client_type client);
-
-int ipa3_wigig_init_i(void);
-int ipa3_wigig_internal_init(
-	struct ipa_wdi_uc_ready_params *inout,
-	ipa_wigig_misc_int_cb int_notify,
-	phys_addr_t *uc_db_pa);
-
-/* Hardware stats */
-
-#define IPA_STATS_MAX_PIPE_BIT 32
-
-struct ipa_teth_stats_endpoints {
-	u32 prod_mask;
-	u32 dst_ep_mask[IPA_STATS_MAX_PIPE_BIT];
-};
-
-int ipa_hw_stats_init(void);
-
-int ipa_init_flt_rt_stats(void);
-
-int ipa_debugfs_init_stats(struct dentry *parent);
-
-int ipa_init_quota_stats(u32 pipe_bitmask);
-
-int ipa_get_quota_stats(struct ipa_quota_stats_all *out);
-
-int ipa_reset_quota_stats(enum ipa_client_type client);
-
-int ipa_reset_all_quota_stats(void);
-
-int ipa_init_drop_stats(u32 pipe_bitmask);
-
-int ipa_get_drop_stats(struct ipa_drop_stats_all *out);
-
-int ipa_reset_drop_stats(enum ipa_client_type client);
-
-int ipa_reset_all_drop_stats(void);
-
-int ipa_init_teth_stats(struct ipa_teth_stats_endpoints *in);
-
-int ipa_get_teth_stats(void);
-
-int ipa_query_teth_stats(enum ipa_client_type prod,
-	struct ipa_quota_stats_all *out, bool reset);
-
-int ipa_reset_teth_stats(enum ipa_client_type prod, enum ipa_client_type cons);
-
-int ipa_reset_all_cons_teth_stats(enum ipa_client_type prod);
-
-int ipa_reset_all_teth_stats(void);
-
-int ipa_get_flt_rt_stats(struct ipa_ioc_flt_rt_query *query);
-
-int ipa_set_flt_rt_stats(int index, struct ipa_flt_rt_stats stats);
-
-u32 ipa3_get_num_pipes(void);
-struct ipa_smmu_cb_ctx *ipa3_get_smmu_ctx(enum ipa_smmu_cb_type);
-struct iommu_domain *ipa3_get_smmu_domain(void);
-struct iommu_domain *ipa3_get_uc_smmu_domain(void);
-struct iommu_domain *ipa3_get_wlan_smmu_domain(void);
-struct iommu_domain *ipa3_get_smmu_domain_by_type
-	(enum ipa_smmu_cb_type cb_type);
-int ipa3_iommu_map(struct iommu_domain *domain, unsigned long iova,
-	phys_addr_t paddr, size_t size, int prot);
-int ipa3_ap_suspend(struct device *dev);
-int ipa3_ap_resume(struct device *dev);
-int ipa3_init_interrupts(void);
-struct iommu_domain *ipa3_get_smmu_domain(void);
-int ipa3_release_wdi_mapping(u32 num_buffers, struct ipa_wdi_buffer_info *info);
-int ipa3_create_wdi_mapping(u32 num_buffers, struct ipa_wdi_buffer_info *info);
-int ipa3_set_flt_tuple_mask(int pipe_idx, struct ipahal_reg_hash_tuple *tuple);
-int ipa3_set_rt_tuple_mask(int tbl_idx, struct ipahal_reg_hash_tuple *tuple);
-void ipa3_set_resorce_groups_min_max_limits(void);
-void ipa3_suspend_apps_pipes(bool suspend);
-int ipa3_flt_read_tbl_from_hw(u32 pipe_idx,
-	enum ipa_ip_type ip_type,
-	bool hashable,
-	struct ipahal_flt_rule_entry entry[],
-	int *num_entry);
-int ipa3_rt_read_tbl_from_hw(u32 tbl_idx,
-	enum ipa_ip_type ip_type,
-	bool hashable,
-	struct ipahal_rt_rule_entry entry[],
-	int *num_entry);
-int ipa3_restore_suspend_handler(void);
-int ipa3_inject_dma_task_for_gsi(void);
-int ipa3_uc_panic_notifier(struct notifier_block *this,
-	unsigned long event, void *ptr);
-void ipa3_inc_acquire_wakelock(void);
-void ipa3_dec_release_wakelock(void);
-int ipa3_load_fws(const struct firmware *firmware, phys_addr_t gsi_mem_base,
-	enum gsi_ver);
-int emulator_load_fws(
-	const struct firmware *firmware,
-	u32 transport_mem_base,
-	u32 transport_mem_size,
-	enum gsi_ver);
-int ipa3_register_ipa_ready_cb(void (*ipa_ready_cb)(void *), void *user_data);
-const char *ipa_hw_error_str(enum ipa3_hw_errors err_type);
-int ipa_gsi_ch20_wa(void);
-int ipa3_rx_poll(u32 clnt_hdl, int budget);
-int ipa3_smmu_map_peer_reg(phys_addr_t phys_addr, bool map,
-	enum ipa_smmu_cb_type cb_type);
-int ipa3_smmu_map_peer_buff(u64 iova, u32 size, bool map, struct sg_table *sgt,
-	enum ipa_smmu_cb_type cb_type);
-void ipa3_reset_freeze_vote(void);
-int ipa3_ntn_init(void);
-int ipa3_get_ntn_stats(struct Ipa3HwStatsNTNInfoData_t *stats);
-struct dentry *ipa_debugfs_get_root(void);
-struct device *ipa3_get_pdev(void);
-void ipa3_enable_dcd(void);
-void ipa3_disable_prefetch(enum ipa_client_type client);
-int ipa3_alloc_common_event_ring(void);
-int ipa3_allocate_dma_task_for_gsi(void);
-void ipa3_free_dma_task_for_gsi(void);
-int ipa3_set_clock_plan_from_pm(int idx);
-void __ipa_gsi_irq_rx_scedule_poll(struct ipa3_sys_context *sys);
-int ipa3_tz_unlock_reg(struct ipa_tz_unlock_reg_info *reg_info, u16 num_regs);
-void ipa3_init_imm_cmd_desc(struct ipa3_desc *desc,
-	struct ipahal_imm_cmd_pyld *cmd_pyld);
-int ipa3_is_vlan_mode(enum ipa_vlan_ifaces iface, bool *res);
-uint ipa3_get_emulation_type(void);
-int ipa3_get_transport_info(
-	phys_addr_t *phys_addr_ptr,
-	unsigned long *size_ptr);
-irq_handler_t ipa3_get_isr(void);
-void ipa_pc_qmp_enable(void);
-u32 ipa3_get_r_rev_version(void);
-#if defined(CONFIG_IPA3_REGDUMP)
-int ipa_reg_save_init(u32 value);
-void ipa_save_registers(void);
-void ipa_save_gsi_ver(void);
-#else
-static inline int ipa_reg_save_init(u32 value) { return 0; }
-static inline void ipa_save_registers(void) {};
-static inline void ipa_save_gsi_ver(void) {};
-#endif
-
-#ifdef CONFIG_IPA_ETH
-int ipa_eth_init(void);
-void ipa_eth_exit(void);
-#else
-static inline int ipa_eth_init(void) { return 0; }
-static inline void ipa_eth_exit(void) { }
-#endif // CONFIG_IPA_ETH
-int ipa3_get_gsi_chan_info(struct gsi_chan_info *gsi_chan_info,
-	unsigned long chan_hdl);
-#ifdef CONFIG_IPA3_MHI_PRIME_MANAGER
-int ipa_mpm_mhip_xdci_pipe_enable(enum ipa_usb_teth_prot prot);
-int ipa_mpm_mhip_xdci_pipe_disable(enum ipa_usb_teth_prot xdci_teth_prot);
-int ipa_mpm_notify_wan_state(struct wan_ioctl_notify_wan_state *state);
-int ipa3_is_mhip_offload_enabled(void);
-int ipa_mpm_reset_dma_mode(enum ipa_client_type src_pipe,
-	enum ipa_client_type dst_pipe);
-int ipa_mpm_panic_handler(char *buf, int size);
-int ipa3_get_mhip_gsi_stats(struct ipa3_uc_dbg_ring_stats *stats);
-#else
-static inline int ipa_mpm_mhip_xdci_pipe_enable(
-	enum ipa_usb_teth_prot prot)
-{
-	return 0;
-}
-static inline int ipa_mpm_mhip_xdci_pipe_disable(
-	enum ipa_usb_teth_prot xdci_teth_prot)
-{
-	return 0;
-}
-static inline int ipa_mpm_notify_wan_state(
-	struct wan_ioctl_notify_wan_state *state)
-{
-	return 0;
-}
-static inline int ipa3_is_mhip_offload_enabled(void)
-{
-	return 0;
-}
-static inline int ipa_mpm_reset_dma_mode(enum ipa_client_type src_pipe,
-	enum ipa_client_type dst_pipe)
-{
-	return 0;
-}
-static inline int ipa_mpm_panic_handler(char *buf, int size)
-{
-	return 0;
-}
-
-static inline int ipa3_get_mhip_gsi_stats(struct ipa3_uc_dbg_ring_stats *stats)
-{
-	return 0;
-}
-#endif /* CONFIG_IPA3_MHI_PRIME_MANAGER */
-
-static inline void *alloc_and_init(u32 size, u32 init_val)
-{
-	void *ptr = kmalloc(size, GFP_KERNEL);
-
-	if (ptr)
-		memset(ptr, init_val, size);
-
-	return ptr;
-}
-
-/* query ipa APQ mode*/
-bool ipa3_is_apq(void);
-#endif /* _IPA3_I_H_ */

ipa/ipa_v3/ipa_interrupts.c

@@ -1,609 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2014-2018, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/interrupt.h>
-#include "ipa_i.h"
-
-#define INTERRUPT_WORKQUEUE_NAME "ipa_interrupt_wq"
-#define DIS_SUSPEND_INTERRUPT_TIMEOUT 5
-#define IPA_IRQ_NUM_MAX 32
-
-struct ipa3_interrupt_info {
-	ipa_irq_handler_t handler;
-	enum ipa_irq_type interrupt;
-	void *private_data;
-	bool deferred_flag;
-};
-
-struct ipa3_interrupt_work_wrap {
-	struct work_struct interrupt_work;
-	ipa_irq_handler_t handler;
-	enum ipa_irq_type interrupt;
-	void *private_data;
-	void *interrupt_data;
-};
-
-static struct ipa3_interrupt_info ipa_interrupt_to_cb[IPA_IRQ_NUM_MAX];
-static struct workqueue_struct *ipa_interrupt_wq;
-static u32 ipa_ee;
-
-static void ipa3_tx_suspend_interrupt_wa(void);
-static void ipa3_enable_tx_suspend_wa(struct work_struct *work);
-static DECLARE_DELAYED_WORK(dwork_en_suspend_int,
-						ipa3_enable_tx_suspend_wa);
-static spinlock_t suspend_wa_lock;
-static void ipa3_process_interrupts(bool isr_context);
-
-static int ipa3_irq_mapping[IPA_IRQ_MAX] = {
-	[IPA_BAD_SNOC_ACCESS_IRQ]		= 0,
-	[IPA_UC_IRQ_0]				= 2,
-	[IPA_UC_IRQ_1]				= 3,
-	[IPA_UC_IRQ_2]				= 4,
-	[IPA_UC_IRQ_3]				= 5,
-	[IPA_UC_IN_Q_NOT_EMPTY_IRQ]		= 6,
-	[IPA_UC_RX_CMD_Q_NOT_FULL_IRQ]		= 7,
-	[IPA_PROC_TO_UC_ACK_Q_NOT_EMPTY_IRQ]	= 8,
-	[IPA_RX_ERR_IRQ]			= 9,
-	[IPA_DEAGGR_ERR_IRQ]			= 10,
-	[IPA_TX_ERR_IRQ]			= 11,
-	[IPA_STEP_MODE_IRQ]			= 12,
-	[IPA_PROC_ERR_IRQ]			= 13,
-	[IPA_TX_SUSPEND_IRQ]			= 14,
-	[IPA_TX_HOLB_DROP_IRQ]			= 15,
-	[IPA_BAM_GSI_IDLE_IRQ]			= 16,
-	[IPA_PIPE_YELLOW_MARKER_BELOW_IRQ]	= 17,
-	[IPA_PIPE_RED_MARKER_BELOW_IRQ]		= 18,
-	[IPA_PIPE_YELLOW_MARKER_ABOVE_IRQ]	= 19,
-	[IPA_PIPE_RED_MARKER_ABOVE_IRQ]		= 20,
-	[IPA_UCP_IRQ]				= 21,
-	[IPA_DCMP_IRQ]				= 22,
-	[IPA_GSI_EE_IRQ]			= 23,
-	[IPA_GSI_IPA_IF_TLV_RCVD_IRQ]		= 24,
-	[IPA_GSI_UC_IRQ]			= 25,
-	[IPA_TLV_LEN_MIN_DSM_IRQ]		= 26,
-};
-
-static void ipa3_interrupt_defer(struct work_struct *work);
-static DECLARE_WORK(ipa3_interrupt_defer_work, ipa3_interrupt_defer);
-
-static void ipa3_deferred_interrupt_work(struct work_struct *work)
-{
-	struct ipa3_interrupt_work_wrap *work_data =
-			container_of(work,
-			struct ipa3_interrupt_work_wrap,
-			interrupt_work);
-	IPADBG("call handler from workq for interrupt %d...\n",
-		work_data->interrupt);
-	work_data->handler(work_data->interrupt, work_data->private_data,
-			work_data->interrupt_data);
-	kfree(work_data->interrupt_data);
-	kfree(work_data);
-}
-
-static bool ipa3_is_valid_ep(u32 ep_suspend_data)
-{
-	u32 bmsk = 1;
-	u32 i = 0;
-
-	for (i = 0; i < ipa3_ctx->ipa_num_pipes; i++) {
-		if ((ep_suspend_data & bmsk) && (ipa3_ctx->ep[i].valid))
-			return true;
-		bmsk = bmsk << 1;
-	}
-	return false;
-}
-
-static int ipa3_handle_interrupt(int irq_num, bool isr_context)
-{
-	struct ipa3_interrupt_info interrupt_info;
-	struct ipa3_interrupt_work_wrap *work_data;
-	u32 suspend_data;
-	void *interrupt_data = NULL;
-	struct ipa_tx_suspend_irq_data *suspend_interrupt_data = NULL;
-	int res;
-
-	interrupt_info = ipa_interrupt_to_cb[irq_num];
-	if (interrupt_info.handler == NULL) {
-		IPAERR("A callback function wasn't set for interrupt num %d\n",
-			irq_num);
-		return -EINVAL;
-	}
-
-	switch (interrupt_info.interrupt) {
-	case IPA_TX_SUSPEND_IRQ:
-		IPADBG_LOW("processing TX_SUSPEND interrupt\n");
-		ipa3_tx_suspend_interrupt_wa();
-		suspend_data = ipahal_read_reg_n(IPA_SUSPEND_IRQ_INFO_EE_n,
-			ipa_ee);
-		IPADBG_LOW("get interrupt %d\n", suspend_data);
-
-		if (ipa3_ctx->ipa_hw_type >= IPA_HW_v3_1) {
-			/* Clearing L2 interrupts status */
-			ipahal_write_reg_n(IPA_SUSPEND_IRQ_CLR_EE_n,
-				ipa_ee, suspend_data);
-		}
-		if (!ipa3_is_valid_ep(suspend_data))
-			return 0;
-
-		suspend_interrupt_data =
-			kzalloc(sizeof(*suspend_interrupt_data), GFP_ATOMIC);
-		if (!suspend_interrupt_data) {
-			IPAERR("failed allocating suspend_interrupt_data\n");
-			return -ENOMEM;
-		}
-		suspend_interrupt_data->endpoints = suspend_data;
-		interrupt_data = suspend_interrupt_data;
-		break;
-	default:
-		break;
-	}
-
-	/* Force defer processing if in ISR context. */
-	if (interrupt_info.deferred_flag || isr_context) {
-		IPADBG_LOW("Defer handling interrupt %d\n",
-			interrupt_info.interrupt);
-		work_data = kzalloc(sizeof(struct ipa3_interrupt_work_wrap),
-				GFP_ATOMIC);
-		if (!work_data) {
-			IPAERR("failed allocating ipa3_interrupt_work_wrap\n");
-			res = -ENOMEM;
-			goto fail_alloc_work;
-		}
-		INIT_WORK(&work_data->interrupt_work,
-				ipa3_deferred_interrupt_work);
-		work_data->handler = interrupt_info.handler;
-		work_data->interrupt = interrupt_info.interrupt;
-		work_data->private_data = interrupt_info.private_data;
-		work_data->interrupt_data = interrupt_data;
-		queue_work(ipa_interrupt_wq, &work_data->interrupt_work);
-
-	} else {
-		IPADBG_LOW("Handle interrupt %d\n", interrupt_info.interrupt);
-		interrupt_info.handler(interrupt_info.interrupt,
-			interrupt_info.private_data,
-			interrupt_data);
-		kfree(interrupt_data);
-	}
-
-	return 0;
-
-fail_alloc_work:
-	kfree(interrupt_data);
-	return res;
-}
-
-static void ipa3_enable_tx_suspend_wa(struct work_struct *work)
-{
-	u32 en;
-	u32 suspend_bmask;
-	int irq_num;
-
-	IPADBG_LOW("Enter\n");
-
-	irq_num = ipa3_irq_mapping[IPA_TX_SUSPEND_IRQ];
-
-	if (irq_num == -1) {
-		WARN_ON(1);
-		return;
-	}
-
-	/* make sure ipa hw is clocked on*/
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	en = ipahal_read_reg_n(IPA_IRQ_EN_EE_n, ipa_ee);
-	suspend_bmask = 1 << irq_num;
-	/*enable  TX_SUSPEND_IRQ*/
-	en |= suspend_bmask;
-	IPADBG("enable TX_SUSPEND_IRQ, IPA_IRQ_EN_EE reg, write val = %u\n"
-		, en);
-	ipahal_write_reg_n(IPA_IRQ_EN_EE_n, ipa_ee, en);
-	ipa3_process_interrupts(false);
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-
-	IPADBG_LOW("Exit\n");
-}
-
-static void ipa3_tx_suspend_interrupt_wa(void)
-{
-	u32 val;
-	u32 suspend_bmask;
-	int irq_num;
-	int wa_delay;
-
-	IPADBG_LOW("Enter\n");
-	irq_num = ipa3_irq_mapping[IPA_TX_SUSPEND_IRQ];
-
-	if (irq_num == -1) {
-		WARN_ON(1);
-		return;
-	}
-
-	/*disable TX_SUSPEND_IRQ*/
-	val = ipahal_read_reg_n(IPA_IRQ_EN_EE_n, ipa_ee);
-	suspend_bmask = 1 << irq_num;
-	val &= ~suspend_bmask;
-	IPADBG("Disabling TX_SUSPEND_IRQ, write val: %u to IPA_IRQ_EN_EE reg\n",
-		val);
-	ipahal_write_reg_n(IPA_IRQ_EN_EE_n, ipa_ee, val);
-
-	IPADBG_LOW(" processing suspend interrupt work-around, delayed work\n");
-
-	wa_delay = DIS_SUSPEND_INTERRUPT_TIMEOUT;
-	if (ipa3_ctx->ipa3_hw_mode == IPA_HW_MODE_VIRTUAL ||
-	    ipa3_ctx->ipa3_hw_mode == IPA_HW_MODE_EMULATION) {
-		wa_delay *= 400;
-	}
-
-	IPADBG_LOW("Delay period %d msec\n", wa_delay);
-
-	queue_delayed_work(ipa_interrupt_wq, &dwork_en_suspend_int,
-			msecs_to_jiffies(wa_delay));
-
-	IPADBG_LOW("Exit\n");
-}
-
-static inline bool is_uc_irq(int irq_num)
-{
-	if (ipa_interrupt_to_cb[irq_num].interrupt >= IPA_UC_IRQ_0 &&
-		ipa_interrupt_to_cb[irq_num].interrupt <= IPA_UC_IRQ_3)
-		return true;
-	else
-		return false;
-}
-
-static void ipa3_process_interrupts(bool isr_context)
-{
-	u32 reg;
-	u32 bmsk;
-	u32 i = 0;
-	u32 en;
-	unsigned long flags;
-	bool uc_irq;
-
-	IPADBG_LOW("Enter isr_context=%d\n", isr_context);
-
-	spin_lock_irqsave(&suspend_wa_lock, flags);
-	en = ipahal_read_reg_n(IPA_IRQ_EN_EE_n, ipa_ee);
-	reg = ipahal_read_reg_n(IPA_IRQ_STTS_EE_n, ipa_ee);
-	while (en & reg) {
-		IPADBG_LOW("en=0x%x reg=0x%x\n", en, reg);
-		bmsk = 1;
-		for (i = 0; i < IPA_IRQ_NUM_MAX; i++) {
-			IPADBG_LOW("Check irq number %d\n", i);
-			if (en & reg & bmsk) {
-				IPADBG_LOW("Irq number %d asserted\n", i);
-				uc_irq = is_uc_irq(i);
-
-				/*
-				 * Clear uC interrupt before processing to avoid
-				 * clearing unhandled interrupts
-				 */
-				if (uc_irq)
-					ipahal_write_reg_n(IPA_IRQ_CLR_EE_n,
-							ipa_ee, bmsk);
-
-				/*
-				 * handle the interrupt with spin_lock
-				 * unlocked to avoid calling client in atomic
-				 * context. mutual exclusion still preserved
-				 * as the read/clr is done with spin_lock
-				 * locked.
-				 */
-				spin_unlock_irqrestore(&suspend_wa_lock, flags);
-				ipa3_handle_interrupt(i, isr_context);
-				spin_lock_irqsave(&suspend_wa_lock, flags);
-
-				/*
-				 * Clear non uC interrupt after processing
-				 * to avoid clearing interrupt data
-				 */
-				if (!uc_irq)
-					ipahal_write_reg_n(IPA_IRQ_CLR_EE_n,
-							ipa_ee, bmsk);
-			}
-			bmsk = bmsk << 1;
-		}
-
-		reg = ipahal_read_reg_n(IPA_IRQ_STTS_EE_n, ipa_ee);
-		/* since the suspend interrupt HW bug we must
-		 * read again the EN register, otherwise the while is endless
-		 */
-		en = ipahal_read_reg_n(IPA_IRQ_EN_EE_n, ipa_ee);
-	}
-
-	spin_unlock_irqrestore(&suspend_wa_lock, flags);
-	IPADBG_LOW("Exit\n");
-}
-
-static void ipa3_interrupt_defer(struct work_struct *work)
-{
-	IPADBG("processing interrupts in wq\n");
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-	ipa3_process_interrupts(false);
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-	IPADBG("Done\n");
-}
-
-static irqreturn_t ipa3_isr(int irq, void *ctxt)
-{
-	struct ipa_active_client_logging_info log_info;
-
-	IPA_ACTIVE_CLIENTS_PREP_SIMPLE(log_info);
-	IPADBG_LOW("Enter\n");
-	/* defer interrupt handling in case IPA is not clocked on */
-	if (ipa3_inc_client_enable_clks_no_block(&log_info)) {
-		IPADBG("defer interrupt processing\n");
-		queue_work(ipa3_ctx->power_mgmt_wq, &ipa3_interrupt_defer_work);
-		return IRQ_HANDLED;
-	}
-
-	ipa3_process_interrupts(true);
-	IPADBG_LOW("Exit\n");
-
-	ipa3_dec_client_disable_clks_no_block(&log_info);
-	return IRQ_HANDLED;
-}
-
-irq_handler_t ipa3_get_isr(void)
-{
-	return ipa3_isr;
-}
-
-/**
- * ipa3_add_interrupt_handler() - Adds handler to an interrupt type
- * @interrupt:		Interrupt type
- * @handler:		The handler to be added
- * @deferred_flag:	whether the handler processing should be deferred in
- *			a workqueue
- * @private_data:	the client's private data
- *
- * Adds handler to an interrupt type and enable the specific bit
- * in IRQ_EN register, associated interrupt in IRQ_STTS register will be enabled
- */
-int ipa3_add_interrupt_handler(enum ipa_irq_type interrupt,
-		ipa_irq_handler_t handler,
-		bool deferred_flag,
-		void *private_data)
-{
-	u32 val;
-	u32 bmsk;
-	int irq_num;
-	int client_idx, ep_idx;
-
-	IPADBG("interrupt_enum(%d)\n", interrupt);
-	if (interrupt < IPA_BAD_SNOC_ACCESS_IRQ ||
-		interrupt >= IPA_IRQ_MAX) {
-		IPAERR("invalid interrupt number %d\n", interrupt);
-		return -EINVAL;
-	}
-
-	irq_num = ipa3_irq_mapping[interrupt];
-	if (irq_num < 0 || irq_num >= IPA_IRQ_NUM_MAX) {
-		IPAERR("interrupt %d not supported\n", interrupt);
-		WARN_ON(1);
-		return -EFAULT;
-	}
-	IPADBG("ipa_interrupt_to_cb irq_num(%d)\n", irq_num);
-
-	ipa_interrupt_to_cb[irq_num].deferred_flag = deferred_flag;
-	ipa_interrupt_to_cb[irq_num].handler = handler;
-	ipa_interrupt_to_cb[irq_num].private_data = private_data;
-	ipa_interrupt_to_cb[irq_num].interrupt = interrupt;
-
-	val = ipahal_read_reg_n(IPA_IRQ_EN_EE_n, ipa_ee);
-	IPADBG("read IPA_IRQ_EN_EE_n register. reg = %d\n", val);
-	bmsk = 1 << irq_num;
-	val |= bmsk;
-	ipahal_write_reg_n(IPA_IRQ_EN_EE_n, ipa_ee, val);
-	IPADBG("wrote IPA_IRQ_EN_EE_n register. reg = %d\n", val);
-
-	/* register SUSPEND_IRQ_EN_EE_n_ADDR for L2 interrupt*/
-	if ((interrupt == IPA_TX_SUSPEND_IRQ) &&
-		(ipa3_ctx->ipa_hw_type >= IPA_HW_v3_1)) {
-		val = ~0;
-		for (client_idx = 0; client_idx < IPA_CLIENT_MAX; client_idx++)
-			if (IPA_CLIENT_IS_Q6_CONS(client_idx) ||
-				IPA_CLIENT_IS_Q6_PROD(client_idx)) {
-				ep_idx = ipa3_get_ep_mapping(client_idx);
-				IPADBG("modem ep_idx(%d) client_idx = %d\n",
-					ep_idx, client_idx);
-			if (ep_idx == -1)
-				IPADBG("Invalid IPA client\n");
-			else
-				val &= ~(1 << ep_idx);
-		}
-
-		ipahal_write_reg_n(IPA_SUSPEND_IRQ_EN_EE_n, ipa_ee, val);
-		IPADBG("wrote IPA_SUSPEND_IRQ_EN_EE_n reg = %d\n", val);
-	}
-	return 0;
-}
-
-/**
- * ipa3_remove_interrupt_handler() - Removes handler to an interrupt type
- * @interrupt:		Interrupt type
- *
- * Removes the handler and disable the specific bit in IRQ_EN register
- */
-int ipa3_remove_interrupt_handler(enum ipa_irq_type interrupt)
-{
-	u32 val;
-	u32 bmsk;
-	int irq_num;
-
-	if (interrupt < IPA_BAD_SNOC_ACCESS_IRQ ||
-		interrupt >= IPA_IRQ_MAX) {
-		IPAERR("invalid interrupt number %d\n", interrupt);
-		return -EINVAL;
-	}
-
-	irq_num = ipa3_irq_mapping[interrupt];
-	if (irq_num < 0 || irq_num >= IPA_IRQ_NUM_MAX) {
-		IPAERR("interrupt %d not supported\n", interrupt);
-		WARN_ON(1);
-		return -EFAULT;
-	}
-
-	kfree(ipa_interrupt_to_cb[irq_num].private_data);
-	ipa_interrupt_to_cb[irq_num].deferred_flag = false;
-	ipa_interrupt_to_cb[irq_num].handler = NULL;
-	ipa_interrupt_to_cb[irq_num].private_data = NULL;
-	ipa_interrupt_to_cb[irq_num].interrupt = -1;
-
-	/* clean SUSPEND_IRQ_EN_EE_n_ADDR for L2 interrupt */
-	if ((interrupt == IPA_TX_SUSPEND_IRQ) &&
-		(ipa3_ctx->ipa_hw_type >= IPA_HW_v3_1)) {
-		ipahal_write_reg_n(IPA_SUSPEND_IRQ_EN_EE_n, ipa_ee, 0);
-		IPADBG("wrote IPA_SUSPEND_IRQ_EN_EE_n reg = %d\n", 0);
-	}
-
-	val = ipahal_read_reg_n(IPA_IRQ_EN_EE_n, ipa_ee);
-	bmsk = 1 << irq_num;
-	val &= ~bmsk;
-	ipahal_write_reg_n(IPA_IRQ_EN_EE_n, ipa_ee, val);
-
-	return 0;
-}
-
-/**
- * ipa3_interrupts_init() - Initialize the IPA interrupts framework
- * @ipa_irq:	The interrupt number to allocate
- * @ee:		Execution environment
- * @ipa_dev:	The basic device structure representing the IPA driver
- *
- * - Initialize the ipa_interrupt_to_cb array
- * - Clear interrupts status
- * - Register the ipa interrupt handler - ipa3_isr
- * - Enable apps processor wakeup by IPA interrupts
- */
-int ipa3_interrupts_init(u32 ipa_irq, u32 ee, struct device *ipa_dev)
-{
-	int idx;
-	int res = 0;
-
-	ipa_ee = ee;
-	for (idx = 0; idx < IPA_IRQ_NUM_MAX; idx++) {
-		ipa_interrupt_to_cb[idx].deferred_flag = false;
-		ipa_interrupt_to_cb[idx].handler = NULL;
-		ipa_interrupt_to_cb[idx].private_data = NULL;
-		ipa_interrupt_to_cb[idx].interrupt = -1;
-	}
-
-	ipa_interrupt_wq = create_singlethread_workqueue(
-			INTERRUPT_WORKQUEUE_NAME);
-	if (!ipa_interrupt_wq) {
-		IPAERR("workqueue creation failed\n");
-		return -ENOMEM;
-	}
-
-	/*
-	 * NOTE:
-	 *
-	 *  We'll only register an isr on non-emulator (ie. real UE)
-	 *  systems.
-	 *
-	 *  On the emulator, emulator_soft_irq_isr() will be calling
-	 *  ipa3_isr, so hence, no isr registration here, and instead,
-	 *  we'll pass the address of ipa3_isr to the gsi layer where
-	 *  emulator interrupts are handled...
-	 */
-	if (ipa3_ctx->ipa3_hw_mode != IPA_HW_MODE_EMULATION) {
-		res = request_irq(ipa_irq, (irq_handler_t) ipa3_isr,
-					IRQF_TRIGGER_RISING, "ipa", ipa_dev);
-		if (res) {
-			IPAERR(
-			    "fail to register IPA IRQ handler irq=%d\n",
-			    ipa_irq);
-			destroy_workqueue(ipa_interrupt_wq);
-			ipa_interrupt_wq = NULL;
-			return -ENODEV;
-		}
-		IPADBG("IPA IRQ handler irq=%d registered\n", ipa_irq);
-
-		res = enable_irq_wake(ipa_irq);
-		if (res)
-			IPAERR("fail to enable IPA IRQ wakeup irq=%d res=%d\n",
-				   ipa_irq, res);
-		else
-			IPADBG("IPA IRQ wakeup enabled irq=%d\n", ipa_irq);
-	}
-	spin_lock_init(&suspend_wa_lock);
-	return 0;
-}
-
-/**
- * ipa3_interrupts_destroy() - Destroy the IPA interrupts framework
- * @ipa_irq:	The interrupt number to allocate
- * @ee:		Execution environment
- * @ipa_dev:	The basic device structure representing the IPA driver
- *
- * - Disable apps processor wakeup by IPA interrupts
- * - Unregister the ipa interrupt handler - ipa3_isr
- * - Destroy the interrupt workqueue
- */
-void ipa3_interrupts_destroy(u32 ipa_irq, struct device *ipa_dev)
-{
-	if (ipa3_ctx->ipa3_hw_mode != IPA_HW_MODE_EMULATION) {
-		disable_irq_wake(ipa_irq);
-		free_irq(ipa_irq, ipa_dev);
-	}
-	destroy_workqueue(ipa_interrupt_wq);
-	ipa_interrupt_wq = NULL;
-}
-
-/**
- * ipa3_suspend_active_aggr_wa() - Emulate suspend IRQ
- * @clnt_hndl:		suspended client handle, IRQ is emulated for this pipe
- *
- *  Emulate suspend IRQ to unsuspend client which was suspended with an open
- *  aggregation frame in order to bypass HW bug of IRQ not generated when
- *  endpoint is suspended during an open aggregation.
- */
-void ipa3_suspend_active_aggr_wa(u32 clnt_hdl)
-{
-	struct ipa3_interrupt_info interrupt_info;
-	struct ipa3_interrupt_work_wrap *work_data;
-	struct ipa_tx_suspend_irq_data *suspend_interrupt_data;
-	int irq_num;
-	int aggr_active_bitmap = ipahal_read_reg(IPA_STATE_AGGR_ACTIVE);
-
-	if (aggr_active_bitmap & (1 << clnt_hdl)) {
-		/* force close aggregation */
-		ipahal_write_reg(IPA_AGGR_FORCE_CLOSE, (1 << clnt_hdl));
-
-		/* simulate suspend IRQ */
-		irq_num = ipa3_irq_mapping[IPA_TX_SUSPEND_IRQ];
-		interrupt_info = ipa_interrupt_to_cb[irq_num];
-		if (interrupt_info.handler == NULL) {
-			IPAERR("no CB function for IPA_TX_SUSPEND_IRQ\n");
-			return;
-		}
-		suspend_interrupt_data = kzalloc(
-				sizeof(*suspend_interrupt_data),
-				GFP_ATOMIC);
-		if (!suspend_interrupt_data) {
-			IPAERR("failed allocating suspend_interrupt_data\n");
-			return;
-		}
-		suspend_interrupt_data->endpoints = 1 << clnt_hdl;
-
-		work_data = kzalloc(sizeof(struct ipa3_interrupt_work_wrap),
-				GFP_ATOMIC);
-		if (!work_data) {
-			IPAERR("failed allocating ipa3_interrupt_work_wrap\n");
-			goto fail_alloc_work;
-		}
-		INIT_WORK(&work_data->interrupt_work,
-				ipa3_deferred_interrupt_work);
-		work_data->handler = interrupt_info.handler;
-		work_data->interrupt = IPA_TX_SUSPEND_IRQ;
-		work_data->private_data = interrupt_info.private_data;
-		work_data->interrupt_data = (void *)suspend_interrupt_data;
-		queue_work(ipa_interrupt_wq, &work_data->interrupt_work);
-		return;
-fail_alloc_work:
-		kfree(suspend_interrupt_data);
-	}
-}

ipa/ipa_v3/ipa_intf.c

@@ -1,810 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2013-2019, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/fs.h>
-#include <linux/sched.h>
-#include <linux/sched/signal.h>
-#include "ipa_i.h"
-#include <linux/msm_ipa.h>
-
-struct ipa3_intf {
-	char name[IPA_RESOURCE_NAME_MAX];
-	struct list_head link;
-	u32 num_tx_props;
-	u32 num_rx_props;
-	u32 num_ext_props;
-	struct ipa_ioc_tx_intf_prop *tx;
-	struct ipa_ioc_rx_intf_prop *rx;
-	struct ipa_ioc_ext_intf_prop *ext;
-	enum ipa_client_type excp_pipe;
-};
-
-struct ipa3_push_msg {
-	struct ipa_msg_meta meta;
-	ipa_msg_free_fn callback;
-	void *buff;
-	struct list_head link;
-};
-
-struct ipa3_pull_msg {
-	struct ipa_msg_meta meta;
-	ipa_msg_pull_fn callback;
-	struct list_head link;
-};
-
-/**
- * ipa3_register_intf() - register "logical" interface
- * @name: [in] interface name
- * @tx:	[in] TX properties of the interface
- * @rx:	[in] RX properties of the interface
- *
- * Register an interface and its tx and rx properties, this allows
- * configuration of rules from user-space
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_register_intf(const char *name, const struct ipa_tx_intf *tx,
-		       const struct ipa_rx_intf *rx)
-{
-	return ipa3_register_intf_ext(name, tx, rx, NULL);
-}
-
-/**
- * ipa3_register_intf_ext() - register "logical" interface which has only
- * extended properties
- * @name: [in] interface name
- * @tx:	[in] TX properties of the interface
- * @rx:	[in] RX properties of the interface
- * @ext: [in] EXT properties of the interface
- *
- * Register an interface and its tx, rx and ext properties, this allows
- * configuration of rules from user-space
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_register_intf_ext(const char *name, const struct ipa_tx_intf *tx,
-		       const struct ipa_rx_intf *rx,
-		       const struct ipa_ext_intf *ext)
-{
-	struct ipa3_intf *intf;
-	u32 len;
-
-	if (name == NULL || (tx == NULL && rx == NULL && ext == NULL)) {
-		IPAERR_RL("invalid params name=%pK tx=%pK rx=%pK ext=%pK\n",
-				name, tx, rx, ext);
-		return -EINVAL;
-	}
-
-	if (tx && tx->num_props > IPA_NUM_PROPS_MAX) {
-		IPAERR_RL("invalid tx num_props=%d max=%d\n", tx->num_props,
-				IPA_NUM_PROPS_MAX);
-		return -EINVAL;
-	}
-
-	if (rx && rx->num_props > IPA_NUM_PROPS_MAX) {
-		IPAERR_RL("invalid rx num_props=%d max=%d\n", rx->num_props,
-				IPA_NUM_PROPS_MAX);
-		return -EINVAL;
-	}
-
-	if (ext && ext->num_props > IPA_NUM_PROPS_MAX) {
-		IPAERR_RL("invalid ext num_props=%d max=%d\n", ext->num_props,
-				IPA_NUM_PROPS_MAX);
-		return -EINVAL;
-	}
-
-	len = sizeof(struct ipa3_intf);
-	intf = kzalloc(len, GFP_KERNEL);
-	if (intf == NULL)
-		return -ENOMEM;
-
-	strlcpy(intf->name, name, IPA_RESOURCE_NAME_MAX);
-
-	if (tx) {
-		intf->num_tx_props = tx->num_props;
-		len = tx->num_props * sizeof(struct ipa_ioc_tx_intf_prop);
-		intf->tx = kmemdup(tx->prop, len, GFP_KERNEL);
-		if (intf->tx == NULL) {
-			kfree(intf);
-			return -ENOMEM;
-		}
-	}
-
-	if (rx) {
-		intf->num_rx_props = rx->num_props;
-		len = rx->num_props * sizeof(struct ipa_ioc_rx_intf_prop);
-		intf->rx = kmemdup(rx->prop, len, GFP_KERNEL);
-		if (intf->rx == NULL) {
-			kfree(intf->tx);
-			kfree(intf);
-			return -ENOMEM;
-		}
-		memcpy(intf->rx, rx->prop, len);
-	}
-
-	if (ext) {
-		intf->num_ext_props = ext->num_props;
-		len = ext->num_props * sizeof(struct ipa_ioc_ext_intf_prop);
-		intf->ext = kmemdup(ext->prop, len, GFP_KERNEL);
-		if (intf->ext == NULL) {
-			kfree(intf->rx);
-			kfree(intf->tx);
-			kfree(intf);
-			return -ENOMEM;
-		}
-		memcpy(intf->ext, ext->prop, len);
-	}
-
-	if (ext && ext->excp_pipe_valid)
-		intf->excp_pipe = ext->excp_pipe;
-	else
-		intf->excp_pipe = IPA_CLIENT_APPS_LAN_CONS;
-
-	mutex_lock(&ipa3_ctx->lock);
-	list_add_tail(&intf->link, &ipa3_ctx->intf_list);
-	mutex_unlock(&ipa3_ctx->lock);
-
-	return 0;
-}
-
-/**
- * ipa3_deregister_intf() - de-register previously registered logical interface
- * @name: [in] interface name
- *
- * De-register a previously registered interface
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_deregister_intf(const char *name)
-{
-	struct ipa3_intf *entry;
-	struct ipa3_intf *next;
-	int result = -EINVAL;
-
-	if ((name == NULL) ||
-	    (strnlen(name, IPA_RESOURCE_NAME_MAX) == IPA_RESOURCE_NAME_MAX)) {
-		IPAERR_RL("invalid param name=%s\n", name);
-		return result;
-	}
-
-	mutex_lock(&ipa3_ctx->lock);
-	list_for_each_entry_safe(entry, next, &ipa3_ctx->intf_list, link) {
-		if (!strcmp(entry->name, name)) {
-			list_del(&entry->link);
-			kfree(entry->ext);
-			kfree(entry->rx);
-			kfree(entry->tx);
-			kfree(entry);
-			result = 0;
-			break;
-		}
-	}
-	mutex_unlock(&ipa3_ctx->lock);
-
-	return result;
-}
-
-/**
- * ipa3_query_intf() - query logical interface properties
- * @lookup:	[inout] interface name and number of properties
- *
- * Obtain the handle and number of tx and rx properties for the named
- * interface, used as part of querying the tx and rx properties for
- * configuration of various rules from user-space
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_query_intf(struct ipa_ioc_query_intf *lookup)
-{
-	struct ipa3_intf *entry;
-	int result = -EINVAL;
-
-	if (lookup == NULL) {
-		IPAERR_RL("invalid param lookup=%pK\n", lookup);
-		return result;
-	}
-
-	lookup->name[IPA_RESOURCE_NAME_MAX-1] = '\0';
-	if (strnlen(lookup->name, IPA_RESOURCE_NAME_MAX) ==
-			IPA_RESOURCE_NAME_MAX) {
-		IPAERR_RL("Interface name too long. (%s)\n", lookup->name);
-		return result;
-	}
-
-	mutex_lock(&ipa3_ctx->lock);
-	list_for_each_entry(entry, &ipa3_ctx->intf_list, link) {
-		if (!strcmp(entry->name, lookup->name)) {
-			lookup->num_tx_props = entry->num_tx_props;
-			lookup->num_rx_props = entry->num_rx_props;
-			lookup->num_ext_props = entry->num_ext_props;
-			lookup->excp_pipe = entry->excp_pipe;
-			result = 0;
-			break;
-		}
-	}
-	mutex_unlock(&ipa3_ctx->lock);
-
-	return result;
-}
-
-/**
- * ipa3_query_intf_tx_props() - qeury TX props of an interface
- * @tx:  [inout] interface tx attributes
- *
- * Obtain the tx properties for the specified interface
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_query_intf_tx_props(struct ipa_ioc_query_intf_tx_props *tx)
-{
-	struct ipa3_intf *entry;
-	int result = -EINVAL;
-
-	if (tx == NULL) {
-		IPAERR_RL("null args: tx\n");
-		return result;
-	}
-
-	tx->name[IPA_RESOURCE_NAME_MAX-1] = '\0';
-	if (strnlen(tx->name, IPA_RESOURCE_NAME_MAX) == IPA_RESOURCE_NAME_MAX) {
-		IPAERR_RL("Interface name too long. (%s)\n", tx->name);
-		return result;
-	}
-
-	mutex_lock(&ipa3_ctx->lock);
-	list_for_each_entry(entry, &ipa3_ctx->intf_list, link) {
-		if (!strcmp(entry->name, tx->name)) {
-			/* add the entry check */
-			if (entry->num_tx_props != tx->num_tx_props) {
-				IPAERR("invalid entry number(%u %u)\n",
-					entry->num_tx_props,
-						tx->num_tx_props);
-				mutex_unlock(&ipa3_ctx->lock);
-				return result;
-			}
-			memcpy(tx->tx, entry->tx, entry->num_tx_props *
-			       sizeof(struct ipa_ioc_tx_intf_prop));
-			result = 0;
-			break;
-		}
-	}
-	mutex_unlock(&ipa3_ctx->lock);
-
-	return result;
-}
-
-/**
- * ipa3_query_intf_rx_props() - qeury RX props of an interface
- * @rx:  [inout] interface rx attributes
- *
- * Obtain the rx properties for the specified interface
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_query_intf_rx_props(struct ipa_ioc_query_intf_rx_props *rx)
-{
-	struct ipa3_intf *entry;
-	int result = -EINVAL;
-
-	if (rx == NULL) {
-		IPAERR_RL("null args: rx\n");
-		return result;
-	}
-
-	rx->name[IPA_RESOURCE_NAME_MAX-1] = '\0';
-	if (strnlen(rx->name, IPA_RESOURCE_NAME_MAX) == IPA_RESOURCE_NAME_MAX) {
-		IPAERR_RL("Interface name too long. (%s)\n", rx->name);
-		return result;
-	}
-
-	mutex_lock(&ipa3_ctx->lock);
-	list_for_each_entry(entry, &ipa3_ctx->intf_list, link) {
-		if (!strcmp(entry->name, rx->name)) {
-			/* add the entry check */
-			if (entry->num_rx_props != rx->num_rx_props) {
-				IPAERR("invalid entry number(%u %u)\n",
-					entry->num_rx_props,
-						rx->num_rx_props);
-				mutex_unlock(&ipa3_ctx->lock);
-				return result;
-			}
-			memcpy(rx->rx, entry->rx, entry->num_rx_props *
-					sizeof(struct ipa_ioc_rx_intf_prop));
-			result = 0;
-			break;
-		}
-	}
-	mutex_unlock(&ipa3_ctx->lock);
-
-	return result;
-}
-
-/**
- * ipa3_query_intf_ext_props() - qeury EXT props of an interface
- * @ext:  [inout] interface ext attributes
- *
- * Obtain the ext properties for the specified interface
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_query_intf_ext_props(struct ipa_ioc_query_intf_ext_props *ext)
-{
-	struct ipa3_intf *entry;
-	int result = -EINVAL;
-
-	if (ext == NULL) {
-		IPAERR_RL("invalid param ext=%pK\n", ext);
-		return result;
-	}
-
-	mutex_lock(&ipa3_ctx->lock);
-	list_for_each_entry(entry, &ipa3_ctx->intf_list, link) {
-		if (!strcmp(entry->name, ext->name)) {
-			/* add the entry check */
-			if (entry->num_ext_props != ext->num_ext_props) {
-				IPAERR("invalid entry number(%u %u)\n",
-					entry->num_ext_props,
-						ext->num_ext_props);
-				mutex_unlock(&ipa3_ctx->lock);
-				return result;
-			}
-			memcpy(ext->ext, entry->ext, entry->num_ext_props *
-					sizeof(struct ipa_ioc_ext_intf_prop));
-			result = 0;
-			break;
-		}
-	}
-	mutex_unlock(&ipa3_ctx->lock);
-	return result;
-}
-
-static void ipa3_send_msg_free(void *buff, u32 len, u32 type)
-{
-	kfree(buff);
-}
-
-static int wlan_msg_process(struct ipa_msg_meta *meta, void *buff)
-{
-	struct ipa3_push_msg *msg_dup;
-	struct ipa_wlan_msg_ex *event_ex_cur_con = NULL;
-	struct ipa_wlan_msg_ex *event_ex_list = NULL;
-	struct ipa_wlan_msg *event_ex_cur_discon = NULL;
-	void *data_dup = NULL;
-	struct ipa3_push_msg *entry;
-	struct ipa3_push_msg *next;
-	int cnt = 0, total = 0, max = 0;
-	uint8_t mac[IPA_MAC_ADDR_SIZE];
-	uint8_t mac2[IPA_MAC_ADDR_SIZE];
-
-	if (!buff)
-		return -EINVAL;
-	if (meta->msg_type == WLAN_CLIENT_CONNECT_EX) {
-		/* debug print */
-		event_ex_cur_con = buff;
-		for (cnt = 0; cnt < event_ex_cur_con->num_of_attribs; cnt++) {
-			if (event_ex_cur_con->attribs[cnt].attrib_type ==
-				WLAN_HDR_ATTRIB_MAC_ADDR) {
-				IPADBG("%02x:%02x:%02x:%02x:%02x:%02x,(%d)\n",
-				event_ex_cur_con->attribs[cnt].u.mac_addr[0],
-				event_ex_cur_con->attribs[cnt].u.mac_addr[1],
-				event_ex_cur_con->attribs[cnt].u.mac_addr[2],
-				event_ex_cur_con->attribs[cnt].u.mac_addr[3],
-				event_ex_cur_con->attribs[cnt].u.mac_addr[4],
-				event_ex_cur_con->attribs[cnt].u.mac_addr[5],
-				meta->msg_type);
-			}
-		}
-
-		mutex_lock(&ipa3_ctx->msg_wlan_client_lock);
-		msg_dup = kzalloc(sizeof(*msg_dup), GFP_KERNEL);
-		if (msg_dup == NULL) {
-			mutex_unlock(&ipa3_ctx->msg_wlan_client_lock);
-			return -ENOMEM;
-		}
-		msg_dup->meta = *meta;
-		if (meta->msg_len > 0 && buff) {
-			data_dup = kmemdup(buff, meta->msg_len, GFP_KERNEL);
-			if (data_dup == NULL) {
-				kfree(msg_dup);
-				mutex_unlock(&ipa3_ctx->msg_wlan_client_lock);
-				return -ENOMEM;
-			}
-			memcpy(data_dup, buff, meta->msg_len);
-			msg_dup->buff = data_dup;
-			msg_dup->callback = ipa3_send_msg_free;
-		} else {
-			IPAERR("msg_len %d\n", meta->msg_len);
-			kfree(msg_dup);
-			mutex_unlock(&ipa3_ctx->msg_wlan_client_lock);
-			return -ENOMEM;
-		}
-		list_add_tail(&msg_dup->link, &ipa3_ctx->msg_wlan_client_list);
-		mutex_unlock(&ipa3_ctx->msg_wlan_client_lock);
-	}
-
-	/* remove the cache */
-	if (meta->msg_type == WLAN_CLIENT_DISCONNECT) {
-		/* debug print */
-		event_ex_cur_discon = buff;
-		IPADBG("Mac %pM, msg %d\n",
-		event_ex_cur_discon->mac_addr,
-		meta->msg_type);
-		memcpy(mac2,
-			event_ex_cur_discon->mac_addr,
-			sizeof(mac2));
-
-		mutex_lock(&ipa3_ctx->msg_wlan_client_lock);
-		list_for_each_entry_safe(entry, next,
-				&ipa3_ctx->msg_wlan_client_list,
-				link) {
-			event_ex_list = entry->buff;
-			max = event_ex_list->num_of_attribs;
-			for (cnt = 0; cnt < max; cnt++) {
-				memcpy(mac,
-					event_ex_list->attribs[cnt].u.mac_addr,
-					sizeof(mac));
-				if (event_ex_list->attribs[cnt].attrib_type ==
-					WLAN_HDR_ATTRIB_MAC_ADDR) {
-					pr_debug("%pM\n", mac);
-
-					/* compare to delete one*/
-					if (memcmp(mac2, mac,
-						sizeof(mac)) == 0) {
-						IPADBG("clean %d\n", total);
-						list_del(&entry->link);
-						kfree(entry);
-						break;
-					}
-				}
-			}
-			total++;
-		}
-		mutex_unlock(&ipa3_ctx->msg_wlan_client_lock);
-	}
-	return 0;
-}
-
-/**
- * ipa3_send_msg() - Send "message" from kernel client to IPA driver
- * @meta: [in] message meta-data
- * @buff: [in] the payload for message
- * @callback: [in] free callback
- *
- * Client supplies the message meta-data and payload which IPA driver buffers
- * till read by user-space. After read from user space IPA driver invokes the
- * callback supplied to free the message payload. Client must not touch/free
- * the message payload after calling this API.
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_send_msg(struct ipa_msg_meta *meta, void *buff,
-		  ipa_msg_free_fn callback)
-{
-	struct ipa3_push_msg *msg;
-	void *data = NULL;
-
-	if (meta == NULL || (buff == NULL && callback != NULL) ||
-	    (buff != NULL && callback == NULL)) {
-		IPAERR_RL("invalid param meta=%pK buff=%pK, callback=%pK\n",
-		       meta, buff, callback);
-		return -EINVAL;
-	}
-
-	if (meta->msg_type >= IPA_EVENT_MAX_NUM) {
-		IPAERR_RL("unsupported message type %d\n", meta->msg_type);
-		return -EINVAL;
-	}
-
-	msg = kzalloc(sizeof(struct ipa3_push_msg), GFP_KERNEL);
-	if (msg == NULL)
-		return -ENOMEM;
-
-	msg->meta = *meta;
-	if (meta->msg_len > 0 && buff) {
-		data = kmemdup(buff, meta->msg_len, GFP_KERNEL);
-		if (data == NULL) {
-			kfree(msg);
-			return -ENOMEM;
-		}
-		msg->buff = data;
-		msg->callback = ipa3_send_msg_free;
-	}
-
-	mutex_lock(&ipa3_ctx->msg_lock);
-	list_add_tail(&msg->link, &ipa3_ctx->msg_list);
-	/* support for softap client event cache */
-	if (wlan_msg_process(meta, buff))
-		IPAERR_RL("wlan_msg_process failed\n");
-
-	/* unlock only after process */
-	mutex_unlock(&ipa3_ctx->msg_lock);
-	IPA_STATS_INC_CNT(ipa3_ctx->stats.msg_w[meta->msg_type]);
-
-	wake_up(&ipa3_ctx->msg_waitq);
-	if (buff)
-		callback(buff, meta->msg_len, meta->msg_type);
-
-	return 0;
-}
-
-/**
- * ipa3_resend_wlan_msg() - Resend cached "message" to IPACM
- *
- * resend wlan client connect events to user-space
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_resend_wlan_msg(void)
-{
-	struct ipa_wlan_msg_ex *event_ex_list = NULL;
-	struct ipa3_push_msg *entry;
-	struct ipa3_push_msg *next;
-	int cnt = 0, total = 0;
-	struct ipa3_push_msg *msg;
-	void *data = NULL;
-
-	IPADBG("\n");
-
-	mutex_lock(&ipa3_ctx->msg_wlan_client_lock);
-	list_for_each_entry_safe(entry, next, &ipa3_ctx->msg_wlan_client_list,
-			link) {
-
-		event_ex_list = entry->buff;
-		for (cnt = 0; cnt < event_ex_list->num_of_attribs; cnt++) {
-			if (event_ex_list->attribs[cnt].attrib_type ==
-				WLAN_HDR_ATTRIB_MAC_ADDR) {
-				IPADBG("%d-Mac %pM\n", total,
-				event_ex_list->attribs[cnt].u.mac_addr);
-			}
-		}
-
-		msg = kzalloc(sizeof(*msg), GFP_KERNEL);
-		if (msg == NULL) {
-			mutex_unlock(&ipa3_ctx->msg_wlan_client_lock);
-			return -ENOMEM;
-		}
-		msg->meta = entry->meta;
-		data = kmemdup(entry->buff, entry->meta.msg_len, GFP_KERNEL);
-		if (data == NULL) {
-			kfree(msg);
-			mutex_unlock(&ipa3_ctx->msg_wlan_client_lock);
-			return -ENOMEM;
-		}
-		msg->buff = data;
-		msg->callback = ipa3_send_msg_free;
-		mutex_lock(&ipa3_ctx->msg_lock);
-		list_add_tail(&msg->link, &ipa3_ctx->msg_list);
-		mutex_unlock(&ipa3_ctx->msg_lock);
-		wake_up(&ipa3_ctx->msg_waitq);
-
-		total++;
-	}
-	mutex_unlock(&ipa3_ctx->msg_wlan_client_lock);
-	return 0;
-}
-
-/**
- * ipa3_register_pull_msg() - register pull message type
- * @meta: [in] message meta-data
- * @callback: [in] pull callback
- *
- * Register message callback by kernel client with IPA driver for IPA driver to
- * pull message on-demand.
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_register_pull_msg(struct ipa_msg_meta *meta, ipa_msg_pull_fn callback)
-{
-	struct ipa3_pull_msg *msg;
-
-	if (meta == NULL || callback == NULL) {
-		IPAERR_RL("invalid param meta=%pK callback=%pK\n",
-				meta, callback);
-		return -EINVAL;
-	}
-
-	msg = kzalloc(sizeof(struct ipa3_pull_msg), GFP_KERNEL);
-	if (msg == NULL)
-		return -ENOMEM;
-
-	msg->meta = *meta;
-	msg->callback = callback;
-
-	mutex_lock(&ipa3_ctx->msg_lock);
-	list_add_tail(&msg->link, &ipa3_ctx->pull_msg_list);
-	mutex_unlock(&ipa3_ctx->msg_lock);
-
-	return 0;
-}
-
-/**
- * ipa3_deregister_pull_msg() - De-register pull message type
- * @meta: [in] message meta-data
- *
- * De-register "message" by kernel client from IPA driver
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_deregister_pull_msg(struct ipa_msg_meta *meta)
-{
-	struct ipa3_pull_msg *entry;
-	struct ipa3_pull_msg *next;
-	int result = -EINVAL;
-
-	if (meta == NULL) {
-		IPAERR_RL("null arg: meta\n");
-		return result;
-	}
-
-	mutex_lock(&ipa3_ctx->msg_lock);
-	list_for_each_entry_safe(entry, next, &ipa3_ctx->pull_msg_list, link) {
-		if (entry->meta.msg_len == meta->msg_len &&
-		    entry->meta.msg_type == meta->msg_type) {
-			list_del(&entry->link);
-			kfree(entry);
-			result = 0;
-			break;
-		}
-	}
-	mutex_unlock(&ipa3_ctx->msg_lock);
-	return result;
-}
-
-/**
- * ipa3_read() - read message from IPA device
- * @filp:	[in] file pointer
- * @buf:	[out] buffer to read into
- * @count:	[in] size of above buffer
- * @f_pos:	[inout] file position
- *
- * Uer-space should continually read from /dev/ipa, read wll block when there
- * are no messages to read. Upon return, user-space should read the ipa_msg_meta
- * from the start of the buffer to know what type of message was read and its
- * length in the remainder of the buffer. Buffer supplied must be big enough to
- * hold the message meta-data and the largest defined message type
- *
- * Returns:	how many bytes copied to buffer
- *
- * Note:	Should not be called from atomic context
- */
-ssize_t ipa3_read(struct file *filp, char __user *buf, size_t count,
-		  loff_t *f_pos)
-{
-	char __user *start;
-	struct ipa3_push_msg *msg = NULL;
-	int ret;
-	DEFINE_WAIT_FUNC(wait, woken_wake_function);
-	int locked;
-
-	start = buf;
-
-	add_wait_queue(&ipa3_ctx->msg_waitq, &wait);
-	while (1) {
-		mutex_lock(&ipa3_ctx->msg_lock);
-		locked = 1;
-
-		if (!list_empty(&ipa3_ctx->msg_list)) {
-			msg = list_first_entry(&ipa3_ctx->msg_list,
-					struct ipa3_push_msg, link);
-			list_del(&msg->link);
-		}
-
-		IPADBG_LOW("msg=%pK\n", msg);
-
-		if (msg) {
-			locked = 0;
-			mutex_unlock(&ipa3_ctx->msg_lock);
-			if (copy_to_user(buf, &msg->meta,
-					  sizeof(struct ipa_msg_meta))) {
-				ret = -EFAULT;
-				kfree(msg);
-				msg = NULL;
-				break;
-			}
-			buf += sizeof(struct ipa_msg_meta);
-			count -= sizeof(struct ipa_msg_meta);
-			if (msg->buff) {
-				if (copy_to_user(buf, msg->buff,
-						  msg->meta.msg_len)) {
-					ret = -EFAULT;
-					kfree(msg);
-					msg = NULL;
-					break;
-				}
-				buf += msg->meta.msg_len;
-				count -= msg->meta.msg_len;
-				msg->callback(msg->buff, msg->meta.msg_len,
-					       msg->meta.msg_type);
-			}
-			IPA_STATS_INC_CNT(
-				ipa3_ctx->stats.msg_r[msg->meta.msg_type]);
-			kfree(msg);
-			msg = NULL;
-		}
-
-		ret = -EAGAIN;
-		if (filp->f_flags & O_NONBLOCK)
-			break;
-
-		ret = -EINTR;
-		if (signal_pending(current))
-			break;
-
-		if (start != buf)
-			break;
-
-		locked = 0;
-		mutex_unlock(&ipa3_ctx->msg_lock);
-		wait_woken(&wait, TASK_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
-	}
-
-	remove_wait_queue(&ipa3_ctx->msg_waitq, &wait);
-	if (start != buf && ret != -EFAULT)
-		ret = buf - start;
-
-	if (locked)
-		mutex_unlock(&ipa3_ctx->msg_lock);
-
-	return ret;
-}
-
-/**
- * ipa3_pull_msg() - pull the specified message from client
- * @meta: [in] message meta-data
- * @buf:  [out] buffer to read into
- * @count: [in] size of above buffer
- *
- * Populate the supplied buffer with the pull message which is fetched
- * from client, the message must have previously been registered with
- * the IPA driver
- *
- * Returns:	how many bytes copied to buffer
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_pull_msg(struct ipa_msg_meta *meta, char *buff, size_t count)
-{
-	struct ipa3_pull_msg *entry;
-	int result = -EINVAL;
-
-	if (meta == NULL || buff == NULL || !count) {
-		IPAERR_RL("invalid param name=%pK buff=%pK count=%zu\n",
-				meta, buff, count);
-		return result;
-	}
-
-	mutex_lock(&ipa3_ctx->msg_lock);
-	list_for_each_entry(entry, &ipa3_ctx->pull_msg_list, link) {
-		if (entry->meta.msg_len == meta->msg_len &&
-		    entry->meta.msg_type == meta->msg_type) {
-			result = entry->callback(buff, count, meta->msg_type);
-			break;
-		}
-	}
-	mutex_unlock(&ipa3_ctx->msg_lock);
-	return result;
-}

ipa/ipa_v3/ipa_mhi.c

@@ -1,750 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2015-2019, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/debugfs.h>
-#include <linux/export.h>
-#include <linux/delay.h>
-#include <linux/module.h>
-#include <linux/mutex.h>
-#include <linux/ipa.h>
-#include <linux/msm_gsi.h>
-#include <linux/ipa_mhi.h>
-#include "../ipa_common_i.h"
-#include "ipa_i.h"
-#include "ipa_qmi_service.h"
-
-#define IPA_MHI_DRV_NAME "ipa_mhi"
-
-
-#define IPA_MHI_DBG(fmt, args...) \
-	do { \
-		pr_debug(IPA_MHI_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-			IPA_MHI_DRV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			IPA_MHI_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-#define IPA_MHI_DBG_LOW(fmt, args...) \
-	do { \
-		pr_debug(IPA_MHI_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			IPA_MHI_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-
-#define IPA_MHI_ERR(fmt, args...) \
-	do { \
-		pr_err(IPA_MHI_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-				IPA_MHI_DRV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-				IPA_MHI_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-
-#define IPA_MHI_FUNC_ENTRY() \
-	IPA_MHI_DBG("ENTRY\n")
-#define IPA_MHI_FUNC_EXIT() \
-	IPA_MHI_DBG("EXIT\n")
-
-#define IPA_MHI_MAX_UL_CHANNELS 1
-#define IPA_MHI_MAX_DL_CHANNELS 2
-
-/* bit #40 in address should be asserted for MHI transfers over pcie */
-#define IPA_MHI_HOST_ADDR_COND(addr) \
-		((params->assert_bit40)?(IPA_MHI_HOST_ADDR(addr)):(addr))
-
-enum ipa3_mhi_polling_mode {
-	IPA_MHI_POLLING_MODE_DB_MODE,
-	IPA_MHI_POLLING_MODE_POLL_MODE,
-};
-
-bool ipa3_mhi_stop_gsi_channel(enum ipa_client_type client)
-{
-	int res;
-	int ipa_ep_idx;
-	struct ipa3_ep_context *ep;
-
-	IPA_MHI_FUNC_ENTRY();
-	ipa_ep_idx = ipa3_get_ep_mapping(client);
-	if (ipa_ep_idx == -1) {
-		IPA_MHI_ERR("Invalid client.\n");
-		return -EINVAL;
-	}
-
-	ep = &ipa3_ctx->ep[ipa_ep_idx];
-	IPA_MHI_DBG_LOW("Stopping GSI channel %ld\n", ep->gsi_chan_hdl);
-	res = gsi_stop_channel(ep->gsi_chan_hdl);
-	if (res != 0 &&
-		res != -GSI_STATUS_AGAIN &&
-		res != -GSI_STATUS_TIMED_OUT) {
-		IPA_MHI_ERR("GSI stop channel failed %d\n",
-			res);
-		WARN_ON(1);
-		return false;
-	}
-
-	if (res == 0) {
-		IPA_MHI_DBG_LOW("GSI channel %ld STOP\n",
-			ep->gsi_chan_hdl);
-		return true;
-	}
-
-	return false;
-}
-
-static int ipa3_mhi_reset_gsi_channel(enum ipa_client_type client)
-{
-	int res;
-	int clnt_hdl;
-
-	IPA_MHI_FUNC_ENTRY();
-
-	clnt_hdl = ipa3_get_ep_mapping(client);
-	if (clnt_hdl < 0)
-		return -EFAULT;
-
-	res = ipa3_reset_gsi_channel(clnt_hdl);
-	if (res) {
-		IPA_MHI_ERR("ipa3_reset_gsi_channel failed %d\n", res);
-		return -EFAULT;
-	}
-
-	IPA_MHI_FUNC_EXIT();
-	return 0;
-}
-
-int ipa3_mhi_reset_channel_internal(enum ipa_client_type client)
-{
-	int res;
-
-	IPA_MHI_FUNC_ENTRY();
-
-	res = ipa3_mhi_reset_gsi_channel(client);
-	if (res) {
-		IPAERR("ipa3_mhi_reset_gsi_channel failed\n");
-		ipa_assert();
-		return res;
-	}
-
-	res = ipa3_disable_data_path(ipa3_get_ep_mapping(client));
-	if (res) {
-		IPA_MHI_ERR("ipa3_disable_data_path failed %d\n", res);
-		return res;
-	}
-	IPA_MHI_FUNC_EXIT();
-
-	return 0;
-}
-
-int ipa3_mhi_start_channel_internal(enum ipa_client_type client)
-{
-	int res;
-	int ipa_ep_idx;
-
-	IPA_MHI_FUNC_ENTRY();
-
-	ipa_ep_idx = ipa3_get_ep_mapping(client);
-	if (ipa_ep_idx < 0) {
-		IPA_MHI_ERR("Invalid client %d\n", client);
-		return -EINVAL;
-	}
-	res = ipa3_enable_data_path(ipa_ep_idx);
-	if (res) {
-		IPA_MHI_ERR("ipa3_enable_data_path failed %d\n", res);
-		return res;
-	}
-	IPA_MHI_FUNC_EXIT();
-
-	return 0;
-}
-
-static int ipa3_mhi_get_ch_poll_cfg(enum ipa_client_type client,
-		struct ipa_mhi_ch_ctx *ch_ctx_host, int ring_size)
-{
-	switch (ch_ctx_host->pollcfg) {
-	case 0:
-	/*set default polling configuration according to MHI spec*/
-		if (IPA_CLIENT_IS_PROD(client))
-			return 7;
-		else
-			return (ring_size/2)/8;
-		break;
-	default:
-		return ch_ctx_host->pollcfg;
-	}
-}
-
-static int ipa_mhi_start_gsi_channel(enum ipa_client_type client,
-	int ipa_ep_idx, struct start_gsi_channel *params)
-{
-	int res = 0;
-	struct gsi_evt_ring_props ev_props;
-	struct ipa_mhi_msi_info *msi;
-	struct gsi_chan_props ch_props;
-	union __packed gsi_channel_scratch ch_scratch;
-	struct ipa3_ep_context *ep;
-	const struct ipa_gsi_ep_config *ep_cfg;
-	struct ipa_ep_cfg_ctrl ep_cfg_ctrl;
-	bool burst_mode_enabled = false;
-
-	IPA_MHI_FUNC_ENTRY();
-
-	ep = &ipa3_ctx->ep[ipa_ep_idx];
-
-	msi = params->msi;
-	ep_cfg = ipa3_get_gsi_ep_info(client);
-	if (!ep_cfg) {
-		IPA_MHI_ERR("Wrong parameter, ep_cfg is NULL\n");
-		return -EPERM;
-	}
-
-	/* allocate event ring only for the first time pipe is connected */
-	if (params->state == IPA_HW_MHI_CHANNEL_STATE_INVALID) {
-		memset(&ev_props, 0, sizeof(ev_props));
-		ev_props.intf = GSI_EVT_CHTYPE_MHI_EV;
-		ev_props.intr = GSI_INTR_MSI;
-		ev_props.re_size = GSI_EVT_RING_RE_SIZE_16B;
-		ev_props.ring_len = params->ev_ctx_host->rlen;
-		ev_props.ring_base_addr = IPA_MHI_HOST_ADDR_COND(
-				params->ev_ctx_host->rbase);
-		ev_props.int_modt = params->ev_ctx_host->intmodt *
-				IPA_SLEEP_CLK_RATE_KHZ;
-		ev_props.int_modc = params->ev_ctx_host->intmodc;
-		ev_props.intvec = ((msi->data & ~msi->mask) |
-				(params->ev_ctx_host->msivec & msi->mask));
-		ev_props.msi_addr = IPA_MHI_HOST_ADDR_COND(
-				(((u64)msi->addr_hi << 32) | msi->addr_low));
-		ev_props.rp_update_addr = IPA_MHI_HOST_ADDR_COND(
-				params->event_context_addr +
-				offsetof(struct ipa_mhi_ev_ctx, rp));
-		ev_props.exclusive = true;
-		ev_props.err_cb = params->ev_err_cb;
-		ev_props.user_data = params->channel;
-		ev_props.evchid_valid = true;
-		ev_props.evchid = params->evchid;
-		IPA_MHI_DBG("allocating event ring ep:%u evchid:%u\n",
-			ipa_ep_idx, ev_props.evchid);
-		res = gsi_alloc_evt_ring(&ev_props, ipa3_ctx->gsi_dev_hdl,
-			&ep->gsi_evt_ring_hdl);
-		if (res) {
-			IPA_MHI_ERR("gsi_alloc_evt_ring failed %d\n", res);
-			goto fail_alloc_evt;
-		}
-		IPA_MHI_DBG("client %d, caching event ring hdl %lu\n",
-				client,
-				ep->gsi_evt_ring_hdl);
-		*params->cached_gsi_evt_ring_hdl =
-			ep->gsi_evt_ring_hdl;
-
-	} else {
-		IPA_MHI_DBG("event ring already exists: evt_ring_hdl=%lu\n",
-			*params->cached_gsi_evt_ring_hdl);
-		ep->gsi_evt_ring_hdl = *params->cached_gsi_evt_ring_hdl;
-	}
-
-	if (params->ev_ctx_host->wp == params->ev_ctx_host->rbase) {
-		IPA_MHI_ERR("event ring wp is not updated. base=wp=0x%llx\n",
-			params->ev_ctx_host->wp);
-		goto fail_alloc_ch;
-	}
-
-	IPA_MHI_DBG("Ring event db: evt_ring_hdl=%lu host_wp=0x%llx\n",
-		ep->gsi_evt_ring_hdl, params->ev_ctx_host->wp);
-	res = gsi_ring_evt_ring_db(ep->gsi_evt_ring_hdl,
-		params->ev_ctx_host->wp);
-	if (res) {
-		IPA_MHI_ERR("fail to ring evt ring db %d. hdl=%lu wp=0x%llx\n",
-			res, ep->gsi_evt_ring_hdl, params->ev_ctx_host->wp);
-		goto fail_alloc_ch;
-	}
-
-	memset(&ch_props, 0, sizeof(ch_props));
-	ch_props.prot = GSI_CHAN_PROT_MHI;
-	ch_props.dir = IPA_CLIENT_IS_PROD(client) ?
-		GSI_CHAN_DIR_TO_GSI : GSI_CHAN_DIR_FROM_GSI;
-	ch_props.ch_id = ep_cfg->ipa_gsi_chan_num;
-	ch_props.evt_ring_hdl = *params->cached_gsi_evt_ring_hdl;
-	ch_props.re_size = GSI_CHAN_RE_SIZE_16B;
-	ch_props.ring_len = params->ch_ctx_host->rlen;
-	ch_props.ring_base_addr = IPA_MHI_HOST_ADDR_COND(
-			params->ch_ctx_host->rbase);
-
-	/* Burst mode is not supported on DPL pipes */
-	if ((client != IPA_CLIENT_MHI_DPL_CONS) &&
-		(params->ch_ctx_host->brstmode == IPA_MHI_BURST_MODE_DEFAULT ||
-		params->ch_ctx_host->brstmode == IPA_MHI_BURST_MODE_ENABLE)) {
-		burst_mode_enabled = true;
-	}
-
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0 &&
-		!burst_mode_enabled)
-		ch_props.use_db_eng = GSI_CHAN_DIRECT_MODE;
-	else
-		ch_props.use_db_eng = GSI_CHAN_DB_MODE;
-
-	ch_props.max_prefetch = GSI_ONE_PREFETCH_SEG;
-	ch_props.low_weight = 1;
-	ch_props.prefetch_mode = ep_cfg->prefetch_mode;
-	ch_props.empty_lvl_threshold = ep_cfg->prefetch_threshold;
-	ch_props.err_cb = params->ch_err_cb;
-	ch_props.chan_user_data = params->channel;
-	res = gsi_alloc_channel(&ch_props, ipa3_ctx->gsi_dev_hdl,
-		&ep->gsi_chan_hdl);
-	if (res) {
-		IPA_MHI_ERR("gsi_alloc_channel failed %d\n",
-			res);
-		goto fail_alloc_ch;
-	}
-
-	memset(&ch_scratch, 0, sizeof(ch_scratch));
-	ch_scratch.mhi.mhi_host_wp_addr = IPA_MHI_HOST_ADDR_COND(
-			params->channel_context_addr +
-			offsetof(struct ipa_mhi_ch_ctx, wp));
-	ch_scratch.mhi.assert_bit40 = params->assert_bit40;
-
-	/*
-	 * Update scratch for MCS smart prefetch:
-	 * Starting IPA4.5, smart prefetch implemented by H/W.
-	 * At IPA 4.0/4.1/4.2, we do not use MCS smart prefetch
-	 *  so keep the fields zero.
-	 */
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_0) {
-		ch_scratch.mhi.max_outstanding_tre =
-			ep_cfg->ipa_if_tlv * ch_props.re_size;
-		ch_scratch.mhi.outstanding_threshold =
-			min(ep_cfg->ipa_if_tlv / 2, 8) * ch_props.re_size;
-	}
-	ch_scratch.mhi.oob_mod_threshold = 4;
-
-	if (burst_mode_enabled) {
-		ch_scratch.mhi.burst_mode_enabled = burst_mode_enabled;
-		ch_scratch.mhi.polling_configuration =
-			ipa3_mhi_get_ch_poll_cfg(client, params->ch_ctx_host,
-				(ch_props.ring_len / ch_props.re_size));
-		ch_scratch.mhi.polling_mode = IPA_MHI_POLLING_MODE_DB_MODE;
-	} else {
-		ch_scratch.mhi.burst_mode_enabled = false;
-	}
-	res = gsi_write_channel_scratch(ep->gsi_chan_hdl,
-		ch_scratch);
-	if (res) {
-		IPA_MHI_ERR("gsi_write_channel_scratch failed %d\n",
-			res);
-		goto fail_ch_scratch;
-	}
-
-	*params->mhi = ch_scratch.mhi;
-
-	if (IPA_CLIENT_IS_PROD(ep->client) && ep->skip_ep_cfg) {
-		memset(&ep_cfg_ctrl, 0, sizeof(struct ipa_ep_cfg_ctrl));
-		ep_cfg_ctrl.ipa_ep_delay = true;
-		ep->ep_delay_set = true;
-		res = ipa3_cfg_ep_ctrl(ipa_ep_idx, &ep_cfg_ctrl);
-		if (res)
-			IPA_MHI_ERR("client (ep: %d) failed result=%d\n",
-			ipa_ep_idx, res);
-		else
-			IPA_MHI_DBG("client (ep: %d) success\n", ipa_ep_idx);
-	} else {
-		ep->ep_delay_set = false;
-	}
-
-	IPA_MHI_DBG("Starting channel\n");
-	res = gsi_start_channel(ep->gsi_chan_hdl);
-	if (res) {
-		IPA_MHI_ERR("gsi_start_channel failed %d\n", res);
-		goto fail_ch_start;
-	}
-
-	IPA_MHI_FUNC_EXIT();
-	return 0;
-
-fail_ch_start:
-fail_ch_scratch:
-	gsi_dealloc_channel(ep->gsi_chan_hdl);
-fail_alloc_ch:
-	gsi_dealloc_evt_ring(ep->gsi_evt_ring_hdl);
-	ep->gsi_evt_ring_hdl = ~0;
-fail_alloc_evt:
-	return res;
-}
-
-int ipa3_mhi_init_engine(struct ipa_mhi_init_engine *params)
-{
-	int res;
-	struct gsi_device_scratch gsi_scratch;
-	const struct ipa_gsi_ep_config *gsi_ep_info;
-
-	IPA_MHI_FUNC_ENTRY();
-
-	if (!params) {
-		IPA_MHI_ERR("null args\n");
-		return -EINVAL;
-	}
-
-	if ((IPA_MHI_MAX_UL_CHANNELS + IPA_MHI_MAX_DL_CHANNELS) >
-		((ipa3_ctx->mhi_evid_limits[1] -
-		ipa3_ctx->mhi_evid_limits[0]) + 1)) {
-		IPAERR("Not enough event rings for MHI\n");
-		ipa_assert();
-		return -EINVAL;
-	}
-
-	/* Initialize IPA MHI engine */
-	gsi_ep_info = ipa3_get_gsi_ep_info(IPA_CLIENT_MHI_PROD);
-	if (!gsi_ep_info) {
-		IPAERR("MHI PROD has no ep allocated\n");
-		ipa_assert();
-	}
-	memset(&gsi_scratch, 0, sizeof(gsi_scratch));
-	gsi_scratch.mhi_base_chan_idx_valid = true;
-	gsi_scratch.mhi_base_chan_idx = gsi_ep_info->ipa_gsi_chan_num +
-		params->gsi.first_ch_idx;
-	res = gsi_write_device_scratch(ipa3_ctx->gsi_dev_hdl,
-		&gsi_scratch);
-	if (res) {
-		IPA_MHI_ERR("failed to write device scratch %d\n", res);
-		goto fail_init_engine;
-	}
-
-	IPA_MHI_FUNC_EXIT();
-	return 0;
-
-fail_init_engine:
-	return res;
-}
-
-/**
- * ipa3_connect_mhi_pipe() - Connect pipe to IPA and start corresponding
- * MHI channel
- * @in: connect parameters
- * @clnt_hdl: [out] client handle for this pipe
- *
- * This function is called by IPA MHI client driver on MHI channel start.
- * This function is called after MHI engine was started.
- *
- * Return codes: 0	  : success
- *		 negative : error
- */
-int ipa3_connect_mhi_pipe(struct ipa_mhi_connect_params_internal *in,
-		u32 *clnt_hdl)
-{
-	struct ipa3_ep_context *ep;
-	int ipa_ep_idx;
-	int res;
-	enum ipa_client_type client;
-
-	IPA_MHI_FUNC_ENTRY();
-
-	if (!in || !clnt_hdl) {
-		IPA_MHI_ERR("NULL args\n");
-		return -EINVAL;
-	}
-
-	in->start.gsi.evchid += ipa3_ctx->mhi_evid_limits[0];
-
-	client = in->sys->client;
-	ipa_ep_idx = ipa3_get_ep_mapping(client);
-	if (ipa_ep_idx == -1) {
-		IPA_MHI_ERR("Invalid client.\n");
-		return -EINVAL;
-	}
-
-	ep = &ipa3_ctx->ep[ipa_ep_idx];
-
-	if (ep->valid == 1) {
-		IPA_MHI_ERR("EP already allocated.\n");
-		return -EPERM;
-	}
-
-	memset(ep, 0, offsetof(struct ipa3_ep_context, sys));
-	ep->valid = 1;
-	ep->skip_ep_cfg = in->sys->skip_ep_cfg;
-	ep->client = client;
-	ep->client_notify = in->sys->notify;
-	ep->priv = in->sys->priv;
-	ep->keep_ipa_awake = in->sys->keep_ipa_awake;
-
-	res = ipa_mhi_start_gsi_channel(client,
-					ipa_ep_idx, &in->start.gsi);
-	if (res) {
-		IPA_MHI_ERR("ipa_mhi_start_gsi_channel failed %d\n",
-			res);
-		goto fail_start_channel;
-	}
-
-	res = ipa3_enable_data_path(ipa_ep_idx);
-	if (res) {
-		IPA_MHI_ERR("enable data path failed res=%d clnt=%d.\n", res,
-			ipa_ep_idx);
-		goto fail_ep_cfg;
-	}
-
-	if (!ep->skip_ep_cfg) {
-		if (ipa3_cfg_ep(ipa_ep_idx, &in->sys->ipa_ep_cfg)) {
-			IPAERR("fail to configure EP.\n");
-			goto fail_ep_cfg;
-		}
-		if (ipa3_cfg_ep_status(ipa_ep_idx, &ep->status)) {
-			IPAERR("fail to configure status of EP.\n");
-			goto fail_ep_cfg;
-		}
-		IPA_MHI_DBG("ep configuration successful\n");
-	} else {
-		IPA_MHI_DBG("skipping ep configuration\n");
-	}
-
-	*clnt_hdl = ipa_ep_idx;
-
-	if (!ep->skip_ep_cfg && IPA_CLIENT_IS_PROD(client))
-		ipa3_install_dflt_flt_rules(ipa_ep_idx);
-
-	ipa3_ctx->skip_ep_cfg_shadow[ipa_ep_idx] = ep->skip_ep_cfg;
-	IPA_MHI_DBG("client %d (ep: %d) connected\n", client,
-		ipa_ep_idx);
-
-	IPA_MHI_FUNC_EXIT();
-
-	return 0;
-
-fail_ep_cfg:
-	ipa3_disable_data_path(ipa_ep_idx);
-fail_start_channel:
-	memset(ep, 0, offsetof(struct ipa3_ep_context, sys));
-	return -EPERM;
-}
-
-/**
- * ipa3_disconnect_mhi_pipe() - Disconnect pipe from IPA and reset corresponding
- * MHI channel
- * @clnt_hdl: client handle for this pipe
- *
- * This function is called by IPA MHI client driver on MHI channel reset.
- * This function is called after MHI channel was started.
- * This function is doing the following:
- *	- Send command to uC/GSI to reset corresponding MHI channel
- *	- Configure IPA EP control
- *
- * Return codes: 0	  : success
- *		 negative : error
- */
-int ipa3_disconnect_mhi_pipe(u32 clnt_hdl)
-{
-	struct ipa3_ep_context *ep;
-	int res;
-	struct ipa_ep_cfg_ctrl ep_cfg_ctrl;
-
-	IPA_MHI_FUNC_ENTRY();
-
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes) {
-		IPAERR("invalid handle %d\n", clnt_hdl);
-		return -EINVAL;
-	}
-
-	if (ipa3_ctx->ep[clnt_hdl].valid == 0) {
-		IPAERR("pipe was not connected %d\n", clnt_hdl);
-		return -EINVAL;
-	}
-
-	ep = &ipa3_ctx->ep[clnt_hdl];
-	if (ep->ep_delay_set) {
-		memset(&ep_cfg_ctrl, 0, sizeof(struct ipa_ep_cfg_ctrl));
-		ep_cfg_ctrl.ipa_ep_delay = false;
-		res = ipa3_cfg_ep_ctrl(clnt_hdl,
-			&ep_cfg_ctrl);
-		if (res) {
-			IPAERR
-			("client(ep:%d) failed to remove delay res=%d\n",
-				clnt_hdl, res);
-		} else {
-			IPADBG("client (ep: %d) delay removed\n",
-				clnt_hdl);
-			ep->ep_delay_set = false;
-		}
-	}
-
-	res = gsi_dealloc_channel(ep->gsi_chan_hdl);
-	if (res) {
-		IPAERR("gsi_dealloc_channel failed %d\n", res);
-		goto fail_reset_channel;
-	}
-
-	ep->valid = 0;
-	ipa3_delete_dflt_flt_rules(clnt_hdl);
-
-	IPA_MHI_DBG("client (ep: %d) disconnected\n", clnt_hdl);
-	IPA_MHI_FUNC_EXIT();
-	return 0;
-
-fail_reset_channel:
-	return res;
-}
-
-int ipa3_mhi_resume_channels_internal(enum ipa_client_type client,
-		bool LPTransitionRejected, bool brstmode_enabled,
-		union __packed gsi_channel_scratch ch_scratch, u8 index)
-{
-	int res;
-	int ipa_ep_idx;
-	struct ipa3_ep_context *ep;
-	union __packed gsi_channel_scratch gsi_ch_scratch;
-
-	IPA_MHI_FUNC_ENTRY();
-
-	ipa_ep_idx = ipa3_get_ep_mapping(client);
-	if (ipa_ep_idx < 0) {
-		IPA_MHI_ERR("Invalid client %d\n", client);
-		return -EINVAL;
-	}
-	ep = &ipa3_ctx->ep[ipa_ep_idx];
-
-	if (brstmode_enabled && !LPTransitionRejected) {
-
-		res = gsi_read_channel_scratch(ep->gsi_chan_hdl,
-			&gsi_ch_scratch);
-		if (res) {
-			IPA_MHI_ERR("read ch scratch fail %d\n", res);
-			return res;
-		}
-
-		/*
-		 * set polling mode bit to DB mode before
-		 * resuming the channel
-		 *
-		 * For MHI-->IPA pipes:
-		 * when resuming due to transition to M0,
-		 * set the polling mode bit to 0.
-		 * In other cases, restore it's value form
-		 * when you stopped the channel.
-		 * Here, after successful resume client move to M0 state.
-		 * So, by default setting polling mode bit to 0.
-		 *
-		 * For IPA-->MHI pipe:
-		 * always restore the polling mode bit.
-		 */
-		if (IPA_CLIENT_IS_PROD(client))
-			ch_scratch.mhi.polling_mode =
-				IPA_MHI_POLLING_MODE_DB_MODE;
-		else
-			ch_scratch.mhi.polling_mode =
-				gsi_ch_scratch.mhi.polling_mode;
-
-		/* Use GSI update API to not affect non-SWI fields
-		 * inside the scratch while in suspend-resume operation
-		 */
-		res = gsi_update_mhi_channel_scratch(
-			ep->gsi_chan_hdl, ch_scratch.mhi);
-		if (res) {
-			IPA_MHI_ERR("write ch scratch fail %d\n"
-				, res);
-			return res;
-		}
-	}
-
-	res = gsi_start_channel(ep->gsi_chan_hdl);
-	if (res) {
-		IPA_MHI_ERR("failed to resume channel error %d\n", res);
-		return res;
-	}
-
-	IPA_MHI_FUNC_EXIT();
-	return 0;
-}
-
-int ipa3_mhi_query_ch_info(enum ipa_client_type client,
-		struct gsi_chan_info *ch_info)
-{
-	int ipa_ep_idx;
-	int res;
-	struct ipa3_ep_context *ep;
-
-	IPA_MHI_FUNC_ENTRY();
-
-	ipa_ep_idx = ipa3_get_ep_mapping(client);
-	if (ipa_ep_idx < 0) {
-		IPA_MHI_ERR("Invalid client %d\n", client);
-		return -EINVAL;
-	}
-	ep = &ipa3_ctx->ep[ipa_ep_idx];
-	res = gsi_query_channel_info(ep->gsi_chan_hdl, ch_info);
-	if (res) {
-		IPA_MHI_ERR("gsi_query_channel_info failed\n");
-		return res;
-	}
-
-	IPA_MHI_FUNC_EXIT();
-	return 0;
-}
-
-bool ipa3_has_open_aggr_frame(enum ipa_client_type client)
-{
-	u32 aggr_state_active;
-	int ipa_ep_idx;
-
-	aggr_state_active = ipahal_read_reg(IPA_STATE_AGGR_ACTIVE);
-	IPA_MHI_DBG_LOW("IPA_STATE_AGGR_ACTIVE_OFST 0x%x\n", aggr_state_active);
-
-	ipa_ep_idx = ipa_get_ep_mapping(client);
-	if (ipa_ep_idx == -1) {
-		ipa_assert();
-		return false;
-	}
-
-	if ((1 << ipa_ep_idx) & aggr_state_active)
-		return true;
-
-	return false;
-}
-
-int ipa3_mhi_destroy_channel(enum ipa_client_type client)
-{
-	int res;
-	int ipa_ep_idx;
-	struct ipa3_ep_context *ep;
-
-	ipa_ep_idx = ipa3_get_ep_mapping(client);
-	if (ipa_ep_idx < 0) {
-		IPA_MHI_ERR("Invalid client %d\n", client);
-		return -EINVAL;
-	}
-	ep = &ipa3_ctx->ep[ipa_ep_idx];
-
-	IPA_ACTIVE_CLIENTS_INC_EP(client);
-
-	IPA_MHI_DBG("reset event ring (hdl: %lu, ep: %d)\n",
-		ep->gsi_evt_ring_hdl, ipa_ep_idx);
-
-	res = gsi_reset_evt_ring(ep->gsi_evt_ring_hdl);
-	if (res) {
-		IPAERR(" failed to reset evt ring %lu, err %d\n"
-			, ep->gsi_evt_ring_hdl, res);
-		goto fail;
-	}
-
-	IPA_MHI_DBG("dealloc event ring (hdl: %lu, ep: %d)\n",
-		ep->gsi_evt_ring_hdl, ipa_ep_idx);
-
-	res = gsi_dealloc_evt_ring(
-		ep->gsi_evt_ring_hdl);
-	if (res) {
-		IPAERR("dealloc evt ring %lu failed, err %d\n"
-			, ep->gsi_evt_ring_hdl, res);
-		goto fail;
-	}
-
-	IPA_ACTIVE_CLIENTS_DEC_EP(client);
-	return 0;
-fail:
-	IPA_ACTIVE_CLIENTS_DEC_EP(client);
-	return res;
-}
-
-MODULE_LICENSE("GPL v2");
-MODULE_DESCRIPTION("IPA MHI driver");

ipa/ipa_v3/ipa_mhi_proxy.c

@@ -1,1075 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/dma-mapping.h>
-#include <linux/list.h>
-#include <linux/slab.h>
-#include <linux/device.h>
-#include <linux/module.h>
-#include <linux/mhi.h>
-#include "ipa_qmi_service.h"
-#include "../ipa_common_i.h"
-#include "ipa_i.h"
-
-#define IMP_DRV_NAME "ipa_mhi_proxy"
-
-#define IMP_DBG(fmt, args...) \
-	do { \
-		pr_debug(IMP_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-			IMP_DRV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			IMP_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-#define IMP_DBG_LOW(fmt, args...) \
-	do { \
-		pr_debug(IMP_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			IMP_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-
-#define IMP_ERR(fmt, args...) \
-	do { \
-		pr_err(IMP_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-				IMP_DRV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-				IMP_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-
-#define IMP_FUNC_ENTRY() \
-	IMP_DBG_LOW("ENTRY\n")
-#define IMP_FUNC_EXIT() \
-	IMP_DBG_LOW("EXIT\n")
-
-#define IMP_IPA_UC_UL_CH_n 0
-#define IMP_IPA_UC_UL_EV_n 1
-#define IMP_IPA_UC_DL_CH_n 2
-#define IMP_IPA_UC_DL_EV_n 3
-#define IMP_IPA_UC_m 1
-
-/* each pair of UL/DL channels are defined below */
-static const struct mhi_device_id mhi_driver_match_table[] = {
-	{ .chan = "IP_HW_OFFLOAD_0" },
-	{},
-};
-
-static int imp_mhi_probe_cb(struct mhi_device *, const struct mhi_device_id *);
-static void imp_mhi_remove_cb(struct mhi_device *);
-static void imp_mhi_status_cb(struct mhi_device *, enum MHI_CB);
-
-static struct mhi_driver mhi_driver = {
-	.id_table = mhi_driver_match_table,
-	.probe = imp_mhi_probe_cb,
-	.remove = imp_mhi_remove_cb,
-	.status_cb = imp_mhi_status_cb,
-	.driver = {
-		.name = IMP_DRV_NAME,
-		.owner = THIS_MODULE,
-	},
-};
-
-struct imp_channel_context_type {
-	u32 chstate:8;
-	u32 brsmode:2;
-	u32 pollcfg:6;
-	u32 reserved:16;
-
-	u32 chtype;
-
-	u32 erindex;
-
-	u64 rbase;
-
-	u64 rlen;
-
-	u64 rpp;
-
-	u64 wpp;
-} __packed;
-
-struct imp_event_context_type {
-	u32 reserved:8;
-	u32 intmodc:8;
-	u32 intmodt:16;
-
-	u32 ertype;
-
-	u32 msivec;
-
-	u64 rbase;
-
-	u64 rlen;
-
-	u64 rpp;
-
-	u64 wpp;
-} __packed;
-
-struct imp_iova_addr {
-	dma_addr_t base;
-	unsigned int size;
-};
-
-struct imp_dev_info {
-	struct platform_device *pdev;
-	bool smmu_enabled;
-	struct imp_iova_addr ctrl;
-	struct imp_iova_addr data;
-	u32 chdb_base;
-	u32 erdb_base;
-};
-
-struct imp_event_props {
-	u16 id;
-	phys_addr_t doorbell;
-	u16 uc_mbox_n;
-	struct imp_event_context_type ev_ctx;
-};
-
-struct imp_event {
-	struct imp_event_props props;
-};
-
-struct imp_channel_props {
-	enum dma_data_direction dir;
-	u16 id;
-	phys_addr_t doorbell;
-	u16 uc_mbox_n;
-	struct imp_channel_context_type ch_ctx;
-
-};
-
-struct imp_channel {
-	struct imp_channel_props props;
-	struct imp_event event;
-};
-
-enum imp_state {
-	IMP_INVALID = 0,
-	IMP_PROBED,
-	IMP_READY,
-	IMP_STARTED
-};
-
-struct imp_qmi_cache {
-	struct ipa_mhi_ready_indication_msg_v01 ready_ind;
-	struct ipa_mhi_alloc_channel_req_msg_v01 alloc_ch_req;
-	struct ipa_mhi_alloc_channel_resp_msg_v01 alloc_ch_resp;
-	struct ipa_mhi_clk_vote_resp_msg_v01 clk_vote_resp;
-};
-
-struct imp_mhi_driver {
-	struct mhi_device *mhi_dev;
-	struct imp_channel ul_chan;
-	struct imp_channel dl_chan;
-};
-
-struct imp_context {
-	struct imp_dev_info dev_info;
-	struct imp_mhi_driver md;
-	struct mutex mutex;
-	struct mutex lpm_mutex;
-	enum imp_state state;
-	bool in_lpm;
-	bool lpm_disabled;
-	struct imp_qmi_cache qmi;
-
-};
-
-static struct imp_context *imp_ctx;
-
-static void _populate_smmu_info(struct ipa_mhi_ready_indication_msg_v01 *req)
-{
-	req->smmu_info_valid = true;
-	req->smmu_info.iova_ctl_base_addr = imp_ctx->dev_info.ctrl.base;
-	req->smmu_info.iova_ctl_size = imp_ctx->dev_info.ctrl.size;
-	req->smmu_info.iova_data_base_addr = imp_ctx->dev_info.data.base;
-	req->smmu_info.iova_data_size = imp_ctx->dev_info.data.size;
-}
-
-static void imp_mhi_trigger_ready_ind(void)
-{
-	struct ipa_mhi_ready_indication_msg_v01 *req
-		= &imp_ctx->qmi.ready_ind;
-	int ret;
-	struct imp_channel *ch;
-	struct ipa_mhi_ch_init_info_type_v01 *ch_info;
-
-	IMP_FUNC_ENTRY();
-	if (imp_ctx->state != IMP_PROBED) {
-		IMP_ERR("invalid state %d\n", imp_ctx->state);
-		goto exit;
-	}
-
-	if (imp_ctx->dev_info.smmu_enabled)
-		_populate_smmu_info(req);
-
-	req->ch_info_arr_len = 0;
-	BUILD_BUG_ON(QMI_IPA_REMOTE_MHI_CHANNELS_NUM_MAX_V01 < 2);
-
-	/* UL channel */
-	ch = &imp_ctx->md.ul_chan;
-	ch_info = &req->ch_info_arr[req->ch_info_arr_len];
-
-	ch_info->ch_id = ch->props.id;
-	ch_info->direction_type = ch->props.dir;
-	ch_info->er_id = ch->event.props.id;
-
-	/* uC is a doorbell proxy between local Q6 and remote Q6 */
-	ch_info->ch_doorbell_addr = ipa3_ctx->ipa_wrapper_base +
-		ipahal_get_reg_base() +
-		ipahal_get_reg_mn_ofst(IPA_UC_MAILBOX_m_n,
-		IMP_IPA_UC_m,
-		ch->props.uc_mbox_n);
-
-	ch_info->er_doorbell_addr = ipa3_ctx->ipa_wrapper_base +
-		ipahal_get_reg_base() +
-		ipahal_get_reg_mn_ofst(IPA_UC_MAILBOX_m_n,
-		IMP_IPA_UC_m,
-		ch->event.props.uc_mbox_n);
-	req->ch_info_arr_len++;
-
-	/* DL channel */
-	ch = &imp_ctx->md.dl_chan;
-	ch_info = &req->ch_info_arr[req->ch_info_arr_len];
-
-	ch_info->ch_id = ch->props.id;
-	ch_info->direction_type = ch->props.dir;
-	ch_info->er_id = ch->event.props.id;
-
-	/* uC is a doorbell proxy between local Q6 and remote Q6 */
-	ch_info->ch_doorbell_addr = ipa3_ctx->ipa_wrapper_base +
-		ipahal_get_reg_base() +
-		ipahal_get_reg_mn_ofst(IPA_UC_MAILBOX_m_n,
-		IMP_IPA_UC_m,
-		ch->props.uc_mbox_n);
-
-	ch_info->er_doorbell_addr = ipa3_ctx->ipa_wrapper_base +
-		ipahal_get_reg_base() +
-		ipahal_get_reg_mn_ofst(IPA_UC_MAILBOX_m_n,
-		IMP_IPA_UC_m,
-		ch->event.props.uc_mbox_n);
-	req->ch_info_arr_len++;
-
-	IMP_DBG("sending IND to modem\n");
-	ret = ipa3_qmi_send_mhi_ready_indication(req);
-	if (ret) {
-		IMP_ERR("failed to send ready indication to modem %d\n", ret);
-		return;
-	}
-
-	imp_ctx->state = IMP_READY;
-
-exit:
-	IMP_FUNC_EXIT();
-}
-
-static struct imp_channel *imp_get_ch_by_id(u16 id)
-{
-	if (imp_ctx->md.ul_chan.props.id == id)
-		return &imp_ctx->md.ul_chan;
-
-	if (imp_ctx->md.dl_chan.props.id == id)
-		return &imp_ctx->md.dl_chan;
-
-	return NULL;
-}
-
-static struct ipa_mhi_er_info_type_v01 *
-	_find_ch_in_er_info_arr(struct ipa_mhi_alloc_channel_req_msg_v01 *req,
-	u16 id)
-{
-	int i;
-
-	if (req->er_info_arr_len > QMI_IPA_REMOTE_MHI_CHANNELS_NUM_MAX_V01)
-		return NULL;
-
-	for (i = 0; i < req->tr_info_arr_len; i++)
-		if (req->er_info_arr[i].er_id == id)
-			return &req->er_info_arr[i];
-	return NULL;
-}
-
-/* round addresses for closest page per SMMU requirements */
-static inline void imp_smmu_round_to_page(uint64_t iova, uint64_t pa,
-	uint64_t size, unsigned long *iova_p, phys_addr_t *pa_p, u32 *size_p)
-{
-	*iova_p = rounddown(iova, PAGE_SIZE);
-	*pa_p = rounddown(pa, PAGE_SIZE);
-	*size_p = roundup(size + pa - *pa_p, PAGE_SIZE);
-}
-
-static void __map_smmu_info(struct device *dev,
-	struct imp_iova_addr *partition, int num_mapping,
-	struct ipa_mhi_mem_addr_info_type_v01 *map_info,
-	bool map)
-{
-	int i;
-	struct iommu_domain *domain;
-	unsigned long iova_p;
-	phys_addr_t pa_p;
-	u32 size_p;
-
-	domain = iommu_get_domain_for_dev(dev);
-	if (!domain) {
-		IMP_ERR("domain is NULL for dev\n");
-		return;
-	}
-
-	for (i = 0; i < num_mapping; i++) {
-		int prot = IOMMU_READ | IOMMU_WRITE;
-		u32 ipa_base = ipa3_ctx->ipa_wrapper_base +
-			ipa3_ctx->ctrl->ipa_reg_base_ofst;
-		u32 ipa_size = ipa3_ctx->ipa_wrapper_size;
-
-		imp_smmu_round_to_page(map_info[i].iova, map_info[i].pa,
-			map_info[i].size, &iova_p, &pa_p, &size_p);
-
-		if (map) {
-			/* boundary check */
-			WARN_ON(partition->base > iova_p ||
-				(partition->base + partition->size) <
-				(iova_p + size_p));
-
-			/* for IPA uC MBOM we need to map with device type */
-			if (pa_p - ipa_base < ipa_size)
-				prot |= IOMMU_MMIO;
-
-			IMP_DBG("mapping 0x%lx to 0x%pa size %d\n",
-				iova_p, &pa_p, size_p);
-			iommu_map(domain,
-				iova_p, pa_p, size_p, prot);
-		} else {
-			IMP_DBG("unmapping 0x%lx to 0x%pa size %d\n",
-				iova_p, &pa_p, size_p);
-			iommu_unmap(domain, iova_p, size_p);
-		}
-	}
-}
-
-static int __imp_configure_mhi_device(
-	struct ipa_mhi_alloc_channel_req_msg_v01 *req,
-	struct ipa_mhi_alloc_channel_resp_msg_v01 *resp)
-{
-	struct mhi_buf ch_config[2];
-	int i;
-	struct ipa_mhi_er_info_type_v01 *er_info;
-	struct imp_channel *ch;
-	int ridx = 0;
-	int ret;
-
-	IMP_FUNC_ENTRY();
-
-	/* configure MHI */
-	for (i = 0; i < req->tr_info_arr_len; i++) {
-		ch = imp_get_ch_by_id(req->tr_info_arr[i].ch_id);
-		if (!ch) {
-			IMP_ERR("unknown channel %d\n",
-				req->tr_info_arr[i].ch_id);
-			resp->alloc_resp_arr[ridx].ch_id =
-				req->tr_info_arr[i].ch_id;
-			resp->alloc_resp_arr[ridx].is_success = 0;
-			ridx++;
-			resp->alloc_resp_arr_len = ridx;
-			resp->resp.result = IPA_QMI_RESULT_FAILURE_V01;
-			/* return INCOMPATIBLE_STATE in any case */
-			resp->resp.error =
-				IPA_QMI_ERR_INCOMPATIBLE_STATE_V01;
-			return -EINVAL;
-		}
-
-		/* populate CCA */
-		if (req->tr_info_arr[i].brst_mode_type ==
-			QMI_IPA_BURST_MODE_ENABLED_V01)
-			ch->props.ch_ctx.brsmode = 3;
-		else if (req->tr_info_arr[i].brst_mode_type ==
-			QMI_IPA_BURST_MODE_DISABLED_V01)
-			ch->props.ch_ctx.brsmode = 2;
-		else
-			ch->props.ch_ctx.brsmode = 0;
-
-		ch->props.ch_ctx.pollcfg = req->tr_info_arr[i].poll_cfg;
-		ch->props.ch_ctx.chtype = ch->props.dir;
-		ch->props.ch_ctx.erindex = ch->event.props.id;
-		ch->props.ch_ctx.rbase = req->tr_info_arr[i].ring_iova;
-		ch->props.ch_ctx.rlen = req->tr_info_arr[i].ring_len;
-		ch->props.ch_ctx.rpp = req->tr_info_arr[i].rp;
-		ch->props.ch_ctx.wpp = req->tr_info_arr[i].wp;
-
-		ch_config[0].buf = &ch->props.ch_ctx;
-		ch_config[0].len = sizeof(ch->props.ch_ctx);
-		ch_config[0].name = "CCA";
-
-		/* populate ECA */
-		er_info = _find_ch_in_er_info_arr(req, ch->event.props.id);
-		if (!er_info) {
-			IMP_ERR("no event ring for ch %d\n",
-				req->tr_info_arr[i].ch_id);
-			resp->alloc_resp_arr[ridx].ch_id =
-				req->tr_info_arr[i].ch_id;
-			resp->alloc_resp_arr[ridx].is_success = 0;
-			ridx++;
-			resp->alloc_resp_arr_len = ridx;
-			resp->resp.result = IPA_QMI_RESULT_FAILURE_V01;
-			resp->resp.error = IPA_QMI_ERR_INTERNAL_V01;
-			return -EINVAL;
-		}
-
-		ch->event.props.ev_ctx.intmodc = er_info->intmod_count;
-		ch->event.props.ev_ctx.intmodt = er_info->intmod_cycles;
-		ch->event.props.ev_ctx.ertype = 1;
-		ch->event.props.ev_ctx.msivec = er_info->msi_addr;
-		ch->event.props.ev_ctx.rbase = er_info->ring_iova;
-		ch->event.props.ev_ctx.rlen = er_info->ring_len;
-		ch->event.props.ev_ctx.rpp = er_info->rp;
-		ch->event.props.ev_ctx.wpp = er_info->wp;
-		ch_config[1].buf = &ch->event.props.ev_ctx;
-		ch_config[1].len = sizeof(ch->event.props.ev_ctx);
-		ch_config[1].name = "ECA";
-
-		IMP_DBG("Configuring MHI device for ch %d\n", ch->props.id);
-		ret = mhi_device_configure(imp_ctx->md.mhi_dev, ch->props.dir,
-			ch_config, 2);
-		/* configure mhi-host, no need check mhi state */
-		if (ret) {
-			IMP_ERR("mhi_device_configure failed for ch %d\n",
-				req->tr_info_arr[i].ch_id);
-			resp->alloc_resp_arr[ridx].ch_id =
-				req->tr_info_arr[i].ch_id;
-			resp->alloc_resp_arr[ridx].is_success = 0;
-			ridx++;
-			resp->alloc_resp_arr_len = ridx;
-			resp->resp.result = IPA_QMI_RESULT_FAILURE_V01;
-			resp->resp.error = IPA_QMI_ERR_INTERNAL_V01;
-			return -EINVAL;
-		}
-	}
-
-	IMP_FUNC_EXIT();
-
-	return 0;
-}
-
-/**
- * imp_handle_allocate_channel_req() - Allocate a new MHI channel
- *
- * Allocates MHI channel and start them.
- *
- * Return: QMI return codes
- */
-struct ipa_mhi_alloc_channel_resp_msg_v01 *imp_handle_allocate_channel_req(
-		struct ipa_mhi_alloc_channel_req_msg_v01 *req)
-{
-	int ret;
-	struct ipa_mhi_alloc_channel_resp_msg_v01 *resp =
-		&imp_ctx->qmi.alloc_ch_resp;
-
-	IMP_FUNC_ENTRY();
-
-	mutex_lock(&imp_ctx->mutex);
-
-	memset(resp, 0, sizeof(*resp));
-
-	if (imp_ctx->state != IMP_READY) {
-		IMP_ERR("invalid state %d\n", imp_ctx->state);
-		resp->resp.result = IPA_QMI_RESULT_FAILURE_V01;
-		resp->resp.error = IPA_QMI_ERR_INTERNAL_V01;
-		mutex_unlock(&imp_ctx->mutex);
-		return resp;
-	}
-
-	/* cache the req */
-	memcpy(&imp_ctx->qmi.alloc_ch_req, req, sizeof(*req));
-
-	if (req->tr_info_arr_len > QMI_IPA_REMOTE_MHI_CHANNELS_NUM_MAX_V01) {
-		IMP_ERR("invalid tr_info_arr_len %d\n", req->tr_info_arr_len);
-		resp->resp.result = IPA_QMI_RESULT_FAILURE_V01;
-		resp->resp.error = IPA_QMI_ERR_NO_MEMORY_V01;
-		mutex_unlock(&imp_ctx->mutex);
-		return resp;
-	}
-
-	if ((req->ctrl_addr_map_info_len == 0 ||
-	     req->data_addr_map_info_len == 0) &&
-	     imp_ctx->dev_info.smmu_enabled) {
-		IMP_ERR("no mapping provided, but smmu is enabled\n");
-		resp->resp.result = IPA_QMI_RESULT_FAILURE_V01;
-		resp->resp.error = IPA_QMI_ERR_INTERNAL_V01;
-		mutex_unlock(&imp_ctx->mutex);
-		return resp;
-	}
-
-	if (imp_ctx->dev_info.smmu_enabled) {
-		/* map CTRL */
-		__map_smmu_info(imp_ctx->md.mhi_dev->dev.parent,
-			&imp_ctx->dev_info.ctrl,
-			req->ctrl_addr_map_info_len,
-			req->ctrl_addr_map_info,
-			true);
-
-		/* map DATA */
-		__map_smmu_info(imp_ctx->md.mhi_dev->dev.parent,
-			&imp_ctx->dev_info.data,
-			req->data_addr_map_info_len,
-			req->data_addr_map_info,
-			true);
-	}
-
-	resp->alloc_resp_arr_valid = true;
-	ret = __imp_configure_mhi_device(req, resp);
-	if (ret)
-		goto fail_smmu;
-
-	IMP_DBG("Starting MHI channels %d and %d\n",
-		imp_ctx->md.ul_chan.props.id,
-		imp_ctx->md.dl_chan.props.id);
-	ret = mhi_prepare_for_transfer(imp_ctx->md.mhi_dev);
-	if (ret) {
-		IMP_ERR("mhi_prepare_for_transfer failed %d\n", ret);
-		resp->alloc_resp_arr[resp->alloc_resp_arr_len]
-			.ch_id = imp_ctx->md.ul_chan.props.id;
-		resp->alloc_resp_arr[resp->alloc_resp_arr_len]
-			.is_success = 0;
-		resp->alloc_resp_arr_len++;
-		resp->alloc_resp_arr[resp->alloc_resp_arr_len]
-			.ch_id = imp_ctx->md.dl_chan.props.id;
-		resp->alloc_resp_arr[resp->alloc_resp_arr_len]
-			.is_success = 0;
-		resp->alloc_resp_arr_len++;
-		resp->resp.result = IPA_QMI_RESULT_FAILURE_V01;
-		/* return INCOMPATIBLE_STATE in any case */
-		resp->resp.error = IPA_QMI_ERR_INCOMPATIBLE_STATE_V01;
-		goto fail_smmu;
-	}
-
-	resp->alloc_resp_arr[resp->alloc_resp_arr_len]
-		.ch_id = imp_ctx->md.ul_chan.props.id;
-	resp->alloc_resp_arr[resp->alloc_resp_arr_len]
-		.is_success = 1;
-	resp->alloc_resp_arr_len++;
-
-	resp->alloc_resp_arr[resp->alloc_resp_arr_len]
-		.ch_id = imp_ctx->md.dl_chan.props.id;
-	resp->alloc_resp_arr[resp->alloc_resp_arr_len]
-		.is_success = 1;
-	resp->alloc_resp_arr_len++;
-
-	imp_ctx->state = IMP_STARTED;
-	mutex_unlock(&imp_ctx->mutex);
-	IMP_FUNC_EXIT();
-
-	resp->resp.result = IPA_QMI_RESULT_SUCCESS_V01;
-	return resp;
-
-fail_smmu:
-	if (imp_ctx->dev_info.smmu_enabled) {
-		/* unmap CTRL */
-		__map_smmu_info(imp_ctx->md.mhi_dev->dev.parent,
-			&imp_ctx->dev_info.ctrl,
-			req->ctrl_addr_map_info_len,
-			req->ctrl_addr_map_info,
-			false);
-
-		/* unmap DATA */
-		__map_smmu_info(imp_ctx->md.mhi_dev->dev.parent,
-			&imp_ctx->dev_info.data,
-			req->data_addr_map_info_len,
-			req->data_addr_map_info,
-			false);
-	}
-	mutex_unlock(&imp_ctx->mutex);
-	return resp;
-}
-
-/**
- * imp_handle_vote_req() - Votes for MHI / PCIe clocks
- *
- * Hold a vote to prevent / allow low power mode on MHI.
- *
- * Return: 0 on success, negative otherwise
- */
-struct ipa_mhi_clk_vote_resp_msg_v01
-	*imp_handle_vote_req(bool vote)
-{
-	int ret;
-	struct ipa_mhi_clk_vote_resp_msg_v01 *resp =
-	&imp_ctx->qmi.clk_vote_resp;
-
-	IMP_DBG_LOW("vote %d\n", vote);
-	memset(resp, 0, sizeof(struct ipa_mhi_clk_vote_resp_msg_v01));
-	resp->resp.result = IPA_QMI_RESULT_FAILURE_V01;
-	resp->resp.error = IPA_QMI_ERR_INCOMPATIBLE_STATE_V01;
-
-	mutex_lock(&imp_ctx->mutex);
-	if (imp_ctx->state != IMP_STARTED) {
-		IMP_ERR("unexpected vote when in state %d\n", imp_ctx->state);
-		mutex_unlock(&imp_ctx->mutex);
-		return resp;
-	}
-
-	if (vote == imp_ctx->lpm_disabled) {
-		IMP_ERR("already voted/devoted %d\n", vote);
-		mutex_unlock(&imp_ctx->mutex);
-		return resp;
-	}
-	mutex_unlock(&imp_ctx->mutex);
-
-	/*
-	 * Unlock the mutex before calling into mhi for clock vote
-	 * to avoid deadlock on imp mutex.
-	 * Calls into mhi are synchronous and imp callbacks are
-	 * executed from mhi context.
-	 */
-	if (vote) {
-		ret = mhi_device_get_sync(imp_ctx->md.mhi_dev, MHI_VOTE_BUS);
-		if (ret) {
-			IMP_ERR("mhi_sync_get failed %d\n", ret);
-			resp->resp.result = IPA_QMI_RESULT_FAILURE_V01;
-			/* return INCOMPATIBLE_STATE in any case */
-			resp->resp.error =
-					IPA_QMI_ERR_INCOMPATIBLE_STATE_V01;
-			return resp;
-		}
-	} else {
-		mhi_device_put(imp_ctx->md.mhi_dev, MHI_VOTE_BUS);
-	}
-
-	mutex_lock(&imp_ctx->mutex);
-	if (vote)
-		imp_ctx->lpm_disabled = true;
-	else
-		imp_ctx->lpm_disabled = false;
-	mutex_unlock(&imp_ctx->mutex);
-
-	resp->resp.result = IPA_QMI_RESULT_SUCCESS_V01;
-	return resp;
-}
-
-static int imp_read_iova_from_dtsi(const char *node, struct imp_iova_addr *out)
-{
-	u32 iova_mapping[2];
-	struct device_node *of_node = imp_ctx->dev_info.pdev->dev.of_node;
-
-	if (of_property_read_u32_array(of_node, node, iova_mapping, 2)) {
-		IMP_DBG("failed to read of_node %s\n", node);
-		return -EINVAL;
-	}
-
-	out->base = iova_mapping[0];
-	out->size = iova_mapping[1];
-	IMP_DBG("%s: base: 0x%pad size: 0x%x\n", node, &out->base, out->size);
-
-	return 0;
-}
-
-static void imp_mhi_shutdown(void)
-{
-	struct ipa_mhi_cleanup_req_msg_v01 req = { 0 };
-
-	IMP_FUNC_ENTRY();
-
-	if (imp_ctx->state == IMP_STARTED ||
-		imp_ctx->state == IMP_READY) {
-		req.cleanup_valid = true;
-		req.cleanup = true;
-		ipa3_qmi_send_mhi_cleanup_request(&req);
-		if (imp_ctx->dev_info.smmu_enabled) {
-			struct ipa_mhi_alloc_channel_req_msg_v01 *creq
-				= &imp_ctx->qmi.alloc_ch_req;
-
-			/* unmap CTRL */
-			__map_smmu_info(imp_ctx->md.mhi_dev->dev.parent,
-				&imp_ctx->dev_info.ctrl,
-				creq->ctrl_addr_map_info_len,
-				creq->ctrl_addr_map_info,
-				false);
-
-			/* unmap DATA */
-			__map_smmu_info(imp_ctx->md.mhi_dev->dev.parent,
-				&imp_ctx->dev_info.data,
-				creq->data_addr_map_info_len,
-				creq->data_addr_map_info,
-				false);
-		}
-		if (imp_ctx->lpm_disabled) {
-			mhi_device_put(imp_ctx->md.mhi_dev, MHI_VOTE_BUS);
-			imp_ctx->lpm_disabled = false;
-		}
-
-		/* unmap MHI doorbells from IPA uC SMMU */
-		if (!ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_UC]) {
-			struct ipa_smmu_cb_ctx *cb =
-				ipa3_get_smmu_ctx(IPA_SMMU_CB_UC);
-			unsigned long iova_p;
-			phys_addr_t pa_p;
-			u32 size_p;
-
-			imp_smmu_round_to_page(imp_ctx->dev_info.chdb_base,
-				imp_ctx->dev_info.chdb_base, PAGE_SIZE,
-				&iova_p, &pa_p, &size_p);
-
-			iommu_unmap(cb->iommu_domain, iova_p, size_p);
-		}
-	}
-	if (!imp_ctx->in_lpm &&
-		(imp_ctx->state == IMP_READY ||
-			imp_ctx->state == IMP_STARTED)) {
-		IMP_DBG("devote IMP with state= %d\n", imp_ctx->state);
-		IPA_ACTIVE_CLIENTS_DEC_SPECIAL("IMP");
-	}
-	imp_ctx->in_lpm = false;
-	imp_ctx->state = IMP_PROBED;
-
-	IMP_FUNC_EXIT();
-}
-
-static int imp_mhi_probe_cb(struct mhi_device *mhi_dev,
-	const struct mhi_device_id *id)
-{
-	struct imp_channel *ch;
-	struct imp_event *ev;
-	int ret;
-
-	IMP_FUNC_ENTRY();
-
-	if (id != &mhi_driver_match_table[0]) {
-		IMP_ERR("only chan=%s is supported for now\n",
-			mhi_driver_match_table[0].chan);
-		return -EPERM;
-	}
-
-	/* vote for IPA clock. IPA clock will be devoted when MHI enters LPM */
-	IPA_ACTIVE_CLIENTS_INC_SPECIAL("IMP");
-
-	imp_ctx->md.mhi_dev = mhi_dev;
-
-	mutex_lock(&imp_ctx->mutex);
-	/* store UL channel properties */
-	ch = &imp_ctx->md.ul_chan;
-	ev = &imp_ctx->md.ul_chan.event;
-
-	ch->props.id = mhi_dev->ul_chan_id;
-	ch->props.dir = DMA_TO_DEVICE;
-	ch->props.doorbell = imp_ctx->dev_info.chdb_base + ch->props.id * 8;
-	ch->props.uc_mbox_n = IMP_IPA_UC_UL_CH_n;
-	IMP_DBG("ul ch id %d doorbell 0x%pa uc_mbox_n %d\n",
-		ch->props.id, &ch->props.doorbell, ch->props.uc_mbox_n);
-
-	ret = ipa3_uc_send_remote_ipa_info(ch->props.doorbell,
-		ch->props.uc_mbox_n);
-	if (ret)
-		goto fail;
-	IMP_DBG("mapped ch db 0x%pad to mbox %d\n", &ch->props.doorbell,
-			ch->props.uc_mbox_n);
-
-	ev->props.id = mhi_dev->ul_event_id;
-	ev->props.doorbell = imp_ctx->dev_info.erdb_base + ev->props.id * 8;
-	ev->props.uc_mbox_n = IMP_IPA_UC_UL_EV_n;
-	IMP_DBG("allocated ev %d\n", ev->props.id);
-
-	ret = ipa3_uc_send_remote_ipa_info(ev->props.doorbell,
-		ev->props.uc_mbox_n);
-	if (ret)
-		goto fail;
-	IMP_DBG("mapped ch db 0x%pad to mbox %d\n", &ev->props.doorbell,
-		ev->props.uc_mbox_n);
-
-	/* store DL channel properties */
-	ch = &imp_ctx->md.dl_chan;
-	ev = &imp_ctx->md.dl_chan.event;
-
-	ch->props.dir = DMA_FROM_DEVICE;
-	ch->props.id = mhi_dev->dl_chan_id;
-	ch->props.doorbell = imp_ctx->dev_info.chdb_base + ch->props.id * 8;
-	ch->props.uc_mbox_n = IMP_IPA_UC_DL_CH_n;
-	IMP_DBG("dl ch id %d doorbell 0x%pa uc_mbox_n %d\n",
-		ch->props.id, &ch->props.doorbell, ch->props.uc_mbox_n);
-
-	ret = ipa3_uc_send_remote_ipa_info(ch->props.doorbell,
-		ch->props.uc_mbox_n);
-	if (ret)
-		goto fail;
-	IMP_DBG("mapped ch db 0x%pad to mbox %d\n", &ch->props.doorbell,
-		ch->props.uc_mbox_n);
-
-	ev->props.id = mhi_dev->dl_event_id;
-	ev->props.doorbell = imp_ctx->dev_info.erdb_base + ev->props.id * 8;
-	ev->props.uc_mbox_n = IMP_IPA_UC_DL_EV_n;
-	IMP_DBG("allocated ev %d\n", ev->props.id);
-
-	ret = ipa3_uc_send_remote_ipa_info(ev->props.doorbell,
-		ev->props.uc_mbox_n);
-	if (ret)
-		goto fail;
-	IMP_DBG("mapped ch db 0x%pad to mbox %d\n", &ev->props.doorbell,
-		ev->props.uc_mbox_n);
-
-	/*
-	 * Map MHI doorbells to IPA uC SMMU.
-	 * Both channel and event doorbells resides in a single page.
-	 */
-	if (!ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_UC]) {
-		struct ipa_smmu_cb_ctx *cb =
-			ipa3_get_smmu_ctx(IPA_SMMU_CB_UC);
-		unsigned long iova_p;
-		phys_addr_t pa_p;
-		u32 size_p;
-
-		imp_smmu_round_to_page(imp_ctx->dev_info.chdb_base,
-			imp_ctx->dev_info.chdb_base, PAGE_SIZE,
-			&iova_p, &pa_p, &size_p);
-
-		ret = ipa3_iommu_map(cb->iommu_domain, iova_p, pa_p, size_p,
-			IOMMU_READ | IOMMU_WRITE | IOMMU_MMIO);
-		if (ret)
-			goto fail;
-	}
-
-	imp_mhi_trigger_ready_ind();
-
-	mutex_unlock(&imp_ctx->mutex);
-
-	IMP_FUNC_EXIT();
-	return 0;
-
-fail:
-	mutex_unlock(&imp_ctx->mutex);
-	IPA_ACTIVE_CLIENTS_DEC_SPECIAL("IMP");
-	return ret;
-}
-
-static void imp_mhi_remove_cb(struct mhi_device *mhi_dev)
-{
-	IMP_FUNC_ENTRY();
-
-	mutex_lock(&imp_ctx->mutex);
-	imp_mhi_shutdown();
-	mutex_unlock(&imp_ctx->mutex);
-	IMP_FUNC_EXIT();
-}
-
-static void imp_mhi_status_cb(struct mhi_device *mhi_dev, enum MHI_CB mhi_cb)
-{
-	IMP_DBG("%d\n", mhi_cb);
-
-	mutex_lock(&imp_ctx->lpm_mutex);
-	if (mhi_dev != imp_ctx->md.mhi_dev) {
-		IMP_DBG("ignoring secondary callbacks\n");
-		mutex_unlock(&imp_ctx->lpm_mutex);
-		return;
-	}
-
-	switch (mhi_cb) {
-	case MHI_CB_IDLE:
-		break;
-	case MHI_CB_LPM_ENTER:
-		if (imp_ctx->state == IMP_STARTED) {
-			if (!imp_ctx->in_lpm) {
-				IPA_ACTIVE_CLIENTS_DEC_SPECIAL("IMP");
-				imp_ctx->in_lpm = true;
-			} else {
-				IMP_ERR("already in LPM\n");
-			}
-		}
-		break;
-	case MHI_CB_LPM_EXIT:
-		if (imp_ctx->state == IMP_STARTED) {
-			if (imp_ctx->in_lpm) {
-				IPA_ACTIVE_CLIENTS_INC_SPECIAL("IMP");
-				imp_ctx->in_lpm = false;
-			} else {
-				IMP_ERR("not in LPM\n");
-			}
-		}
-		break;
-
-	case MHI_CB_EE_RDDM:
-	case MHI_CB_PENDING_DATA:
-	default:
-		IMP_ERR("unexpected event %d\n", mhi_cb);
-		break;
-	}
-	mutex_unlock(&imp_ctx->lpm_mutex);
-}
-
-static int imp_probe(struct platform_device *pdev)
-{
-	int ret;
-
-	IMP_FUNC_ENTRY();
-
-	if (ipa3_uc_state_check()) {
-		IMP_DBG("uC not ready yet\n");
-		return -EPROBE_DEFER;
-	}
-
-	imp_ctx->dev_info.pdev = pdev;
-	imp_ctx->dev_info.smmu_enabled = true;
-	ret = imp_read_iova_from_dtsi("qcom,ctrl-iova",
-		&imp_ctx->dev_info.ctrl);
-	if (ret)
-		imp_ctx->dev_info.smmu_enabled = false;
-
-	ret = imp_read_iova_from_dtsi("qcom,data-iova",
-		&imp_ctx->dev_info.data);
-	if (ret)
-		imp_ctx->dev_info.smmu_enabled = false;
-
-	IMP_DBG("smmu_enabled=%d\n", imp_ctx->dev_info.smmu_enabled);
-
-	if (of_property_read_u32(pdev->dev.of_node, "qcom,mhi-chdb-base",
-		&imp_ctx->dev_info.chdb_base)) {
-		IMP_ERR("failed to read of_node %s\n", "qcom,mhi-chdb-base");
-		return -EINVAL;
-	}
-	IMP_DBG("chdb-base=0x%x\n", imp_ctx->dev_info.chdb_base);
-
-	if (of_property_read_u32(pdev->dev.of_node, "qcom,mhi-erdb-base",
-		&imp_ctx->dev_info.erdb_base)) {
-		IMP_ERR("failed to read of_node %s\n", "qcom,mhi-erdb-base");
-		return -EINVAL;
-	}
-	IMP_DBG("erdb-base=0x%x\n", imp_ctx->dev_info.erdb_base);
-
-	imp_ctx->state = IMP_PROBED;
-	ret = mhi_driver_register(&mhi_driver);
-	if (ret) {
-		IMP_ERR("mhi_driver_register failed %d\n", ret);
-		mutex_unlock(&imp_ctx->mutex);
-		return ret;
-	}
-
-	IMP_FUNC_EXIT();
-	return 0;
-}
-
-static int imp_remove(struct platform_device *pdev)
-{
-	IMP_FUNC_ENTRY();
-	mhi_driver_unregister(&mhi_driver);
-	mutex_lock(&imp_ctx->mutex);
-	if (!imp_ctx->in_lpm && (imp_ctx->state == IMP_READY ||
-		imp_ctx->state == IMP_STARTED)) {
-		IMP_DBG("devote IMP with state= %d\n", imp_ctx->state);
-		IPA_ACTIVE_CLIENTS_DEC_SPECIAL("IMP");
-	}
-	imp_ctx->lpm_disabled = false;
-	imp_ctx->state = IMP_INVALID;
-	mutex_unlock(&imp_ctx->mutex);
-
-	mutex_lock(&imp_ctx->lpm_mutex);
-	imp_ctx->in_lpm = false;
-	mutex_unlock(&imp_ctx->lpm_mutex);
-
-	return 0;
-}
-
-static const struct of_device_id imp_dt_match[] = {
-	{ .compatible = "qcom,ipa-mhi-proxy" },
-	{},
-};
-MODULE_DEVICE_TABLE(of, imp_dt_match);
-
-static struct platform_driver ipa_mhi_proxy_driver = {
-	.driver = {
-		.name = "ipa_mhi_proxy",
-		.of_match_table = imp_dt_match,
-	},
-	.probe = imp_probe,
-	.remove = imp_remove,
-};
-
-/**
- * imp_handle_modem_ready() - Registers IMP as a platform device
- *
- * This function is called after modem is loaded and QMI handshake is done.
- * IMP will register itself as a platform device, and on support device the
- * probe function will get called.
- *
- * Return: None
- */
-void imp_handle_modem_ready(void)
-{
-
-	if (!imp_ctx) {
-		imp_ctx = kzalloc(sizeof(*imp_ctx), GFP_KERNEL);
-		if (!imp_ctx)
-			return;
-
-		mutex_init(&imp_ctx->mutex);
-		mutex_init(&imp_ctx->lpm_mutex);
-	}
-
-	if (imp_ctx->state != IMP_INVALID) {
-		IMP_ERR("unexpected state %d\n", imp_ctx->state);
-		return;
-	}
-
-	IMP_DBG("register platform device\n");
-	platform_driver_register(&ipa_mhi_proxy_driver);
-}
-
-/**
- * imp_handle_modem_shutdown() - Handles modem SSR
- *
- * Performs MHI cleanup when modem is going to SSR (Subsystem Restart).
- *
- * Return: None
- */
-void imp_handle_modem_shutdown(void)
-{
-	IMP_FUNC_ENTRY();
-
-	if (!imp_ctx)
-		return;
-
-	mutex_lock(&imp_ctx->mutex);
-
-	if (imp_ctx->state == IMP_INVALID) {
-		mutex_unlock(&imp_ctx->mutex);
-		return;
-	}
-	if (imp_ctx->state == IMP_STARTED) {
-		mhi_unprepare_from_transfer(imp_ctx->md.mhi_dev);
-		imp_ctx->state = IMP_READY;
-	}
-
-	if (imp_ctx->state == IMP_READY) {
-		if (imp_ctx->dev_info.smmu_enabled) {
-			struct ipa_mhi_alloc_channel_req_msg_v01 *creq
-				= &imp_ctx->qmi.alloc_ch_req;
-
-			/* unmap CTRL */
-			__map_smmu_info(imp_ctx->md.mhi_dev->dev.parent,
-				&imp_ctx->dev_info.ctrl,
-				creq->ctrl_addr_map_info_len,
-				creq->ctrl_addr_map_info,
-				false);
-
-			/* unmap DATA */
-			__map_smmu_info(imp_ctx->md.mhi_dev->dev.parent,
-				&imp_ctx->dev_info.data,
-				creq->data_addr_map_info_len,
-				creq->data_addr_map_info,
-				false);
-		}
-	}
-
-	mutex_unlock(&imp_ctx->mutex);
-
-	IMP_FUNC_EXIT();
-
-	platform_driver_unregister(&ipa_mhi_proxy_driver);
-}
-
-MODULE_LICENSE("GPL v2");
-MODULE_DESCRIPTION("IPA MHI Proxy Driver");

ipa/ipa_v3/ipa_mhi_proxy.h

@@ -1,49 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2018, The Linux Foundation. All rights reserved.
- */
-
-#ifndef __IMP_H_
-#define __IMP_H_
-
-#ifdef CONFIG_IPA3_MHI_PROXY
-
-#include "ipa_qmi_service.h"
-
-void imp_handle_modem_ready(void);
-
-struct ipa_mhi_alloc_channel_resp_msg_v01 *imp_handle_allocate_channel_req(
-	struct ipa_mhi_alloc_channel_req_msg_v01 *req);
-
-struct ipa_mhi_clk_vote_resp_msg_v01 *imp_handle_vote_req(bool vote);
-
-void imp_handle_modem_shutdown(void);
-
-#else /* CONFIG_IPA3_MHI_PROXY */
-
-static inline void imp_handle_modem_ready(void)
-{
-
-}
-
-static inline struct ipa_mhi_alloc_channel_resp_msg_v01
-	*imp_handle_allocate_channel_req(
-		struct ipa_mhi_alloc_channel_req_msg_v01 *req)
-{
-		return NULL;
-}
-
-static inline struct ipa_mhi_clk_vote_resp_msg_v01
-	*imp_handle_vote_req(bool vote)
-{
-	return NULL;
-}
-
-static inline  void imp_handle_modem_shutdown(void)
-{
-
-}
-
-#endif /* CONFIG_IPA3_MHI_PROXY */
-
-#endif /* __IMP_H_ */

ipa/ipa_v3/ipa_mpm.c

@@ -1,2854 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2019, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/dma-mapping.h>
-#include <linux/list.h>
-#include <linux/slab.h>
-#include <linux/device.h>
-#include <linux/module.h>
-#include <linux/mhi.h>
-#include <linux/msm_gsi.h>
-#include <linux/delay.h>
-#include <linux/log2.h>
-#include "../ipa_common_i.h"
-#include "ipa_i.h"
-
-#define IPA_MPM_DRV_NAME "ipa_mpm"
-
-#define IPA_MPM_DBG(fmt, args...) \
-	do { \
-		pr_debug(IPA_MPM_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-			IPA_MPM_DRV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			IPA_MPM_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-#define IPA_MPM_DBG_LOW(fmt, args...) \
-	do { \
-		pr_debug(IPA_MPM_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			IPA_MPM_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-
-#define IPA_MPM_ERR(fmt, args...) \
-	do { \
-		pr_err(IPA_MPM_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-				IPA_MPM_DRV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-				IPA_MPM_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-
-#define IPA_MPM_FUNC_ENTRY() \
-	IPA_MPM_DBG("ENTRY\n")
-#define IPA_MPM_FUNC_EXIT() \
-	IPA_MPM_DBG("EXIT\n")
-
-#define IPA_MPM_MAX_MHIP_CHAN 3
-
-#define IPA_MPM_NUM_RING_DESC 0x400
-#define IPA_MPM_RING_LEN (IPA_MPM_NUM_RING_DESC - 10)
-
-#define IPA_MPM_MHI_HOST_UL_CHANNEL 4
-#define IPA_MPM_MHI_HOST_DL_CHANNEL  5
-#define TETH_AGGR_TIME_LIMIT 10000 /* 10ms */
-#define TETH_AGGR_BYTE_LIMIT 24
-#define TETH_AGGR_DL_BYTE_LIMIT 16
-#define TRE_BUFF_SIZE 32768
-#define IPA_HOLB_TMR_EN 0x1
-#define IPA_HOLB_TMR_DIS 0x0
-#define RNDIS_IPA_DFLT_RT_HDL 0
-#define IPA_POLL_FOR_EMPTINESS_NUM 50
-#define IPA_POLL_FOR_EMPTINESS_SLEEP_USEC 20
-#define IPA_CHANNEL_STOP_IN_PROC_TO_MSEC 5
-#define IPA_CHANNEL_STOP_IN_PROC_SLEEP_USEC 200
-
-enum mhip_re_type {
-	MHIP_RE_XFER = 0x2,
-	MHIP_RE_NOP = 0x4,
-};
-
-enum ipa_mpm_mhi_ch_id_type {
-	IPA_MPM_MHIP_CH_ID_0,
-	IPA_MPM_MHIP_CH_ID_1,
-	IPA_MPM_MHIP_CH_ID_2,
-	IPA_MPM_MHIP_CH_ID_MAX,
-};
-
-enum ipa_mpm_dma_data_direction {
-	DMA_HIPA_BIDIRECTIONAL = 0,
-	DMA_TO_HIPA = 1,
-	DMA_FROM_HIPA = 2,
-	DMA_HIPA_NONE = 3,
-};
-
-enum ipa_mpm_ipa_teth_client_type {
-	IPA_MPM_MHIP_USB,
-	IPA_MPM_MHIP_WIFI,
-};
-
-enum ipa_mpm_mhip_client_type {
-	IPA_MPM_MHIP_INIT,
-	/* USB RMNET CLIENT */
-	IPA_MPM_MHIP_USB_RMNET,
-	/* USB RNDIS / WIFI CLIENT */
-	IPA_MPM_MHIP_TETH,
-	/* USB DPL CLIENT */
-	IPA_MPM_MHIP_USB_DPL,
-	IPA_MPM_MHIP_NONE,
-};
-
-enum ipa_mpm_clk_vote_type {
-	CLK_ON,
-	CLK_OFF,
-};
-
-enum mhip_status_type {
-	MHIP_STATUS_SUCCESS,
-	MHIP_STATUS_NO_OP,
-	MHIP_STATUS_FAIL,
-	MHIP_STATUS_BAD_STATE,
-	MHIP_STATUS_EP_NOT_FOUND,
-	MHIP_STATUS_EP_NOT_READY,
-};
-
-enum mhip_smmu_domain_type {
-	MHIP_SMMU_DOMAIN_IPA,
-	MHIP_SMMU_DOMAIN_PCIE,
-	MHIP_SMMU_DOMAIN_NONE,
-};
-
-enum ipa_mpm_start_stop_type {
-	MPM_MHIP_STOP,
-	MPM_MHIP_START,
-};
-/* each pair of UL/DL channels are defined below */
-static const struct mhi_device_id mhi_driver_match_table[] = {
-	{ .chan = "IP_HW_MHIP_0" }, /* for rndis/Wifi teth pipes */
-	{ .chan = "IP_HW_MHIP_1" }, /* for MHIP rmnet */
-	{ .chan = "IP_HW_ADPL" }, /* ADPL/ODL DL pipe */
-};
-
-static const char *ipa_mpm_mhip_chan_str[IPA_MPM_MHIP_CH_ID_MAX] = {
-	__stringify(IPA_MPM_MHIP_TETH),
-	__stringify(IPA_MPM_MHIP_USB_RMNET),
-	__stringify(IPA_MPM_MHIP_USB_DPL),
-};
-/*
- * MHI PRIME GSI Descriptor format that Host IPA uses.
- */
-struct __packed mhi_p_desc {
-	uint64_t buffer_ptr;
-	uint16_t buff_len;
-	uint16_t resvd1;
-	uint16_t chain : 1;
-	uint16_t resvd4 : 7;
-	uint16_t ieob : 1;
-	uint16_t ieot : 1;
-	uint16_t bei : 1;
-	uint16_t sct : 1;
-	uint16_t resvd3 : 4;
-	uint8_t re_type;
-	uint8_t resvd2;
-};
-
-/*
- * MHI PRIME Channel Context and Event Context Array
- * Information that is sent to Device IPA.
- */
-struct ipa_mpm_channel_context_type {
-	u32 chstate : 8;
-	u32 reserved1 : 24;
-	u32 chtype;
-	u32 erindex;
-	u64 rbase;
-	u64 rlen;
-	u64 reserved2;
-	u64 reserved3;
-} __packed;
-
-struct ipa_mpm_event_context_type {
-	u32 reserved1 : 8;
-	u32 update_rp_modc : 8;
-	u32 update_rp_intmodt : 16;
-	u32 ertype;
-	u32 update_rp_addr;
-	u64 rbase;
-	u64 rlen;
-	u32 buff_size : 16;
-	u32 reserved2 : 16;
-	u32 reserved3;
-	u64 reserved4;
-} __packed;
-
-struct ipa_mpm_pipes_info_type {
-	enum ipa_client_type ipa_client;
-	struct ipa_ep_cfg ep_cfg;
-};
-
-struct ipa_mpm_channel_type {
-	struct ipa_mpm_pipes_info_type dl_cons;
-	struct ipa_mpm_pipes_info_type ul_prod;
-	enum ipa_mpm_mhip_client_type mhip_client;
-};
-
-static struct ipa_mpm_channel_type ipa_mpm_pipes[IPA_MPM_MHIP_CH_ID_MAX];
-
-/* For configuring IPA_CLIENT_MHI_PRIME_TETH_CONS */
-static struct ipa_ep_cfg mhip_dl_teth_ep_cfg = {
-	.mode = {
-		.mode = IPA_BASIC,
-		.dst = IPA_CLIENT_MHI_PRIME_TETH_CONS,
-	},
-	.hdr = {
-		.hdr_len = 4,
-		.hdr_ofst_metadata_valid = 1,
-		.hdr_ofst_metadata = 1,
-		.hdr_ofst_pkt_size_valid = 1,
-		.hdr_ofst_pkt_size = 2,
-	},
-	.hdr_ext = {
-		.hdr_total_len_or_pad_valid = true,
-		.hdr_payload_len_inc_padding = true,
-	},
-	.aggr = {
-		.aggr_en = IPA_ENABLE_DEAGGR,
-		.aggr = IPA_QCMAP,
-		.aggr_byte_limit = TETH_AGGR_DL_BYTE_LIMIT,
-		.aggr_time_limit = TETH_AGGR_TIME_LIMIT,
-	},
-};
-
-static struct ipa_ep_cfg mhip_ul_teth_ep_cfg = {
-	.mode = {
-		.mode = IPA_BASIC,
-		.dst = IPA_CLIENT_MHI_PRIME_TETH_PROD,
-	},
-	.hdr = {
-		.hdr_len = 4,
-		.hdr_ofst_metadata_valid = 1,
-		.hdr_ofst_metadata = 0,
-		.hdr_ofst_pkt_size_valid = 1,
-		.hdr_ofst_pkt_size = 2,
-	},
-	.hdr_ext = {
-		.hdr_total_len_or_pad_valid = true,
-		.hdr_payload_len_inc_padding = true,
-	},
-	.aggr = {
-		.aggr_en = IPA_ENABLE_AGGR,
-		.aggr = IPA_QCMAP,
-		.aggr_byte_limit = TETH_AGGR_BYTE_LIMIT,
-		.aggr_time_limit = TETH_AGGR_TIME_LIMIT,
-	},
-
-};
-
-/* WARNING!! Temporary for rndis intgration only */
-
-
-/* For configuring IPA_CLIENT_MHIP_RMNET_PROD */
-static struct ipa_ep_cfg mhip_dl_rmnet_ep_cfg = {
-	.mode = {
-		.mode = IPA_DMA,
-		.dst = IPA_CLIENT_USB_CONS,
-	},
-};
-
-/* For configuring IPA_CLIENT_MHIP_RMNET_CONS */
-static struct ipa_ep_cfg mhip_ul_rmnet_ep_cfg = {
-	.mode = {
-		.mode = IPA_DMA,
-		.dst = IPA_CLIENT_USB_CONS,
-	},
-};
-
-/* For configuring IPA_CLIENT_MHIP_DPL_PROD */
-static struct ipa_ep_cfg mhip_dl_dpl_ep_cfg = {
-	.mode = {
-		.mode = IPA_DMA,
-		.dst = IPA_CLIENT_USB_DPL_CONS,
-	},
-};
-
-
-struct ipa_mpm_iova_addr {
-	dma_addr_t base;
-	unsigned int size;
-};
-
-struct ipa_mpm_dev_info {
-	struct platform_device *pdev;
-	struct device *dev;
-	bool ipa_smmu_enabled;
-	bool pcie_smmu_enabled;
-	struct ipa_mpm_iova_addr ctrl;
-	struct ipa_mpm_iova_addr data;
-	u32 chdb_base;
-	u32 erdb_base;
-	bool is_cache_coherent;
-};
-
-struct ipa_mpm_event_props {
-	u16 id;
-	phys_addr_t device_db;
-	struct ipa_mpm_event_context_type ev_ctx;
-};
-
-struct ipa_mpm_channel_props {
-	u16 id;
-	phys_addr_t device_db;
-	struct ipa_mpm_channel_context_type ch_ctx;
-};
-
-enum ipa_mpm_gsi_state {
-	GSI_ERR,
-	GSI_INIT,
-	GSI_ALLOCATED,
-	GSI_STARTED,
-	GSI_STOPPED,
-};
-
-struct ipa_mpm_channel {
-	struct ipa_mpm_channel_props chan_props;
-	struct ipa_mpm_event_props evt_props;
-	enum ipa_mpm_gsi_state gsi_state;
-	dma_addr_t db_host_iova;
-	dma_addr_t db_device_iova;
-};
-
-enum ipa_mpm_teth_state {
-	IPA_MPM_TETH_INIT = 0,
-	IPA_MPM_TETH_INPROGRESS,
-	IPA_MPM_TETH_CONNECTED,
-};
-
-enum ipa_mpm_mhip_chan {
-	IPA_MPM_MHIP_CHAN_UL,
-	IPA_MPM_MHIP_CHAN_DL,
-	IPA_MPM_MHIP_CHAN_BOTH,
-};
-
-struct ipa_mpm_clk_cnt_type {
-	atomic_t pcie_clk_cnt;
-	atomic_t ipa_clk_cnt;
-};
-
-struct producer_rings {
-	struct mhi_p_desc *tr_va;
-	struct mhi_p_desc *er_va;
-	void *tre_buff_va[IPA_MPM_RING_LEN];
-	dma_addr_t tr_pa;
-	dma_addr_t er_pa;
-	dma_addr_t tre_buff_iova[IPA_MPM_RING_LEN];
-	/*
-	 * The iova generated for AP CB,
-	 * used only for dma_map_single to flush the cache.
-	 */
-	dma_addr_t ap_iova_er;
-	dma_addr_t ap_iova_tr;
-	dma_addr_t ap_iova_buff[IPA_MPM_RING_LEN];
-};
-
-struct ipa_mpm_mhi_driver {
-	struct mhi_device *mhi_dev;
-	struct producer_rings ul_prod_ring;
-	struct producer_rings dl_prod_ring;
-	struct ipa_mpm_channel ul_prod;
-	struct ipa_mpm_channel dl_cons;
-	enum ipa_mpm_mhip_client_type mhip_client;
-	enum ipa_mpm_teth_state teth_state;
-	struct mutex mutex;
-	bool init_complete;
-	struct mutex lpm_mutex;
-	bool in_lpm;
-	struct ipa_mpm_clk_cnt_type clk_cnt;
-};
-
-struct ipa_mpm_context {
-	struct ipa_mpm_dev_info dev_info;
-	struct ipa_mpm_mhi_driver md[IPA_MPM_MAX_MHIP_CHAN];
-	struct mutex mutex;
-	atomic_t probe_cnt;
-	atomic_t pcie_clk_total_cnt;
-	atomic_t ipa_clk_total_cnt;
-	struct device *parent_pdev;
-	struct ipa_smmu_cb_ctx carved_smmu_cb;
-	struct device *mhi_parent_dev;
-};
-
-#define IPA_MPM_DESC_SIZE (sizeof(struct mhi_p_desc))
-#define IPA_MPM_RING_TOTAL_SIZE (IPA_MPM_RING_LEN * IPA_MPM_DESC_SIZE)
-/* WA: Make the IPA_MPM_PAGE_SIZE from 16k (next power of ring size) to
- * 32k. This is to make sure IOMMU map happens for the same size
- * for all TR/ER and doorbells.
- */
-#define IPA_MPM_PAGE_SIZE TRE_BUFF_SIZE
-
-
-static struct ipa_mpm_context *ipa_mpm_ctx;
-static struct platform_device *m_pdev;
-static int ipa_mpm_mhi_probe_cb(struct mhi_device *,
-	const struct mhi_device_id *);
-static void ipa_mpm_mhi_remove_cb(struct mhi_device *);
-static void ipa_mpm_mhi_status_cb(struct mhi_device *, enum MHI_CB);
-static void ipa_mpm_change_teth_state(int probe_id,
-	enum ipa_mpm_teth_state ip_state);
-static void ipa_mpm_change_gsi_state(int probe_id,
-	enum ipa_mpm_mhip_chan mhip_chan,
-	enum ipa_mpm_gsi_state next_state);
-static int ipa_mpm_start_stop_ul_mhip_data_path(int probe_id,
-	enum ipa_mpm_start_stop_type start);
-static int ipa_mpm_probe(struct platform_device *pdev);
-static int ipa_mpm_vote_unvote_pcie_clk(enum ipa_mpm_clk_vote_type vote,
-	int probe_id);
-static void ipa_mpm_vote_unvote_ipa_clk(enum ipa_mpm_clk_vote_type vote,
-	int probe_id);
-static enum mhip_status_type ipa_mpm_start_stop_mhip_chan(
-	enum ipa_mpm_mhip_chan mhip_chan,
-	int probe_id,
-	enum ipa_mpm_start_stop_type start_stop);
-
-static struct mhi_driver mhi_driver = {
-	.id_table = mhi_driver_match_table,
-	.probe = ipa_mpm_mhi_probe_cb,
-	.remove = ipa_mpm_mhi_remove_cb,
-	.status_cb = ipa_mpm_mhi_status_cb,
-	.driver = {
-		.name = IPA_MPM_DRV_NAME,
-		.owner = THIS_MODULE,
-	},
-};
-
-static void ipa_mpm_ipa3_delayed_probe(struct work_struct *work)
-{
-	(void)ipa_mpm_probe(m_pdev);
-}
-
-static DECLARE_WORK(ipa_mpm_ipa3_scheduled_probe, ipa_mpm_ipa3_delayed_probe);
-
-static void ipa_mpm_ipa3_ready_cb(void *user_data)
-{
-	struct platform_device *pdev = (struct platform_device *)(user_data);
-
-	m_pdev = pdev;
-
-	IPA_MPM_DBG("IPA ready callback has been triggered\n");
-
-	schedule_work(&ipa_mpm_ipa3_scheduled_probe);
-}
-
-static void ipa_mpm_gsi_evt_ring_err_cb(struct gsi_evt_err_notify *err_data)
-{
-	IPA_MPM_ERR("GSI EVT RING ERROR, not expected..\n");
-	ipa_assert();
-}
-
-static void ipa_mpm_gsi_chan_err_cb(struct gsi_chan_err_notify *err_data)
-{
-	IPA_MPM_ERR("GSI CHAN ERROR, not expected..\n");
-	ipa_assert();
-}
-
-/**
- * ipa_mpm_smmu_map() - SMMU maps ring and the buffer pointer.
- * @va_addr: virtual address that needs to be mapped
- * @sz: size of the address to be mapped
- * @dir: ipa_mpm_dma_data_direction
- * @ap_cb_iova: iova for AP context bank
- *
- * This function SMMU maps both ring and the buffer pointer.
- * The ring pointers will be aligned to ring size and
- * the buffer pointers should be aligned to buffer size.
- *
- * Returns: iova of the mapped address
- */
-static dma_addr_t ipa_mpm_smmu_map(void *va_addr,
-	int sz,
-	int dir,
-	dma_addr_t *ap_cb_iova)
-{
-	struct iommu_domain *ipa_smmu_domain, *pcie_smmu_domain;
-	phys_addr_t phys_addr;
-	dma_addr_t iova;
-	int smmu_enabled;
-	unsigned long iova_p;
-	phys_addr_t pa_p;
-	u32 size_p;
-	int prot = IOMMU_READ | IOMMU_WRITE;
-	struct ipa_smmu_cb_ctx *cb = &ipa_mpm_ctx->carved_smmu_cb;
-	unsigned long carved_iova = roundup(cb->next_addr, IPA_MPM_PAGE_SIZE);
-	int ret = 0;
-
-	/* check cache coherent */
-	if (ipa_mpm_ctx->dev_info.is_cache_coherent)  {
-		IPA_MPM_DBG(" enable cache coherent\n");
-		prot |= IOMMU_CACHE;
-	}
-
-	if (carved_iova >= cb->va_end) {
-		IPA_MPM_ERR("running out of carved_iova %lx\n", carved_iova);
-		ipa_assert();
-	}
-	/*
-	 * Both Host IPA and PCIE SMMU should be enabled or disabled
-	 * for proceed.
-	 * If SMMU Enabled => iova == pa
-	 * If SMMU Disabled => iova == iommu mapped iova
-	 * dma_map_single ensures cache is flushed and the memory is not
-	 * touched again until dma_unmap_single() is called
-	 */
-	smmu_enabled = (ipa_mpm_ctx->dev_info.ipa_smmu_enabled &&
-		ipa_mpm_ctx->dev_info.pcie_smmu_enabled) ? 1 : 0;
-
-	if (smmu_enabled) {
-		/* Map the phys addr to both PCIE and IPA AP CB
-		 * from the carved out common iova range.
-		 */
-		ipa_smmu_domain = ipa3_get_smmu_domain();
-
-		if (!ipa_smmu_domain) {
-			IPA_MPM_ERR("invalid IPA smmu domain\n");
-			ipa_assert();
-		}
-
-		if (!ipa_mpm_ctx->mhi_parent_dev) {
-			IPA_MPM_ERR("invalid PCIE SMMU domain\n");
-			ipa_assert();
-		}
-
-		phys_addr = virt_to_phys((void *) va_addr);
-		IPA_SMMU_ROUND_TO_PAGE(carved_iova, phys_addr, sz,
-					iova_p, pa_p, size_p);
-
-		/* Flush the cache with dma_map_single for IPA AP CB */
-		*ap_cb_iova = dma_map_single(ipa3_ctx->pdev, va_addr,
-						IPA_MPM_RING_TOTAL_SIZE, dir);
-		ret = ipa3_iommu_map(ipa_smmu_domain, iova_p,
-					pa_p, size_p, prot);
-		if (ret) {
-			IPA_MPM_ERR("IPA IOMMU returned failure, ret = %d\n",
-					ret);
-			ipa_assert();
-		}
-
-		pcie_smmu_domain = iommu_get_domain_for_dev(
-			ipa_mpm_ctx->mhi_parent_dev);
-		if (!pcie_smmu_domain) {
-			IPA_MPM_ERR("invalid pcie smmu domain\n");
-			ipa_assert();
-		}
-		ret = iommu_map(pcie_smmu_domain, iova_p, pa_p, size_p, prot);
-
-		if (ret) {
-			IPA_MPM_ERR("PCIe IOMMU returned failure, ret = %d\n",
-				ret);
-			ipa_assert();
-		}
-
-		iova = iova_p;
-		cb->next_addr = iova_p + size_p;
-	} else {
-		iova = dma_map_single(ipa3_ctx->pdev, va_addr,
-					IPA_MPM_RING_TOTAL_SIZE, dir);
-		*ap_cb_iova = iova;
-	}
-	return iova;
-}
-
-/**
- * ipa_mpm_smmu_unmap() - SMMU unmaps ring and the buffer pointer.
- * @va_addr: virtual address that needs to be mapped
- * @sz: size of the address to be mapped
- * @dir: ipa_mpm_dma_data_direction
- * @ap_cb_iova: iova for AP context bank
- *
- * This function SMMU unmaps both ring and the buffer pointer.
- * The ring pointers will be aligned to ring size and
- * the buffer pointers should be aligned to buffer size.
- *
- * Return: none
- */
-static void ipa_mpm_smmu_unmap(dma_addr_t carved_iova, int sz, int dir,
-	dma_addr_t ap_cb_iova)
-{
-	unsigned long iova_p;
-	unsigned long pa_p;
-	u32 size_p = 0;
-	struct iommu_domain *ipa_smmu_domain, *pcie_smmu_domain;
-	struct ipa_smmu_cb_ctx *cb = &ipa_mpm_ctx->carved_smmu_cb;
-	int smmu_enabled = (ipa_mpm_ctx->dev_info.ipa_smmu_enabled &&
-		ipa_mpm_ctx->dev_info.pcie_smmu_enabled) ? 1 : 0;
-
-	if (carved_iova <= 0) {
-		IPA_MPM_ERR("carved_iova is zero/negative\n");
-		WARN_ON(1);
-		return;
-	}
-
-	if (smmu_enabled) {
-		ipa_smmu_domain = ipa3_get_smmu_domain();
-		if (!ipa_smmu_domain) {
-			IPA_MPM_ERR("invalid IPA smmu domain\n");
-			ipa_assert();
-		}
-
-		if (!ipa_mpm_ctx->mhi_parent_dev) {
-			IPA_MPM_ERR("invalid PCIE SMMU domain\n");
-			ipa_assert();
-		}
-
-		IPA_SMMU_ROUND_TO_PAGE(carved_iova, carved_iova, sz,
-			iova_p, pa_p, size_p);
-		pcie_smmu_domain = iommu_get_domain_for_dev(
-			ipa_mpm_ctx->mhi_parent_dev);
-		if (pcie_smmu_domain) {
-			iommu_unmap(pcie_smmu_domain, iova_p, size_p);
-		} else {
-			IPA_MPM_ERR("invalid PCIE SMMU domain\n");
-			ipa_assert();
-		}
-		iommu_unmap(ipa_smmu_domain, iova_p, size_p);
-
-		cb->next_addr -= size_p;
-		dma_unmap_single(ipa3_ctx->pdev, ap_cb_iova,
-			IPA_MPM_RING_TOTAL_SIZE, dir);
-	} else {
-		dma_unmap_single(ipa3_ctx->pdev, ap_cb_iova,
-			IPA_MPM_RING_TOTAL_SIZE, dir);
-	}
-}
-
-static u32 ipa_mpm_smmu_map_doorbell(enum mhip_smmu_domain_type smmu_domain,
-	u32 pa_addr)
-{
-	/*
-	 * Doorbells are already in PA, map these to
-	 * PCIE/IPA doman if SMMUs are enabled.
-	 */
-	struct iommu_domain *ipa_smmu_domain, *pcie_smmu_domain;
-	int smmu_enabled;
-	unsigned long iova_p;
-	phys_addr_t pa_p;
-	u32 size_p;
-	int ret = 0;
-	int prot = IOMMU_READ | IOMMU_WRITE;
-	struct ipa_smmu_cb_ctx *cb = &ipa_mpm_ctx->carved_smmu_cb;
-	unsigned long carved_iova = roundup(cb->next_addr, IPA_MPM_PAGE_SIZE);
-	u32 iova = 0;
-	u64 offset = 0;
-
-	/* check cache coherent */
-	if (ipa_mpm_ctx->dev_info.is_cache_coherent)  {
-		IPA_MPM_DBG(" enable cache coherent\n");
-		prot |= IOMMU_CACHE;
-	}
-
-	if (carved_iova >= cb->va_end) {
-		IPA_MPM_ERR("running out of carved_iova %lx\n", carved_iova);
-		ipa_assert();
-	}
-
-	smmu_enabled = (ipa_mpm_ctx->dev_info.ipa_smmu_enabled &&
-		ipa_mpm_ctx->dev_info.pcie_smmu_enabled) ? 1 : 0;
-
-	if (smmu_enabled) {
-		IPA_SMMU_ROUND_TO_PAGE(carved_iova, pa_addr, IPA_MPM_PAGE_SIZE,
-					iova_p, pa_p, size_p);
-		if (smmu_domain == MHIP_SMMU_DOMAIN_IPA) {
-			ipa_smmu_domain = ipa3_get_smmu_domain();
-			if (!ipa_smmu_domain) {
-				IPA_MPM_ERR("invalid IPA smmu domain\n");
-				ipa_assert();
-			}
-			ret = ipa3_iommu_map(ipa_smmu_domain,
-				iova_p, pa_p, size_p, prot);
-			if (ret) {
-				IPA_MPM_ERR("IPA doorbell mapping failed\n");
-				ipa_assert();
-			}
-			offset = pa_addr - pa_p;
-		} else if (smmu_domain == MHIP_SMMU_DOMAIN_PCIE) {
-			pcie_smmu_domain = iommu_get_domain_for_dev(
-				ipa_mpm_ctx->mhi_parent_dev);
-			if (!pcie_smmu_domain) {
-				IPA_MPM_ERR("invalid IPA smmu domain\n");
-				ipa_assert();
-			}
-			ret = iommu_map(pcie_smmu_domain,
-					iova_p, pa_p, size_p, prot);
-			if (ret) {
-				IPA_MPM_ERR("PCIe doorbell mapping failed\n");
-				ipa_assert();
-			}
-			offset = pa_addr - pa_p;
-		}
-		iova = iova_p + offset;
-		cb->next_addr = iova_p + IPA_MPM_PAGE_SIZE;
-	} else {
-		iova = pa_addr;
-	}
-	return iova;
-}
-
-static void ipa_mpm_smmu_unmap_doorbell(enum mhip_smmu_domain_type smmu_domain,
-	dma_addr_t iova)
-{
-	/*
-	 * Doorbells are already in PA, map these to
-	 * PCIE/IPA doman if SMMUs are enabled.
-	 */
-	struct iommu_domain *ipa_smmu_domain, *pcie_smmu_domain;
-	int smmu_enabled;
-	unsigned long iova_p;
-	phys_addr_t pa_p;
-	u32 size_p;
-	struct ipa_smmu_cb_ctx *cb = &ipa_mpm_ctx->carved_smmu_cb;
-
-	smmu_enabled = (ipa_mpm_ctx->dev_info.ipa_smmu_enabled &&
-		ipa_mpm_ctx->dev_info.pcie_smmu_enabled) ? 1 : 0;
-
-	if (smmu_enabled) {
-		IPA_SMMU_ROUND_TO_PAGE(iova, iova, IPA_MPM_PAGE_SIZE,
-					iova_p, pa_p, size_p);
-		if (smmu_domain == MHIP_SMMU_DOMAIN_IPA) {
-			ipa_smmu_domain = ipa3_get_smmu_domain();
-			if (ipa_smmu_domain) {
-				iommu_unmap(ipa_smmu_domain, iova_p, size_p);
-			} else {
-				IPA_MPM_ERR("invalid IPA smmu domain\n");
-				ipa_assert();
-			}
-		} else if (smmu_domain == MHIP_SMMU_DOMAIN_PCIE) {
-			pcie_smmu_domain = iommu_get_domain_for_dev(
-				ipa_mpm_ctx->mhi_parent_dev);
-			if (pcie_smmu_domain) {
-				iommu_unmap(pcie_smmu_domain, iova_p, size_p);
-			} else {
-				IPA_MPM_ERR("invalid PCIE smmu domain\n");
-				ipa_assert();
-			}
-			cb->next_addr -=  IPA_MPM_PAGE_SIZE;
-		}
-	}
-}
-static int get_idx_from_id(const struct mhi_device_id *id)
-{
-	return (id - mhi_driver_match_table);
-}
-
-static void get_ipa3_client(int id,
-	enum ipa_client_type *ul_prod,
-	enum ipa_client_type *dl_cons)
-{
-	IPA_MPM_FUNC_ENTRY();
-
-	if (id >= IPA_MPM_MHIP_CH_ID_MAX) {
-		*ul_prod = IPA_CLIENT_MAX;
-		*dl_cons = IPA_CLIENT_MAX;
-	} else {
-		*ul_prod = ipa_mpm_pipes[id].ul_prod.ipa_client;
-		*dl_cons = ipa_mpm_pipes[id].dl_cons.ipa_client;
-	}
-	IPA_MPM_FUNC_EXIT();
-}
-
-static int ipa_mpm_connect_mhip_gsi_pipe(enum ipa_client_type mhip_client,
-	int mhi_idx, struct ipa_req_chan_out_params *out_params)
-{
-	int ipa_ep_idx;
-	int res;
-	struct mhi_p_desc *ev_ring;
-	struct mhi_p_desc *tr_ring;
-	int tr_ring_sz, ev_ring_sz;
-	dma_addr_t ev_ring_iova, tr_ring_iova;
-	dma_addr_t ap_cb_iova;
-	dma_addr_t ap_cb_er_iova;
-	struct ipa_request_gsi_channel_params gsi_params;
-	int dir;
-	int i;
-	void *buff;
-	int result;
-	int k;
-	struct ipa3_ep_context *ep;
-
-	if (mhip_client == IPA_CLIENT_MAX)
-		goto fail_gen;
-
-	if ((mhi_idx < IPA_MPM_MHIP_CH_ID_0) ||
-		(mhi_idx >= IPA_MPM_MHIP_CH_ID_MAX))
-		goto fail_gen;
-
-	ipa_ep_idx = ipa3_get_ep_mapping(mhip_client);
-	if (ipa_ep_idx == IPA_EP_NOT_ALLOCATED) {
-		IPA_MPM_ERR("fail to find channel EP.\n");
-		goto fail_gen;
-	}
-	ep = &ipa3_ctx->ep[ipa_ep_idx];
-	if (ep->valid == 1) {
-		IPAERR("EP %d already allocated.\n", ipa_ep_idx);
-		return 0;
-	}
-
-	IPA_MPM_DBG("connecting client %d (ep: %d)\n", mhip_client, ipa_ep_idx);
-
-	IPA_MPM_FUNC_ENTRY();
-
-	ev_ring_sz = IPA_MPM_RING_TOTAL_SIZE;
-	ev_ring = kzalloc(ev_ring_sz, GFP_KERNEL);
-	if (!ev_ring)
-		goto fail_evt_alloc;
-
-	tr_ring_sz = IPA_MPM_RING_TOTAL_SIZE;
-	tr_ring = kzalloc(tr_ring_sz, GFP_KERNEL);
-	if (!tr_ring)
-		goto fail_tr_alloc;
-
-	tr_ring[0].re_type = MHIP_RE_NOP;
-
-	dir = IPA_CLIENT_IS_PROD(mhip_client) ?
-		DMA_TO_HIPA : DMA_FROM_HIPA;
-
-	/* allocate transfer ring elements */
-	for (i = 1, k = 1; i < IPA_MPM_RING_LEN; i++, k++) {
-		buff = kzalloc(TRE_BUFF_SIZE, GFP_KERNEL);
-
-		if (!buff)
-			goto fail_buff_alloc;
-
-		tr_ring[i].buffer_ptr =
-			ipa_mpm_smmu_map(buff, TRE_BUFF_SIZE, dir,
-				&ap_cb_iova);
-		if (!tr_ring[i].buffer_ptr)
-			goto fail_smmu_map_ring;
-
-		if (IPA_CLIENT_IS_PROD(mhip_client)) {
-			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.tre_buff_va[k] =
-							buff;
-			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.tre_buff_iova[k] =
-							tr_ring[i].buffer_ptr;
-		} else {
-			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.tre_buff_va[k] =
-							buff;
-			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.tre_buff_iova[k] =
-							tr_ring[i].buffer_ptr;
-		}
-
-
-		tr_ring[i].buff_len = TRE_BUFF_SIZE;
-		tr_ring[i].chain = 0;
-		tr_ring[i].ieob = 0;
-		tr_ring[i].ieot = 0;
-		tr_ring[i].bei = 0;
-		tr_ring[i].sct = 0;
-		tr_ring[i].re_type = MHIP_RE_XFER;
-
-		if (IPA_CLIENT_IS_PROD(mhip_client))
-			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.ap_iova_buff[k] =
-				ap_cb_iova;
-		else
-			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.ap_iova_buff[k] =
-				ap_cb_iova;
-	}
-
-	tr_ring_iova = ipa_mpm_smmu_map(tr_ring, IPA_MPM_PAGE_SIZE, dir,
-		&ap_cb_iova);
-	if (!tr_ring_iova)
-		goto fail_smmu_map_ring;
-
-	ev_ring_iova = ipa_mpm_smmu_map(ev_ring, IPA_MPM_PAGE_SIZE, dir,
-		&ap_cb_er_iova);
-	if (!ev_ring_iova)
-		goto fail_smmu_map_ring;
-
-	/* Store Producer channel rings */
-	if (IPA_CLIENT_IS_PROD(mhip_client)) {
-		/* Device UL */
-		ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.er_va = ev_ring;
-		ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.tr_va = tr_ring;
-		ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.er_pa = ev_ring_iova;
-		ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.tr_pa = tr_ring_iova;
-		ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.ap_iova_tr =
-			ap_cb_iova;
-		ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.ap_iova_er =
-			ap_cb_er_iova;
-	} else {
-		/* Host UL */
-		ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.er_va = ev_ring;
-		ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.tr_va = tr_ring;
-		ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.er_pa = ev_ring_iova;
-		ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.tr_pa = tr_ring_iova;
-		ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.ap_iova_tr =
-			ap_cb_iova;
-		ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.ap_iova_er =
-			ap_cb_er_iova;
-	}
-
-	memset(&gsi_params, 0, sizeof(struct ipa_request_gsi_channel_params));
-
-	if (IPA_CLIENT_IS_PROD(mhip_client))
-		gsi_params.ipa_ep_cfg =
-		ipa_mpm_pipes[mhi_idx].dl_cons.ep_cfg;
-	else
-		gsi_params.ipa_ep_cfg =
-		ipa_mpm_pipes[mhi_idx].ul_prod.ep_cfg;
-
-	gsi_params.client = mhip_client;
-	gsi_params.skip_ep_cfg = false;
-
-	/*
-	 * RP update address = Device channel DB address
-	 * CLIENT_PROD -> Host DL
-	 * CLIENT_CONS -> Host UL
-	 */
-	if (IPA_CLIENT_IS_PROD(mhip_client)) {
-		gsi_params.evt_ring_params.rp_update_addr =
-			ipa_mpm_smmu_map_doorbell(
-			MHIP_SMMU_DOMAIN_IPA,
-			ipa_mpm_ctx->md[mhi_idx].dl_cons.chan_props.device_db);
-		if (gsi_params.evt_ring_params.rp_update_addr == 0)
-			goto fail_smmu_map_db;
-
-		ipa_mpm_ctx->md[mhi_idx].dl_cons.db_host_iova =
-			gsi_params.evt_ring_params.rp_update_addr;
-
-		gsi_params.evt_ring_params.ring_base_addr =
-			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.tr_pa;
-		gsi_params.chan_params.ring_base_addr =
-			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.er_pa;
-	} else {
-		gsi_params.evt_ring_params.rp_update_addr =
-			ipa_mpm_smmu_map_doorbell(
-			MHIP_SMMU_DOMAIN_IPA,
-			ipa_mpm_ctx->md[mhi_idx].ul_prod.chan_props.device_db);
-		if (gsi_params.evt_ring_params.rp_update_addr == 0)
-			goto fail_smmu_map_db;
-		ipa_mpm_ctx->md[mhi_idx].ul_prod.db_host_iova =
-			gsi_params.evt_ring_params.rp_update_addr;
-		gsi_params.evt_ring_params.ring_base_addr =
-			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.er_pa;
-		gsi_params.chan_params.ring_base_addr =
-			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.tr_pa;
-	}
-
-	/* Fill Event ring params */
-	gsi_params.evt_ring_params.intf = GSI_EVT_CHTYPE_MHIP_EV;
-	gsi_params.evt_ring_params.intr = GSI_INTR_MSI;
-	gsi_params.evt_ring_params.re_size = GSI_EVT_RING_RE_SIZE_16B;
-	gsi_params.evt_ring_params.ring_len =
-		(IPA_MPM_RING_LEN) * GSI_EVT_RING_RE_SIZE_16B;
-	gsi_params.evt_ring_params.ring_base_vaddr = NULL;
-	gsi_params.evt_ring_params.int_modt = 0;
-	gsi_params.evt_ring_params.int_modc = 0;
-	gsi_params.evt_ring_params.intvec = 0;
-	gsi_params.evt_ring_params.msi_addr = 0;
-	gsi_params.evt_ring_params.exclusive = true;
-	gsi_params.evt_ring_params.err_cb = ipa_mpm_gsi_evt_ring_err_cb;
-	gsi_params.evt_ring_params.user_data = NULL;
-
-	/* Evt Scratch Params */
-	/* Disable the Moderation for ringing doorbells */
-	gsi_params.evt_scratch.mhip.rp_mod_threshold = 1;
-	gsi_params.evt_scratch.mhip.rp_mod_timer = 0;
-	gsi_params.evt_scratch.mhip.rp_mod_counter = 0;
-	gsi_params.evt_scratch.mhip.rp_mod_timer_id = 0;
-	gsi_params.evt_scratch.mhip.rp_mod_timer_running = 0;
-	gsi_params.evt_scratch.mhip.fixed_buffer_sz = TRE_BUFF_SIZE;
-
-	if (IPA_CLIENT_IS_PROD(mhip_client))
-		gsi_params.evt_scratch.mhip.rp_mod_threshold = 4;
-
-	/* Channel Params */
-	gsi_params.chan_params.prot = GSI_CHAN_PROT_MHIP;
-	gsi_params.chan_params.dir = IPA_CLIENT_IS_PROD(mhip_client) ?
-		GSI_CHAN_DIR_TO_GSI : GSI_CHAN_DIR_FROM_GSI;
-	/* chan_id is set in ipa3_request_gsi_channel() */
-	gsi_params.chan_params.re_size = GSI_CHAN_RE_SIZE_16B;
-	gsi_params.chan_params.ring_len =
-		(IPA_MPM_RING_LEN) * GSI_EVT_RING_RE_SIZE_16B;
-	gsi_params.chan_params.ring_base_vaddr = NULL;
-	gsi_params.chan_params.use_db_eng = GSI_CHAN_DIRECT_MODE;
-	gsi_params.chan_params.max_prefetch = GSI_ONE_PREFETCH_SEG;
-	gsi_params.chan_params.low_weight = 1;
-	gsi_params.chan_params.xfer_cb = NULL;
-	gsi_params.chan_params.err_cb = ipa_mpm_gsi_chan_err_cb;
-	gsi_params.chan_params.chan_user_data = NULL;
-
-	/* Channel scratch */
-	gsi_params.chan_scratch.mhip.assert_bit_40 = 0;
-	gsi_params.chan_scratch.mhip.host_channel = 1;
-
-	res = ipa3_request_gsi_channel(&gsi_params, out_params);
-	if (res) {
-		IPA_MPM_ERR("failed to allocate GSI channel res=%d\n", res);
-		goto fail_alloc_channel;
-	}
-
-	if (IPA_CLIENT_IS_PROD(mhip_client))
-		ipa_mpm_change_gsi_state(mhi_idx,
-			IPA_MPM_MHIP_CHAN_DL,
-			GSI_ALLOCATED);
-	else
-		ipa_mpm_change_gsi_state(mhi_idx,
-			IPA_MPM_MHIP_CHAN_UL,
-			GSI_ALLOCATED);
-	result = ipa3_start_gsi_channel(ipa_ep_idx);
-	if (result) {
-		IPA_MPM_ERR("start MHIP channel %d failed\n", mhip_client);
-		if (IPA_CLIENT_IS_PROD(mhip_client))
-			ipa_mpm_change_gsi_state(mhi_idx,
-				IPA_MPM_MHIP_CHAN_DL, GSI_ERR);
-		else
-			ipa_mpm_change_gsi_state(mhi_idx,
-				IPA_MPM_MHIP_CHAN_UL, GSI_ERR);
-		goto fail_start_channel;
-	}
-	if (IPA_CLIENT_IS_PROD(mhip_client))
-		ipa_mpm_change_gsi_state(mhi_idx,
-			IPA_MPM_MHIP_CHAN_DL, GSI_STARTED);
-	else
-		ipa_mpm_change_gsi_state(mhi_idx,
-			IPA_MPM_MHIP_CHAN_UL, GSI_STARTED);
-
-	/* Fill in the Device Context params */
-	if (IPA_CLIENT_IS_PROD(mhip_client)) {
-		/* This is the DL channel :: Device -> Host */
-		ipa_mpm_ctx->md[mhi_idx].dl_cons.evt_props.ev_ctx.rbase =
-			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.er_pa;
-		ipa_mpm_ctx->md[mhi_idx].dl_cons.chan_props.ch_ctx.rbase =
-			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.tr_pa;
-	} else {
-		ipa_mpm_ctx->md[mhi_idx].ul_prod.evt_props.ev_ctx.rbase =
-			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.tr_pa;
-		ipa_mpm_ctx->md[mhi_idx].ul_prod.chan_props.ch_ctx.rbase =
-			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.er_pa;
-	}
-
-	IPA_MPM_FUNC_EXIT();
-
-	return 0;
-
-fail_start_channel:
-	ipa3_disable_data_path(ipa_ep_idx);
-	ipa3_stop_gsi_channel(ipa_ep_idx);
-fail_alloc_channel:
-	ipa3_release_gsi_channel(ipa_ep_idx);
-fail_smmu_map_db:
-fail_smmu_map_ring:
-fail_tr_alloc:
-fail_evt_alloc:
-fail_buff_alloc:
-	ipa_assert();
-fail_gen:
-	return -EFAULT;
-}
-
-static void ipa_mpm_clean_mhip_chan(int mhi_idx,
-	enum ipa_client_type mhip_client)
-{
-	int dir;
-	int i;
-	int ipa_ep_idx;
-	int result;
-
-	IPA_MPM_FUNC_ENTRY();
-
-	if (mhip_client == IPA_CLIENT_MAX)
-		return;
-
-	if ((mhi_idx < IPA_MPM_MHIP_CH_ID_0) ||
-		(mhi_idx >= IPA_MPM_MHIP_CH_ID_MAX))
-		return;
-
-	dir = IPA_CLIENT_IS_PROD(mhip_client) ?
-		DMA_TO_HIPA : DMA_FROM_HIPA;
-
-	ipa_ep_idx = ipa3_get_ep_mapping(mhip_client);
-	if (ipa_ep_idx == IPA_EP_NOT_ALLOCATED) {
-		IPA_MPM_ERR("fail to find channel EP.\n");
-		return;
-	}
-
-	/* For the uplink channels, enable HOLB. */
-	if (IPA_CLIENT_IS_CONS(mhip_client))
-		ipa3_disable_data_path(ipa_ep_idx);
-
-	/* Release channel */
-	result = ipa3_stop_gsi_channel(ipa_ep_idx);
-	if (result) {
-		IPA_MPM_ERR("Stop channel for MHIP_Client =  %d failed\n",
-					mhip_client);
-		goto fail_chan;
-	}
-	result = ipa3_reset_gsi_channel(ipa_ep_idx);
-	if (result) {
-		IPA_MPM_ERR("Reset channel for MHIP_Client =  %d failed\n",
-					mhip_client);
-		goto fail_chan;
-	}
-	result = ipa3_reset_gsi_event_ring(ipa_ep_idx);
-	if (result) {
-		IPA_MPM_ERR("Reset ev ring for MHIP_Client =  %d failed\n",
-					mhip_client);
-		goto fail_chan;
-	}
-	result = ipa3_release_gsi_channel(ipa_ep_idx);
-	if (result) {
-		IPA_MPM_ERR("Release tr ring for MHIP_Client =  %d failed\n",
-					mhip_client);
-		if (IPA_CLIENT_IS_PROD(mhip_client))
-			ipa_mpm_change_gsi_state(mhi_idx,
-				IPA_MPM_MHIP_CHAN_DL, GSI_ERR);
-		else
-			ipa_mpm_change_gsi_state(mhi_idx,
-				IPA_MPM_MHIP_CHAN_UL, GSI_ERR);
-		goto fail_chan;
-	}
-
-	if (IPA_CLIENT_IS_PROD(mhip_client))
-		ipa_mpm_change_gsi_state(mhi_idx,
-					IPA_MPM_MHIP_CHAN_DL, GSI_INIT);
-	else
-		ipa_mpm_change_gsi_state(mhi_idx,
-					IPA_MPM_MHIP_CHAN_UL, GSI_INIT);
-
-	memset(&ipa3_ctx->ep[ipa_ep_idx], 0, sizeof(struct ipa3_ep_context));
-
-	/* Unmap Doorbells */
-	if (IPA_CLIENT_IS_PROD(mhip_client)) {
-		ipa_mpm_smmu_unmap_doorbell(MHIP_SMMU_DOMAIN_PCIE,
-			ipa_mpm_ctx->md[mhi_idx].dl_cons.db_device_iova);
-
-		ipa_mpm_smmu_unmap_doorbell(MHIP_SMMU_DOMAIN_IPA,
-			ipa_mpm_ctx->md[mhi_idx].dl_cons.db_host_iova);
-
-		ipa_mpm_ctx->md[mhi_idx].dl_cons.db_host_iova = 0;
-		ipa_mpm_ctx->md[mhi_idx].dl_cons.db_device_iova = 0;
-
-	} else {
-		ipa_mpm_smmu_unmap_doorbell(MHIP_SMMU_DOMAIN_PCIE,
-			ipa_mpm_ctx->md[mhi_idx].ul_prod.db_device_iova);
-
-		ipa_mpm_smmu_unmap_doorbell(MHIP_SMMU_DOMAIN_IPA,
-			ipa_mpm_ctx->md[mhi_idx].ul_prod.db_host_iova);
-
-		ipa_mpm_ctx->md[mhi_idx].ul_prod.db_host_iova = 0;
-		ipa_mpm_ctx->md[mhi_idx].ul_prod.db_device_iova = 0;
-	}
-
-	/* deallocate/Unmap transfer ring buffers */
-	for (i = 1; i < IPA_MPM_RING_LEN; i++) {
-		if (IPA_CLIENT_IS_PROD(mhip_client)) {
-			ipa_mpm_smmu_unmap(
-			(dma_addr_t)
-			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.tre_buff_iova[i],
-			TRE_BUFF_SIZE, dir,
-			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.ap_iova_buff[i]);
-			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.tre_buff_iova[i]
-								= 0;
-			kfree(
-			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.tre_buff_va[i]);
-			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.tre_buff_va[i]
-								= NULL;
-			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.ap_iova_buff[i]
-								= 0;
-			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.tre_buff_iova[i]
-								= 0;
-		} else {
-			ipa_mpm_smmu_unmap(
-			(dma_addr_t)
-			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.tre_buff_iova[i],
-			TRE_BUFF_SIZE, dir,
-			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.ap_iova_buff[i]
-			);
-			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.tre_buff_iova[i]
-								= 0;
-			kfree(
-			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.tre_buff_va[i]);
-			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.tre_buff_va[i]
-								= NULL;
-			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.ap_iova_buff[i]
-								= 0;
-			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.tre_buff_iova[i]
-								= 0;
-		}
-	}
-
-	/* deallocate/Unmap rings */
-	if (IPA_CLIENT_IS_PROD(mhip_client)) {
-		ipa_mpm_smmu_unmap(
-			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.er_pa,
-			IPA_MPM_PAGE_SIZE, dir,
-			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.ap_iova_er);
-
-		ipa_mpm_smmu_unmap(
-			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.tr_pa,
-			IPA_MPM_PAGE_SIZE, dir,
-			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.ap_iova_tr);
-
-		kfree(ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.er_va);
-		kfree(ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.tr_va);
-
-		ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.er_va = NULL;
-		ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.tr_va = NULL;
-		ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.ap_iova_tr = 0;
-		ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.ap_iova_er = 0;
-
-
-	} else {
-		ipa_mpm_smmu_unmap(
-			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.tr_pa,
-			IPA_MPM_PAGE_SIZE, dir,
-			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.ap_iova_tr);
-		ipa_mpm_smmu_unmap(
-			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.er_pa,
-			IPA_MPM_PAGE_SIZE, dir,
-			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.ap_iova_er);
-
-		ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.tr_pa = 0;
-		ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.er_pa = 0;
-
-		kfree(ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.er_va);
-		kfree(ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.tr_va);
-
-		ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.er_va = NULL;
-		ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.tr_va = NULL;
-		ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.ap_iova_er = 0;
-		ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.ap_iova_tr = 0;
-	}
-
-	IPA_MPM_FUNC_EXIT();
-	return;
-fail_chan:
-	ipa_assert();
-}
-
-/* round addresses for closest page per SMMU requirements */
-static inline void ipa_mpm_smmu_round_to_page(uint64_t iova, uint64_t pa,
-	uint64_t size, unsigned long *iova_p, phys_addr_t *pa_p, u32 *size_p)
-{
-	*iova_p = rounddown(iova, PAGE_SIZE);
-	*pa_p = rounddown(pa, PAGE_SIZE);
-	*size_p = roundup(size + pa - *pa_p, PAGE_SIZE);
-}
-
-
-static int __ipa_mpm_configure_mhi_device(struct ipa_mpm_channel *ch,
-	int mhi_idx, int dir)
-{
-	struct mhi_buf ch_config[2];
-	int ret;
-
-	IPA_MPM_FUNC_ENTRY();
-
-	if (ch == NULL) {
-		IPA_MPM_ERR("ch config is NULL\n");
-		return -EINVAL;
-	}
-
-	/* Populate CCA */
-	ch_config[0].buf = &ch->chan_props.ch_ctx;
-	ch_config[0].len = sizeof(ch->chan_props.ch_ctx);
-	ch_config[0].name = "CCA";
-
-	/* populate ECA */
-	ch_config[1].buf = &ch->evt_props.ev_ctx;
-	ch_config[1].len = sizeof(ch->evt_props.ev_ctx);
-	ch_config[1].name = "ECA";
-
-	IPA_MPM_DBG("Configuring MHI PRIME device for mhi_idx %d\n", mhi_idx);
-
-	ret = mhi_device_configure(ipa_mpm_ctx->md[mhi_idx].mhi_dev, dir,
-			ch_config, 2);
-	if (ret) {
-		IPA_MPM_ERR("mhi_device_configure failed\n");
-		return -EINVAL;
-	}
-	IPA_MPM_FUNC_EXIT();
-	return 0;
-}
-
-static void ipa_mpm_mhip_shutdown(int mhip_idx)
-{
-	enum ipa_client_type ul_prod_chan, dl_cons_chan;
-
-	IPA_MPM_FUNC_ENTRY();
-
-	get_ipa3_client(mhip_idx, &ul_prod_chan, &dl_cons_chan);
-
-	if (mhip_idx != IPA_MPM_MHIP_CH_ID_2) {
-		/* For DPL, stop only DL channel */
-		ipa_mpm_start_stop_ul_mhip_data_path(mhip_idx, MPM_MHIP_STOP);
-		ipa_mpm_clean_mhip_chan(mhip_idx, ul_prod_chan);
-	}
-
-	ipa_mpm_clean_mhip_chan(mhip_idx, dl_cons_chan);
-
-	mutex_lock(&ipa_mpm_ctx->md[mhip_idx].lpm_mutex);
-	if (!ipa_mpm_ctx->md[mhip_idx].in_lpm) {
-		ipa_mpm_vote_unvote_ipa_clk(CLK_OFF, mhip_idx);
-		/* while in modem shutdown scenarios such as SSR, no explicit
-		 * PCIe vote is needed.
-		 */
-		ipa_mpm_ctx->md[mhip_idx].in_lpm = true;
-	}
-	mutex_unlock(&ipa_mpm_ctx->md[mhip_idx].lpm_mutex);
-	ipa_mpm_ctx->md[mhip_idx].mhi_dev = NULL;
-	ipa_mpm_ctx->md[mhip_idx].init_complete = false;
-	IPA_MPM_FUNC_EXIT();
-}
-
-/*
- * Turning on/OFF PCIE Clock is done once for all clients.
- * Always vote for Probe_ID 0 as a standard.
- */
-static int ipa_mpm_vote_unvote_pcie_clk(enum ipa_mpm_clk_vote_type vote,
-	int probe_id)
-{
-	int result = 0;
-
-	if (probe_id >= IPA_MPM_MHIP_CH_ID_MAX) {
-		IPA_MPM_ERR("probe_id not found\n");
-		return -EINVAL;
-	}
-
-	if (vote > CLK_OFF) {
-		IPA_MPM_ERR("Invalid vote\n");
-		return -EINVAL;
-	}
-
-	if (ipa_mpm_ctx->md[probe_id].mhi_dev == NULL) {
-		IPA_MPM_ERR("MHI not initialized yet\n");
-		return 0;
-	}
-
-	IPA_MPM_ERR("PCIe clock vote/unvote = %d probe_id = %d clk_cnt = %d\n",
-		vote, probe_id,
-		atomic_read(&ipa_mpm_ctx->md[probe_id].clk_cnt.pcie_clk_cnt));
-
-	if (vote == CLK_ON) {
-		result = mhi_device_get_sync(
-			ipa_mpm_ctx->md[probe_id].mhi_dev, MHI_VOTE_BUS);
-		if (result) {
-			IPA_MPM_ERR("mhi_sync_get failed for probe_id %d\n",
-				result, probe_id);
-			return result;
-		}
-
-		IPA_MPM_DBG("probe_id %d PCIE clock now ON\n", probe_id);
-		atomic_inc(&ipa_mpm_ctx->md[probe_id].clk_cnt.pcie_clk_cnt);
-		atomic_inc(&ipa_mpm_ctx->pcie_clk_total_cnt);
-	} else {
-		if ((atomic_read(
-			&ipa_mpm_ctx->md[probe_id].clk_cnt.pcie_clk_cnt)
-								== 0)) {
-			IPA_MPM_DBG("probe_id %d PCIE clock already devoted\n",
-				probe_id);
-			WARN_ON(1);
-			return 0;
-		}
-		mhi_device_put(ipa_mpm_ctx->md[probe_id].mhi_dev, MHI_VOTE_BUS);
-		IPA_MPM_DBG("probe_id %d PCIE clock off\n", probe_id);
-		atomic_dec(&ipa_mpm_ctx->md[probe_id].clk_cnt.pcie_clk_cnt);
-		atomic_dec(&ipa_mpm_ctx->pcie_clk_total_cnt);
-	}
-
-	return result;
-}
-
-/*
- * Turning on/OFF IPA Clock is done only once- for all clients
- */
-static void ipa_mpm_vote_unvote_ipa_clk(enum ipa_mpm_clk_vote_type vote,
-	int probe_id)
-{
-	if (vote > CLK_OFF)
-		return;
-
-	IPA_MPM_ERR("IPA clock vote/unvote = %d probe_id = %d clk_cnt = %d\n",
-		vote, probe_id,
-		atomic_read(&ipa_mpm_ctx->md[probe_id].clk_cnt.ipa_clk_cnt));
-
-	if (vote == CLK_ON) {
-		IPA_ACTIVE_CLIENTS_INC_SPECIAL(ipa_mpm_mhip_chan_str[probe_id]);
-		IPA_MPM_DBG("IPA clock now ON for probe_id %d\n", probe_id);
-		atomic_inc(&ipa_mpm_ctx->md[probe_id].clk_cnt.ipa_clk_cnt);
-		atomic_inc(&ipa_mpm_ctx->ipa_clk_total_cnt);
-	} else {
-		if ((atomic_read
-			(&ipa_mpm_ctx->md[probe_id].clk_cnt.ipa_clk_cnt)
-								== 0)) {
-			IPA_MPM_DBG("probe_id %d IPA clock count < 0\n",
-				probe_id);
-			WARN_ON(1);
-			return;
-		}
-		IPA_ACTIVE_CLIENTS_DEC_SPECIAL(ipa_mpm_mhip_chan_str[probe_id]);
-		IPA_MPM_DBG("probe_id %d IPA clock off\n", probe_id);
-		atomic_dec(&ipa_mpm_ctx->md[probe_id].clk_cnt.ipa_clk_cnt);
-		atomic_dec(&ipa_mpm_ctx->ipa_clk_total_cnt);
-	}
-}
-
-static enum mhip_status_type ipa_mpm_start_stop_mhip_chan(
-	enum ipa_mpm_mhip_chan mhip_chan,
-	int probe_id,
-	enum ipa_mpm_start_stop_type start_stop)
-{
-	int ipa_ep_idx;
-	struct ipa3_ep_context *ep;
-	bool is_start;
-	enum ipa_client_type ul_chan, dl_chan;
-	u32 source_pipe_bitmask = 0;
-	enum gsi_status gsi_res = GSI_STATUS_SUCCESS;
-	int result;
-
-	IPA_MPM_FUNC_ENTRY();
-
-	if (mhip_chan > IPA_MPM_MHIP_CHAN_BOTH) {
-		IPA_MPM_ERR("MHI not initialized yet\n");
-		return MHIP_STATUS_FAIL;
-	}
-
-	if (probe_id >= IPA_MPM_MHIP_CH_ID_MAX) {
-		IPA_MPM_ERR("MHI not initialized yet\n");
-		return MHIP_STATUS_FAIL;
-	}
-
-	get_ipa3_client(probe_id, &ul_chan, &dl_chan);
-
-	if (mhip_chan == IPA_MPM_MHIP_CHAN_UL) {
-		ipa_ep_idx = ipa3_get_ep_mapping(ul_chan);
-	} else if (mhip_chan == IPA_MPM_MHIP_CHAN_DL) {
-		ipa_ep_idx = ipa3_get_ep_mapping(dl_chan);
-	} else if (mhip_chan == IPA_MPM_MHIP_CHAN_BOTH) {
-		ipa_ep_idx = ipa3_get_ep_mapping(ul_chan);
-		ipa_ep_idx = ipa3_get_ep_mapping(dl_chan);
-	}
-
-	if (ipa_ep_idx == IPA_EP_NOT_ALLOCATED) {
-		IPA_MPM_ERR("fail to get EP# for idx %d\n", ipa_ep_idx);
-		return MHIP_STATUS_EP_NOT_FOUND;
-	}
-	ep = &ipa3_ctx->ep[ipa_ep_idx];
-
-	if (mhip_chan == IPA_MPM_MHIP_CHAN_UL) {
-		IPA_MPM_DBG("current GSI state = %d, action = %d\n",
-			ipa_mpm_ctx->md[probe_id].ul_prod.gsi_state,
-			start_stop);
-		if (ipa_mpm_ctx->md[probe_id].ul_prod.gsi_state <
-			GSI_ALLOCATED) {
-			IPA_MPM_ERR("GSI chan is not allocated yet\n");
-			return MHIP_STATUS_EP_NOT_READY;
-		}
-	} else if (mhip_chan == IPA_MPM_MHIP_CHAN_DL) {
-		IPA_MPM_DBG("current GSI state = %d, action = %d\n",
-			ipa_mpm_ctx->md[probe_id].dl_cons.gsi_state,
-			start_stop);
-		if (ipa_mpm_ctx->md[probe_id].dl_cons.gsi_state <
-			GSI_ALLOCATED) {
-			IPA_MPM_ERR("GSI chan is not allocated yet\n");
-			return MHIP_STATUS_EP_NOT_READY;
-		}
-	}
-
-	is_start = (start_stop == MPM_MHIP_START) ? true : false;
-
-	if (is_start) {
-		if (mhip_chan == IPA_MPM_MHIP_CHAN_UL) {
-			if (ipa_mpm_ctx->md[probe_id].ul_prod.gsi_state ==
-				GSI_STARTED) {
-				IPA_MPM_ERR("GSI chan is already started\n");
-				return MHIP_STATUS_NO_OP;
-			}
-		}
-
-		if (mhip_chan == IPA_MPM_MHIP_CHAN_DL) {
-			if (ipa_mpm_ctx->md[probe_id].dl_cons.gsi_state ==
-				GSI_STARTED) {
-				IPA_MPM_ERR("GSI chan is already started\n");
-				return MHIP_STATUS_NO_OP;
-			}
-		}
-		/* Start GSI channel */
-		gsi_res = ipa3_start_gsi_channel(ipa_ep_idx);
-		if (gsi_res != GSI_STATUS_SUCCESS) {
-			IPA_MPM_ERR("Error starting channel: err = %d\n",
-					gsi_res);
-			goto gsi_chan_fail;
-		} else {
-			ipa_mpm_change_gsi_state(probe_id, mhip_chan,
-					GSI_STARTED);
-		}
-	} else {
-		if (mhip_chan == IPA_MPM_MHIP_CHAN_UL) {
-			if (ipa_mpm_ctx->md[probe_id].ul_prod.gsi_state ==
-				GSI_STOPPED) {
-				IPA_MPM_ERR("GSI chan is already stopped\n");
-				return MHIP_STATUS_NO_OP;
-			} else if (ipa_mpm_ctx->md[probe_id].ul_prod.gsi_state
-				!= GSI_STARTED) {
-				IPA_MPM_ERR("GSI chan isn't already started\n");
-				return MHIP_STATUS_NO_OP;
-			}
-		}
-
-		if (mhip_chan == IPA_MPM_MHIP_CHAN_DL) {
-			if (ipa_mpm_ctx->md[probe_id].dl_cons.gsi_state ==
-				GSI_STOPPED) {
-				IPA_MPM_ERR("GSI chan is already stopped\n");
-				return MHIP_STATUS_NO_OP;
-			} else if (ipa_mpm_ctx->md[probe_id].dl_cons.gsi_state
-				!= GSI_STARTED) {
-				IPA_MPM_ERR("GSI chan isn't already started\n");
-				return MHIP_STATUS_NO_OP;
-			}
-		}
-
-		if (mhip_chan == IPA_MPM_MHIP_CHAN_UL) {
-			source_pipe_bitmask = 1 <<
-				ipa3_get_ep_mapping(ep->client);
-			/* First Stop UL GSI channel before unvote PCIe clock */
-			result = ipa3_stop_gsi_channel(ipa_ep_idx);
-
-			if (result) {
-				IPA_MPM_ERR("UL chan stop failed\n");
-				goto gsi_chan_fail;
-			} else {
-				ipa_mpm_change_gsi_state(probe_id, mhip_chan,
-							GSI_STOPPED);
-			}
-		}
-
-		if (mhip_chan == IPA_MPM_MHIP_CHAN_DL) {
-			result = ipa3_stop_gsi_channel(ipa_ep_idx);
-			if (result) {
-				IPA_MPM_ERR("Fail to stop DL channel\n");
-				goto gsi_chan_fail;
-			} else {
-				ipa_mpm_change_gsi_state(probe_id, mhip_chan,
-							GSI_STOPPED);
-			}
-		}
-	}
-	IPA_MPM_FUNC_EXIT();
-
-	return MHIP_STATUS_SUCCESS;
-gsi_chan_fail:
-	ipa3_disable_data_path(ipa_ep_idx);
-	ipa_mpm_change_gsi_state(probe_id, mhip_chan, GSI_ERR);
-	ipa_assert();
-	return MHIP_STATUS_FAIL;
-}
-
-int ipa_mpm_notify_wan_state(struct wan_ioctl_notify_wan_state *state)
-{
-	int probe_id = IPA_MPM_MHIP_CH_ID_MAX;
-	int i;
-	static enum mhip_status_type status;
-	int ret = 0;
-	enum ipa_mpm_mhip_client_type mhip_client = IPA_MPM_MHIP_TETH;
-
-	if (!state)
-		return -EPERM;
-
-	if (!ipa3_is_mhip_offload_enabled())
-		return -EPERM;
-
-	for (i = 0; i < IPA_MPM_MHIP_CH_ID_MAX; i++) {
-		if (ipa_mpm_pipes[i].mhip_client == mhip_client) {
-			probe_id = i;
-			break;
-		}
-	}
-
-	if (probe_id == IPA_MPM_MHIP_CH_ID_MAX) {
-		IPA_MPM_ERR("Unknown probe_id\n");
-		return -EPERM;
-	}
-
-	IPA_MPM_DBG("WAN backhaul available for probe_id = %d\n", probe_id);
-
-	if (state->up) {
-		/* Start UL MHIP channel for offloading tethering connection */
-		ret = ipa_mpm_vote_unvote_pcie_clk(CLK_ON, probe_id);
-		if (ret) {
-			IPA_MPM_ERR("Error cloking on PCIe clk, err = %d\n",
-				ret);
-			return ret;
-		}
-		status = ipa_mpm_start_stop_mhip_chan(
-				IPA_MPM_MHIP_CHAN_UL, probe_id, MPM_MHIP_START);
-		switch (status) {
-		case MHIP_STATUS_SUCCESS:
-			ipa_mpm_ctx->md[probe_id].teth_state =
-						IPA_MPM_TETH_CONNECTED;
-			ret = ipa_mpm_start_stop_ul_mhip_data_path(
-						probe_id, MPM_MHIP_START);
-			if (ret) {
-				IPA_MPM_ERR("err UL chan start\n");
-				ipa_mpm_vote_unvote_pcie_clk(CLK_OFF, probe_id);
-				return ret;
-			}
-			break;
-		case MHIP_STATUS_EP_NOT_READY:
-		case MHIP_STATUS_NO_OP:
-			IPA_MPM_DBG("UL chan already start, status = %d\n",
-					status);
-			ret = ipa_mpm_vote_unvote_pcie_clk(CLK_OFF, probe_id);
-			return ret;
-		case MHIP_STATUS_FAIL:
-		case MHIP_STATUS_BAD_STATE:
-		case MHIP_STATUS_EP_NOT_FOUND:
-			IPA_MPM_ERR("UL chan start err =%d\n", status);
-			ret = ipa_mpm_vote_unvote_pcie_clk(CLK_OFF, probe_id);
-			ipa_assert();
-			return -EFAULT;
-		default:
-			IPA_MPM_ERR("Err not found\n");
-			ipa_mpm_vote_unvote_pcie_clk(CLK_OFF, probe_id);
-			ret = -EFAULT;
-			break;
-		}
-		ipa_mpm_ctx->md[probe_id].mhip_client = mhip_client;
-	} else {
-		status = ipa_mpm_start_stop_mhip_chan(
-					IPA_MPM_MHIP_CHAN_UL, probe_id,
-					MPM_MHIP_STOP);
-		switch (status) {
-		case MHIP_STATUS_SUCCESS:
-			ipa_mpm_change_teth_state(probe_id, IPA_MPM_TETH_INIT);
-			ipa_mpm_start_stop_ul_mhip_data_path(probe_id,
-							MPM_MHIP_STOP);
-			break;
-		case MHIP_STATUS_NO_OP:
-		case MHIP_STATUS_EP_NOT_READY:
-			IPA_MPM_DBG("UL chan already stop, status = %d\n",
-					status);
-			break;
-		case MHIP_STATUS_FAIL:
-		case MHIP_STATUS_BAD_STATE:
-		case MHIP_STATUS_EP_NOT_FOUND:
-			IPA_MPM_ERR("UL chan cant be stopped err =%d\n",
-				status);
-			ipa_assert();
-			return -EFAULT;
-		default:
-			IPA_MPM_ERR("Err not found\n");
-			return -EFAULT;
-		}
-		/* Stop UL MHIP channel for offloading tethering connection */
-		ret = ipa_mpm_vote_unvote_pcie_clk(CLK_OFF, probe_id);
-
-		if (ret) {
-			IPA_MPM_ERR("Error cloking on PCIe clk, err = %d\n",
-				ret);
-			return ret;
-		}
-		ipa_mpm_ctx->md[probe_id].mhip_client = IPA_MPM_MHIP_NONE;
-	}
-	return ret;
-}
-
-static void ipa_mpm_change_gsi_state(int probe_id,
-	enum ipa_mpm_mhip_chan mhip_chan,
-	enum ipa_mpm_gsi_state next_state)
-{
-
-	if (probe_id >= IPA_MPM_MHIP_CH_ID_MAX)
-		return;
-
-	if (mhip_chan == IPA_MPM_MHIP_CHAN_UL) {
-		mutex_lock(&ipa_mpm_ctx->md[probe_id].mutex);
-		ipa_mpm_ctx->md[probe_id].ul_prod.gsi_state = next_state;
-		IPA_MPM_DBG("GSI next_state = %d\n",
-			ipa_mpm_ctx->md[probe_id].ul_prod.gsi_state);
-		 mutex_unlock(&ipa_mpm_ctx->md[probe_id].mutex);
-	}
-
-	if (mhip_chan == IPA_MPM_MHIP_CHAN_DL) {
-		mutex_lock(&ipa_mpm_ctx->md[probe_id].mutex);
-		ipa_mpm_ctx->md[probe_id].dl_cons.gsi_state = next_state;
-		IPA_MPM_DBG("GSI next_state = %d\n",
-			ipa_mpm_ctx->md[probe_id].dl_cons.gsi_state);
-		 mutex_unlock(&ipa_mpm_ctx->md[probe_id].mutex);
-	}
-}
-
-static void ipa_mpm_change_teth_state(int probe_id,
-	enum ipa_mpm_teth_state next_state)
-{
-	enum ipa_mpm_teth_state curr_state;
-
-	if (probe_id >= IPA_MPM_MHIP_CH_ID_MAX) {
-		IPA_MPM_ERR("Unknown probe_id\n");
-		return;
-	}
-
-	curr_state = ipa_mpm_ctx->md[probe_id].teth_state;
-
-	IPA_MPM_DBG("curr_state = %d, ip_state = %d mhip_s\n",
-		curr_state, next_state);
-
-	switch (curr_state) {
-	case IPA_MPM_TETH_INIT:
-		if (next_state == IPA_MPM_TETH_CONNECTED)
-			next_state = IPA_MPM_TETH_INPROGRESS;
-		break;
-	case IPA_MPM_TETH_INPROGRESS:
-		break;
-	case IPA_MPM_TETH_CONNECTED:
-		break;
-	default:
-		IPA_MPM_ERR("No change in state\n");
-		break;
-	}
-
-	ipa_mpm_ctx->md[probe_id].teth_state = next_state;
-	IPA_MPM_DBG("next_state = %d\n", next_state);
-}
-
-static void ipa_mpm_read_channel(enum ipa_client_type chan)
-{
-	struct gsi_chan_info chan_info;
-	int ipa_ep_idx;
-	struct ipa3_ep_context *ep;
-	int res;
-
-	ipa_ep_idx = ipa3_get_ep_mapping(chan);
-
-	if (ipa_ep_idx == IPA_EP_NOT_ALLOCATED) {
-		IPAERR("failed to get idx");
-		return;
-	}
-
-	ep = &ipa3_ctx->ep[ipa_ep_idx];
-
-	IPA_MPM_ERR("Reading channel for chan %d, ep = %d, gsi_chan_hdl = %d\n",
-		chan, ep, ep->gsi_chan_hdl);
-
-	res = ipa3_get_gsi_chan_info(&chan_info, ep->gsi_chan_hdl);
-	if (res)
-		IPA_MPM_ERR("Reading of channel failed for ep %d\n", ep);
-}
-
-static int ipa_mpm_start_stop_ul_mhip_data_path(int probe_id,
-	enum ipa_mpm_start_stop_type start)
-{
-	int ipa_ep_idx;
-	int res = 0;
-	enum ipa_client_type ul_chan, dl_chan;
-
-	if (probe_id >= IPA_MPM_MHIP_CH_ID_MAX) {
-		IPA_MPM_ERR("Unknown probe_id\n");
-		return 0;
-	}
-	get_ipa3_client(probe_id, &ul_chan, &dl_chan);
-	IPA_MPM_DBG("Start/Stop Data Path ? = %d\n", start);
-
-
-	/* MHIP Start Data path:
-	 * IPA MHIP Producer: remove HOLB
-	 * IPA MHIP Consumer : no op as there is no delay on these pipes.
-	 */
-	if (start) {
-		IPA_MPM_DBG("Enabling data path\n");
-		if (ul_chan != IPA_CLIENT_MAX) {
-			/* Remove HOLB on the producer pipe */
-			IPA_MPM_DBG("Removing HOLB on ep = %s\n",
-				__stringify(ul_chan));
-			ipa_ep_idx = ipa3_get_ep_mapping(ul_chan);
-
-			if (ipa_ep_idx == IPA_EP_NOT_ALLOCATED) {
-				IPAERR("failed to get idx");
-				return ipa_ep_idx;
-			}
-
-			res = ipa3_enable_data_path(ipa_ep_idx);
-			if (res)
-				IPA_MPM_ERR("Enable data path failed res=%d\n",
-					res);
-		}
-	} else {
-		IPA_MPM_DBG("Disabling data path\n");
-		if (ul_chan != IPA_CLIENT_MAX) {
-			/* Set HOLB on the producer pipe */
-			ipa_ep_idx = ipa3_get_ep_mapping(ul_chan);
-
-			if (ipa_ep_idx == IPA_EP_NOT_ALLOCATED) {
-				IPAERR("failed to get idx");
-				return ipa_ep_idx;
-			}
-
-			res = ipa3_disable_data_path(ipa_ep_idx);
-			if (res)
-				IPA_MPM_ERR("disable data path failed res=%d\n",
-					res);
-		}
-	}
-
-	return res;
-}
-
-/* ipa_mpm_mhi_probe_cb is received for each MHI'/MHI channel
- * Currently we have 4 MHI channels.
- */
-static int ipa_mpm_mhi_probe_cb(struct mhi_device *mhi_dev,
-	const struct mhi_device_id *mhi_id)
-{
-	struct ipa_mpm_channel *ch;
-	int ret;
-	enum ipa_client_type ul_prod, dl_cons;
-	int probe_id;
-	struct ipa_req_chan_out_params ul_out_params, dl_out_params;
-	void __iomem  *db_addr;
-	int ipa_ep_idx;
-	struct ipa3_ep_context *ep;
-	u32 evt_ring_db_addr_low, evt_ring_db_addr_high;
-	u32 wp_addr;
-	int pipe_idx;
-
-	IPA_MPM_FUNC_ENTRY();
-
-	if (ipa_mpm_ctx == NULL) {
-		IPA_MPM_ERR("ipa_mpm_ctx is NULL not expected, returning..\n");
-		return -ENOMEM;
-	}
-
-	probe_id = get_idx_from_id(mhi_id);
-
-	if (probe_id >= IPA_MPM_MHIP_CH_ID_MAX) {
-		IPA_MPM_ERR("chan=%pK is not supported for now\n", mhi_id);
-		return -EPERM;
-	}
-
-	if (ipa_mpm_ctx->md[probe_id].init_complete) {
-		IPA_MPM_ERR("Probe initialization already done, returning\n");
-		return 0;
-	}
-
-	IPA_MPM_DBG("Received probe for id=%d\n", probe_id);
-
-	get_ipa3_client(probe_id, &ul_prod, &dl_cons);
-
-	/* Vote for IPA clock for first time in initialization seq.
-	 * IPA clock will be devoted when MHI enters LPM
-	 * PCIe clock will be voted / devoted with every channel probe
-	 * we receive.
-	 * ul_prod = Host -> Device
-	 * dl_cons = Device -> Host
-	 */
-	ipa_mpm_ctx->md[probe_id].mhi_dev = mhi_dev;
-	ipa_mpm_ctx->mhi_parent_dev =
-		ipa_mpm_ctx->md[probe_id].mhi_dev->dev.parent;
-
-	ipa_mpm_vote_unvote_pcie_clk(CLK_ON, probe_id);
-	mutex_lock(&ipa_mpm_ctx->md[probe_id].lpm_mutex);
-	ipa_mpm_vote_unvote_ipa_clk(CLK_ON, probe_id);
-	ipa_mpm_ctx->md[probe_id].in_lpm = false;
-	mutex_unlock(&ipa_mpm_ctx->md[probe_id].lpm_mutex);
-	IPA_MPM_DBG("ul chan = %d, dl_chan = %d\n", ul_prod, dl_cons);
-
-	/*
-	 * Set up MHI' pipes for Device IPA filling in
-	 * Channel Context and Event Context.
-	 * These params will be sent to Device side.
-	 * UL CHAN = HOST -> Device
-	 * DL CHAN = Device -> HOST
-	 * per channel a TRE and EV is allocated.
-	 * for a UL channel -
-	 * IPA HOST PROD TRE -> IPA DEVICE CONS EV
-	 * IPA HOST PROD EV ->  IPA DEVICE CONS TRE
-	 * for a DL channel -
-	 * IPA Device PROD TRE -> IPA HOST CONS EV
-	 * IPA Device PROD EV ->  IPA HOST CONS TRE
-	 */
-	if (ul_prod != IPA_CLIENT_MAX) {
-		/* store UL properties */
-		ch = &ipa_mpm_ctx->md[probe_id].ul_prod;
-		/* Store Channel properties */
-		ch->chan_props.id = mhi_dev->ul_chan_id;
-		ch->chan_props.device_db =
-			ipa_mpm_ctx->dev_info.chdb_base +
-			ch->chan_props.id * 8;
-		/* Fill Channel Conext to be sent to Device side */
-		ch->chan_props.ch_ctx.chtype =
-			IPA_MPM_MHI_HOST_UL_CHANNEL;
-		ch->chan_props.ch_ctx.erindex =
-			mhi_dev->ul_event_id;
-		ch->chan_props.ch_ctx.rlen = (IPA_MPM_RING_LEN) *
-			GSI_EVT_RING_RE_SIZE_16B;
-		/* Store Event properties */
-		ch->evt_props.ev_ctx.update_rp_modc = 0;
-		ch->evt_props.ev_ctx.update_rp_intmodt = 0;
-		ch->evt_props.ev_ctx.ertype = 1;
-		ch->evt_props.ev_ctx.rlen = (IPA_MPM_RING_LEN) *
-			GSI_EVT_RING_RE_SIZE_16B;
-		ch->evt_props.ev_ctx.buff_size = TRE_BUFF_SIZE;
-		ch->evt_props.device_db =
-			ipa_mpm_ctx->dev_info.erdb_base +
-			ch->chan_props.ch_ctx.erindex * 8;
-	}
-
-	if (dl_cons != IPA_CLIENT_MAX) {
-		/* store DL channel properties */
-		ch = &ipa_mpm_ctx->md[probe_id].dl_cons;
-		/* Store Channel properties */
-		ch->chan_props.id = mhi_dev->dl_chan_id;
-		ch->chan_props.device_db =
-			ipa_mpm_ctx->dev_info.chdb_base +
-			ch->chan_props.id * 8;
-		/* Fill Channel Conext to be be sent to Dev side */
-		ch->chan_props.ch_ctx.chstate = 1;
-		ch->chan_props.ch_ctx.chtype =
-			IPA_MPM_MHI_HOST_DL_CHANNEL;
-		ch->chan_props.ch_ctx.erindex = mhi_dev->dl_event_id;
-		ch->chan_props.ch_ctx.rlen = (IPA_MPM_RING_LEN) *
-			GSI_EVT_RING_RE_SIZE_16B;
-		/* Store Event properties */
-		ch->evt_props.ev_ctx.update_rp_modc = 0;
-		ch->evt_props.ev_ctx.update_rp_intmodt = 0;
-		ch->evt_props.ev_ctx.ertype = 1;
-		ch->evt_props.ev_ctx.rlen = (IPA_MPM_RING_LEN) *
-			GSI_EVT_RING_RE_SIZE_16B;
-		ch->evt_props.ev_ctx.buff_size = TRE_BUFF_SIZE;
-		ch->evt_props.device_db =
-			ipa_mpm_ctx->dev_info.erdb_base +
-			ch->chan_props.ch_ctx.erindex * 8;
-	}
-	/* connect Host GSI pipes with MHI' protocol */
-	if (ul_prod != IPA_CLIENT_MAX)  {
-		ret = ipa_mpm_connect_mhip_gsi_pipe(ul_prod,
-			probe_id, &ul_out_params);
-		if (ret) {
-			IPA_MPM_ERR("failed connecting MPM client %d\n",
-					ul_prod);
-			goto fail_gsi_setup;
-		}
-	}
-
-	if (dl_cons != IPA_CLIENT_MAX) {
-		ret = ipa_mpm_connect_mhip_gsi_pipe(dl_cons,
-			probe_id, &dl_out_params);
-		if (ret) {
-			IPA_MPM_ERR("connecting MPM client = %d failed\n",
-				dl_cons);
-			goto fail_gsi_setup;
-		}
-	}
-
-	if (ul_prod != IPA_CLIENT_MAX) {
-		ch = &ipa_mpm_ctx->md[probe_id].ul_prod;
-		ch->evt_props.ev_ctx.update_rp_addr =
-			ipa_mpm_smmu_map_doorbell(
-				MHIP_SMMU_DOMAIN_PCIE,
-				ul_out_params.db_reg_phs_addr_lsb);
-		if (ch->evt_props.ev_ctx.update_rp_addr == 0)
-			ipa_assert();
-		ipa_mpm_ctx->md[probe_id].ul_prod.db_device_iova =
-			ch->evt_props.ev_ctx.update_rp_addr;
-
-		ret = __ipa_mpm_configure_mhi_device(
-				ch, probe_id, DMA_TO_HIPA);
-		if (ret) {
-			IPA_MPM_ERR("configure_mhi_dev fail %d\n",
-					ret);
-			goto fail_smmu;
-		}
-	}
-
-	if (dl_cons != IPA_CLIENT_MAX) {
-		ch = &ipa_mpm_ctx->md[probe_id].dl_cons;
-		ch->evt_props.ev_ctx.update_rp_addr =
-			ipa_mpm_smmu_map_doorbell(
-					MHIP_SMMU_DOMAIN_PCIE,
-					dl_out_params.db_reg_phs_addr_lsb);
-
-		if (ch->evt_props.ev_ctx.update_rp_addr == 0)
-			ipa_assert();
-
-		ipa_mpm_ctx->md[probe_id].dl_cons.db_device_iova =
-			ch->evt_props.ev_ctx.update_rp_addr;
-
-		ret = __ipa_mpm_configure_mhi_device(ch, probe_id,
-					DMA_FROM_HIPA);
-		if (ret) {
-			IPA_MPM_ERR("mpm_config_mhi_dev failed %d\n", ret);
-			goto fail_smmu;
-		}
-	}
-
-	ret = mhi_prepare_for_transfer(ipa_mpm_ctx->md[probe_id].mhi_dev);
-	if (ret) {
-		IPA_MPM_ERR("mhi_prepare_for_transfer failed %d\n", ret);
-		WARN_ON(1);
-		/*
-		 * WA to handle prepare_for_tx failures.
-		 * Though prepare for transfer fails, indicate success
-		 * to MHI driver. remove_cb will be called eventually when
-		 * Device side comes from where pending cleanup happens.
-		 */
-		atomic_inc(&ipa_mpm_ctx->probe_cnt);
-		ipa_mpm_ctx->md[probe_id].init_complete = true;
-		IPA_MPM_FUNC_EXIT();
-		return 0;
-	}
-
-	/*
-	 * Ring initial channel db - Host Side UL and Device side DL channel.
-	 * To ring doorbell, write "WP" into doorbell register.
-	 * This WP should be set to 1 element less than ring max.
-	 */
-
-	/* Ring UL PRODUCER TRANSFER RING (HOST IPA -> DEVICE IPA) Doorbell */
-	if (ul_prod != IPA_CLIENT_MAX) {
-		IPA_MPM_DBG("Host UL TR PA DB = 0X%0x\n",
-			ul_out_params.db_reg_phs_addr_lsb);
-
-		db_addr = ioremap(
-			(phys_addr_t)(ul_out_params.db_reg_phs_addr_lsb), 4);
-
-		wp_addr = ipa_mpm_ctx->md[probe_id].ul_prod_ring.tr_pa +
-			((IPA_MPM_RING_LEN - 1) * GSI_CHAN_RE_SIZE_16B);
-
-		iowrite32(wp_addr, db_addr);
-
-		IPA_MPM_DBG("Host UL TR  DB = 0X%pK, wp_addr = 0X%0x",
-			db_addr, wp_addr);
-
-		iounmap(db_addr);
-		ipa_mpm_read_channel(ul_prod);
-	}
-
-	/* Ring UL PRODUCER EVENT RING (HOST IPA -> DEVICE IPA) Doorbell
-	 * Ring the event DB to a value outside the
-	 * ring range such that rp and wp never meet.
-	 */
-	if (ul_prod != IPA_CLIENT_MAX) {
-		ipa_ep_idx = ipa3_get_ep_mapping(ul_prod);
-
-		if (ipa_ep_idx == IPA_EP_NOT_ALLOCATED) {
-			IPA_MPM_ERR("fail to alloc EP.\n");
-			goto fail_start_channel;
-		}
-		ep = &ipa3_ctx->ep[ipa_ep_idx];
-
-		IPA_MPM_DBG("for ep_idx %d , gsi_evt_ring_hdl = %ld\n",
-			ipa_ep_idx, ep->gsi_evt_ring_hdl);
-		gsi_query_evt_ring_db_addr(ep->gsi_evt_ring_hdl,
-			&evt_ring_db_addr_low, &evt_ring_db_addr_high);
-
-		IPA_MPM_DBG("Host UL ER PA DB = 0X%0x\n",
-			evt_ring_db_addr_low);
-
-		db_addr = ioremap((phys_addr_t)(evt_ring_db_addr_low), 4);
-
-		wp_addr = ipa_mpm_ctx->md[probe_id].ul_prod_ring.er_pa +
-			((IPA_MPM_RING_LEN + 1) * GSI_EVT_RING_RE_SIZE_16B);
-		IPA_MPM_DBG("Host UL ER  DB = 0X%pK, wp_addr = 0X%0x",
-			db_addr, wp_addr);
-
-		iowrite32(wp_addr, db_addr);
-		iounmap(db_addr);
-	}
-
-	/* Ring DEVICE IPA DL CONSUMER Event Doorbell */
-	if (ul_prod != IPA_CLIENT_MAX) {
-		db_addr = ioremap((phys_addr_t)
-			(ipa_mpm_ctx->md[probe_id].ul_prod.evt_props.device_db),
-			4);
-
-		wp_addr = ipa_mpm_ctx->md[probe_id].ul_prod_ring.tr_pa +
-			((IPA_MPM_RING_LEN + 1) * GSI_EVT_RING_RE_SIZE_16B);
-
-		iowrite32(wp_addr, db_addr);
-		iounmap(db_addr);
-	}
-
-	/* Ring DL PRODUCER (DEVICE IPA -> HOST IPA) Doorbell */
-	if (dl_cons != IPA_CLIENT_MAX) {
-		db_addr = ioremap((phys_addr_t)
-		(ipa_mpm_ctx->md[probe_id].dl_cons.chan_props.device_db),
-		4);
-
-		wp_addr = ipa_mpm_ctx->md[probe_id].dl_prod_ring.tr_pa +
-			((IPA_MPM_RING_LEN - 1) * GSI_CHAN_RE_SIZE_16B);
-
-		IPA_MPM_DBG("Device DL TR  DB = 0X%pK, wp_addr = 0X%0x",
-			db_addr, wp_addr);
-
-		iowrite32(wp_addr, db_addr);
-
-		iounmap(db_addr);
-	}
-
-	/*
-	 * Ring event ring DB on Device side.
-	 * ipa_mpm should ring the event DB to a value outside the
-	 * ring range such that rp and wp never meet.
-	 */
-	if (dl_cons != IPA_CLIENT_MAX) {
-		db_addr =
-		ioremap(
-		(phys_addr_t)
-		(ipa_mpm_ctx->md[probe_id].dl_cons.evt_props.device_db),
-		4);
-
-		wp_addr = ipa_mpm_ctx->md[probe_id].dl_prod_ring.er_pa +
-			((IPA_MPM_RING_LEN + 1) * GSI_EVT_RING_RE_SIZE_16B);
-
-		iowrite32(wp_addr, db_addr);
-		IPA_MPM_DBG("Device  UL ER  DB = 0X%pK,wp_addr = 0X%0x",
-			db_addr, wp_addr);
-		iounmap(db_addr);
-	}
-
-	/* Ring DL EVENT RING CONSUMER (DEVICE IPA CONSUMER) Doorbell */
-	if (dl_cons != IPA_CLIENT_MAX) {
-		ipa_ep_idx = ipa3_get_ep_mapping(dl_cons);
-
-		if (ipa_ep_idx == IPA_EP_NOT_ALLOCATED) {
-			IPA_MPM_ERR("fail to alloc EP.\n");
-			goto fail_start_channel;
-		}
-		ep = &ipa3_ctx->ep[ipa_ep_idx];
-
-		gsi_query_evt_ring_db_addr(ep->gsi_evt_ring_hdl,
-			&evt_ring_db_addr_low, &evt_ring_db_addr_high);
-		IPA_MPM_DBG("Host DL ER PA DB = 0X%0x\n",
-				evt_ring_db_addr_low);
-		db_addr = ioremap((phys_addr_t)(evt_ring_db_addr_low), 4);
-
-		wp_addr = ipa_mpm_ctx->md[probe_id].dl_prod_ring.tr_pa +
-			((IPA_MPM_RING_LEN + 1) * GSI_EVT_RING_RE_SIZE_16B);
-		iowrite32(wp_addr, db_addr);
-		IPA_MPM_DBG("Host  DL ER  DB = 0X%pK, wp_addr = 0X%0x",
-			db_addr, wp_addr);
-		iounmap(db_addr);
-	}
-
-	/* Check if TETH connection is in progress.
-	 * If teth isn't started by now, then Stop UL channel.
-	 */
-	switch (ipa_mpm_ctx->md[probe_id].teth_state) {
-	case IPA_MPM_TETH_INIT:
-		if (ul_prod != IPA_CLIENT_MAX) {
-			/* No teth started yet, disable UL channel */
-			ipa_mpm_start_stop_mhip_chan(IPA_MPM_MHIP_CHAN_UL,
-						probe_id, MPM_MHIP_STOP);
-			/* Disable data path */
-			if (ipa_mpm_start_stop_ul_mhip_data_path(probe_id,
-				MPM_MHIP_STOP)) {
-				IPA_MPM_ERR("MHIP Enable data path failed\n");
-				goto fail_start_channel;
-			}
-		}
-		ipa_mpm_vote_unvote_pcie_clk(CLK_OFF, probe_id);
-		break;
-	case IPA_MPM_TETH_INPROGRESS:
-	case IPA_MPM_TETH_CONNECTED:
-		IPA_MPM_DBG("UL channel is already started, continue\n");
-		ipa_mpm_change_teth_state(probe_id, IPA_MPM_TETH_CONNECTED);
-
-		/* Enable data path */
-		if (ul_prod != IPA_CLIENT_MAX) {
-			if (ipa_mpm_start_stop_ul_mhip_data_path(probe_id,
-				MPM_MHIP_START)) {
-				IPA_MPM_ERR("MHIP Enable data path failed\n");
-				goto fail_start_channel;
-			}
-		}
-		/* Lift the delay for rmnet USB prod pipe */
-		if (probe_id == IPA_MPM_MHIP_CH_ID_1) {
-			pipe_idx = ipa3_get_ep_mapping(IPA_CLIENT_USB_PROD);
-			ipa3_xdci_ep_delay_rm(pipe_idx);
-		}
-		break;
-	default:
-		IPA_MPM_DBG("No op for UL channel, in teth state = %d",
-			ipa_mpm_ctx->md[probe_id].teth_state);
-		break;
-	}
-
-	atomic_inc(&ipa_mpm_ctx->probe_cnt);
-	ipa_mpm_ctx->md[probe_id].init_complete = true;
-	IPA_MPM_FUNC_EXIT();
-	return 0;
-
-fail_gsi_setup:
-fail_start_channel:
-fail_smmu:
-	if (ipa_mpm_ctx->dev_info.ipa_smmu_enabled)
-		IPA_MPM_DBG("SMMU failed\n");
-	ipa_mpm_vote_unvote_pcie_clk(CLK_OFF, probe_id);
-	ipa_mpm_vote_unvote_ipa_clk(CLK_OFF, probe_id);
-	ipa_assert();
-	return ret;
-}
-
-static void ipa_mpm_init_mhip_channel_info(void)
-{
-	/* IPA_MPM_MHIP_CH_ID_0 => MHIP TETH PIPES  */
-	ipa_mpm_pipes[IPA_MPM_MHIP_CH_ID_0].dl_cons.ipa_client =
-		IPA_CLIENT_MHI_PRIME_TETH_PROD;
-	ipa_mpm_pipes[IPA_MPM_MHIP_CH_ID_0].dl_cons.ep_cfg =
-		mhip_dl_teth_ep_cfg;
-	ipa_mpm_pipes[IPA_MPM_MHIP_CH_ID_0].ul_prod.ipa_client =
-		IPA_CLIENT_MHI_PRIME_TETH_CONS;
-	ipa_mpm_pipes[IPA_MPM_MHIP_CH_ID_0].ul_prod.ep_cfg =
-		mhip_ul_teth_ep_cfg;
-	ipa_mpm_pipes[IPA_MPM_MHIP_CH_ID_0].mhip_client =
-		IPA_MPM_MHIP_TETH;
-
-	/* IPA_MPM_MHIP_CH_ID_1 => MHIP RMNET PIPES */
-	ipa_mpm_pipes[IPA_MPM_MHIP_CH_ID_1].dl_cons.ipa_client =
-		IPA_CLIENT_MHI_PRIME_RMNET_PROD;
-	ipa_mpm_pipes[IPA_MPM_MHIP_CH_ID_1].dl_cons.ep_cfg =
-		mhip_dl_rmnet_ep_cfg;
-	ipa_mpm_pipes[IPA_MPM_MHIP_CH_ID_1].ul_prod.ipa_client =
-		IPA_CLIENT_MHI_PRIME_RMNET_CONS;
-	ipa_mpm_pipes[IPA_MPM_MHIP_CH_ID_1].ul_prod.ep_cfg =
-		mhip_ul_rmnet_ep_cfg;
-	ipa_mpm_pipes[IPA_MPM_MHIP_CH_ID_1].mhip_client =
-		IPA_MPM_MHIP_USB_RMNET;
-
-	/* IPA_MPM_MHIP_CH_ID_2 => MHIP ADPL PIPE */
-	ipa_mpm_pipes[IPA_MPM_MHIP_CH_ID_2].dl_cons.ipa_client =
-		IPA_CLIENT_MHI_PRIME_DPL_PROD;
-	ipa_mpm_pipes[IPA_MPM_MHIP_CH_ID_2].dl_cons.ep_cfg =
-		mhip_dl_dpl_ep_cfg;
-	ipa_mpm_pipes[IPA_MPM_MHIP_CH_ID_2].ul_prod.ipa_client =
-		IPA_CLIENT_MAX;
-	ipa_mpm_pipes[IPA_MPM_MHIP_CH_ID_2].mhip_client =
-		IPA_MPM_MHIP_USB_DPL;
-}
-
-static void ipa_mpm_mhi_remove_cb(struct mhi_device *mhi_dev)
-{
-	int mhip_idx;
-
-	IPA_MPM_FUNC_ENTRY();
-
-	for (mhip_idx = 0; mhip_idx < IPA_MPM_MHIP_CH_ID_MAX; mhip_idx++) {
-		if (mhi_dev == ipa_mpm_ctx->md[mhip_idx].mhi_dev)
-			break;
-	}
-	if (mhip_idx >= IPA_MPM_MHIP_CH_ID_MAX) {
-		IPA_MPM_DBG("remove_cb for mhip_idx = %d not probed before\n",
-			mhip_idx);
-		return;
-	}
-
-	IPA_MPM_DBG("remove_cb for mhip_idx = %d", mhip_idx);
-
-	ipa_mpm_mhip_shutdown(mhip_idx);
-	atomic_dec(&ipa_mpm_ctx->probe_cnt);
-
-	if (atomic_read(&ipa_mpm_ctx->probe_cnt) == 0) {
-		/* Last probe done, reset Everything here */
-		ipa_mpm_ctx->mhi_parent_dev = NULL;
-		ipa_mpm_ctx->carved_smmu_cb.next_addr =
-			ipa_mpm_ctx->carved_smmu_cb.va_start;
-		atomic_set(&ipa_mpm_ctx->pcie_clk_total_cnt, 0);
-		for (mhip_idx = 0;
-			mhip_idx < IPA_MPM_MHIP_CH_ID_MAX; mhip_idx++) {
-			atomic_set(
-				&ipa_mpm_ctx->md[mhip_idx].clk_cnt.pcie_clk_cnt,
-				0);
-		}
-	}
-
-	IPA_MPM_FUNC_EXIT();
-}
-
-static void ipa_mpm_mhi_status_cb(struct mhi_device *mhi_dev,
-				enum MHI_CB mhi_cb)
-{
-	int mhip_idx;
-	enum mhip_status_type status;
-
-	IPA_MPM_DBG("%d\n", mhi_cb);
-
-	for (mhip_idx = 0; mhip_idx < IPA_MPM_MHIP_CH_ID_MAX; mhip_idx++) {
-		if (mhi_dev == ipa_mpm_ctx->md[mhip_idx].mhi_dev)
-			break;
-	}
-	if (mhip_idx >= IPA_MPM_MHIP_CH_ID_MAX) {
-		IPA_MPM_DBG("ignoring secondary callbacks\n");
-		return;
-	}
-
-	mutex_lock(&ipa_mpm_ctx->md[mhip_idx].lpm_mutex);
-	switch (mhi_cb) {
-	case MHI_CB_IDLE:
-		break;
-	case MHI_CB_LPM_ENTER:
-		if (!ipa_mpm_ctx->md[mhip_idx].in_lpm) {
-			status = ipa_mpm_start_stop_mhip_chan(
-				IPA_MPM_MHIP_CHAN_DL,
-				mhip_idx, MPM_MHIP_STOP);
-			IPA_MPM_DBG("status = %d\n", status);
-			ipa_mpm_vote_unvote_ipa_clk(CLK_OFF, mhip_idx);
-			ipa_mpm_ctx->md[mhip_idx].in_lpm = true;
-		} else {
-			IPA_MPM_DBG("Already in lpm\n");
-		}
-		break;
-	case MHI_CB_LPM_EXIT:
-		if (ipa_mpm_ctx->md[mhip_idx].in_lpm) {
-			ipa_mpm_vote_unvote_ipa_clk(CLK_ON, mhip_idx);
-			status = ipa_mpm_start_stop_mhip_chan(
-				IPA_MPM_MHIP_CHAN_DL,
-				mhip_idx, MPM_MHIP_START);
-			IPA_MPM_DBG("status = %d\n", status);
-			ipa_mpm_ctx->md[mhip_idx].in_lpm = false;
-		} else {
-			IPA_MPM_DBG("Already out of lpm\n");
-		}
-		break;
-	case MHI_CB_EE_RDDM:
-	case MHI_CB_PENDING_DATA:
-	case MHI_CB_SYS_ERROR:
-	case MHI_CB_FATAL_ERROR:
-	case MHI_CB_BW_REQ:
-	case MHI_CB_EE_MISSION_MODE:
-		IPA_MPM_ERR("unexpected event %d\n", mhi_cb);
-		break;
-	}
-	mutex_unlock(&ipa_mpm_ctx->md[mhip_idx].lpm_mutex);
-}
-
-static int ipa_mpm_set_dma_mode(enum ipa_client_type src_pipe,
-	enum ipa_client_type dst_pipe)
-{
-	int result = 0;
-	struct ipa_ep_cfg ep_cfg = { { 0 } };
-
-	IPA_MPM_FUNC_ENTRY();
-	IPA_MPM_DBG("DMA from %d to %d\n", src_pipe, dst_pipe);
-
-	/* Set USB PROD PIPE DMA to MHIP PROD PIPE */
-	ep_cfg.mode.mode = IPA_DMA;
-	ep_cfg.mode.dst = dst_pipe;
-	ep_cfg.seq.set_dynamic = true;
-
-	result = ipa_cfg_ep(ipa_get_ep_mapping(src_pipe), &ep_cfg);
-	IPA_MPM_FUNC_EXIT();
-
-	return result;
-}
-
-int ipa_mpm_reset_dma_mode(enum ipa_client_type src_pipe,
-	enum ipa_client_type dst_pipe)
-{
-	int result = 0;
-	struct ipa_ep_cfg ep_cfg = { { 0 } };
-
-	IPA_MPM_FUNC_ENTRY();
-	IPA_MPM_DBG("DMA from %d to %d\n", src_pipe, dst_pipe);
-
-	/* Set USB PROD PIPE DMA to MHIP PROD PIPE */
-	ep_cfg.mode.mode = IPA_BASIC;
-	ep_cfg.mode.dst = IPA_CLIENT_APPS_LAN_CONS;
-	ep_cfg.seq.set_dynamic = true;
-
-	result = ipa_cfg_ep(ipa_get_ep_mapping(src_pipe), &ep_cfg);
-	IPA_MPM_FUNC_EXIT();
-
-	return result;
-}
-
-static void ipa_mpm_mhip_map_prot(enum ipa_usb_teth_prot prot,
-	enum ipa_mpm_mhip_client_type *mhip_client)
-{
-	switch (prot) {
-	case IPA_USB_RNDIS:
-		*mhip_client = IPA_MPM_MHIP_TETH;
-		break;
-	case IPA_USB_RMNET:
-		*mhip_client = IPA_MPM_MHIP_USB_RMNET;
-		break;
-	case IPA_USB_DIAG:
-		*mhip_client = IPA_MPM_MHIP_USB_DPL;
-		break;
-	default:
-		*mhip_client = IPA_MPM_MHIP_NONE;
-		break;
-	}
-	IPA_MPM_DBG("Mapped xdci prot %d -> MHIP prot %d\n", prot,
-		*mhip_client);
-}
-
-int ipa_mpm_mhip_xdci_pipe_enable(enum ipa_usb_teth_prot xdci_teth_prot)
-{
-	int probe_id = IPA_MPM_MHIP_CH_ID_MAX;
-	int i;
-	enum ipa_mpm_mhip_client_type mhip_client;
-	enum mhip_status_type status;
-	int ret = 0;
-	int pipe_idx;
-
-	if (ipa_mpm_ctx == NULL) {
-		IPA_MPM_ERR("MPM not platform probed yet, returning ..\n");
-		return 0;
-	}
-
-	ipa_mpm_mhip_map_prot(xdci_teth_prot, &mhip_client);
-
-	for (i = 0; i < IPA_MPM_MHIP_CH_ID_MAX; i++) {
-		if (ipa_mpm_pipes[i].mhip_client == mhip_client) {
-			probe_id = i;
-			break;
-		}
-	}
-
-	if (probe_id == IPA_MPM_MHIP_CH_ID_MAX) {
-		IPA_MPM_ERR("Unknown probe_id\n");
-		return 0;
-	}
-
-	IPA_MPM_DBG("Connect xdci prot %d -> mhip_client = %d probe_id = %d\n",
-			xdci_teth_prot, mhip_client, probe_id);
-
-	ipa_mpm_ctx->md[probe_id].mhip_client = mhip_client;
-
-	ret = ipa_mpm_vote_unvote_pcie_clk(CLK_ON, probe_id);
-	if (ret) {
-		IPA_MPM_ERR("Error cloking on PCIe clk, err = %d\n", ret);
-		return ret;
-	}
-
-	switch (mhip_client) {
-	case IPA_MPM_MHIP_USB_RMNET:
-		ipa_mpm_set_dma_mode(IPA_CLIENT_USB_PROD,
-			IPA_CLIENT_MHI_PRIME_RMNET_CONS);
-		break;
-	case IPA_MPM_MHIP_USB_DPL:
-		IPA_MPM_DBG("connecting DPL prot %d\n", mhip_client);
-		ipa_mpm_change_teth_state(probe_id, IPA_MPM_TETH_CONNECTED);
-		return 0;
-	default:
-		IPA_MPM_DBG("mhip_client = %d not processed\n", mhip_client);
-		ret = ipa_mpm_vote_unvote_pcie_clk(CLK_OFF, probe_id);
-		if (ret) {
-			IPA_MPM_ERR("Error unvoting on PCIe clk, err = %d\n",
-					ret);
-			return ret;
-		}
-		return 0;
-	}
-
-	if (mhip_client != IPA_MPM_MHIP_USB_DPL)
-		/* Start UL MHIP channel for offloading teth connection */
-		status = ipa_mpm_start_stop_mhip_chan(IPA_MPM_MHIP_CHAN_UL,
-							probe_id,
-							MPM_MHIP_START);
-	switch (status) {
-	case MHIP_STATUS_SUCCESS:
-	case MHIP_STATUS_NO_OP:
-		ipa_mpm_change_teth_state(probe_id, IPA_MPM_TETH_CONNECTED);
-		ipa_mpm_start_stop_ul_mhip_data_path(probe_id,
-						MPM_MHIP_START);
-
-		pipe_idx = ipa3_get_ep_mapping(IPA_CLIENT_USB_PROD);
-
-		/* Lift the delay for rmnet USB prod pipe */
-		ipa3_xdci_ep_delay_rm(pipe_idx);
-		if (status == MHIP_STATUS_NO_OP) {
-			/* Channels already have been started,
-			 * we can devote for pcie clocks
-			 */
-			ipa_mpm_vote_unvote_pcie_clk(CLK_OFF, probe_id);
-		}
-		break;
-	case MHIP_STATUS_EP_NOT_READY:
-		ipa_mpm_vote_unvote_pcie_clk(CLK_OFF, probe_id);
-		ipa_mpm_change_teth_state(probe_id, IPA_MPM_TETH_INPROGRESS);
-		break;
-	case MHIP_STATUS_FAIL:
-	case MHIP_STATUS_BAD_STATE:
-	case MHIP_STATUS_EP_NOT_FOUND:
-		IPA_MPM_ERR("UL chan cant be started err =%d\n", status);
-		ipa_mpm_vote_unvote_pcie_clk(CLK_OFF, probe_id);
-		ret = -EFAULT;
-		break;
-	default:
-		ipa_mpm_vote_unvote_pcie_clk(CLK_OFF, probe_id);
-		IPA_MPM_ERR("Err not found\n");
-		break;
-	}
-	return ret;
-}
-
-int ipa_mpm_mhip_xdci_pipe_disable(enum ipa_usb_teth_prot xdci_teth_prot)
-{
-	int probe_id = IPA_MPM_MHIP_CH_ID_MAX;
-	int i;
-	enum ipa_mpm_mhip_client_type mhip_client;
-	enum mhip_status_type status;
-	int ret = 0;
-
-	if (ipa_mpm_ctx == NULL) {
-		IPA_MPM_ERR("MPM not platform probed, returning ..\n");
-		return 0;
-	}
-
-	ipa_mpm_mhip_map_prot(xdci_teth_prot, &mhip_client);
-
-	for (i = 0; i < IPA_MPM_MHIP_CH_ID_MAX; i++) {
-		if (ipa_mpm_pipes[i].mhip_client == mhip_client) {
-			probe_id = i;
-			break;
-		}
-	}
-
-	if (probe_id == IPA_MPM_MHIP_CH_ID_MAX) {
-		IPA_MPM_ERR("Invalid probe_id\n");
-		return 0;
-	}
-
-	IPA_MPM_ERR("xdci disconnect prot %d mhip_client = %d probe_id = %d\n",
-			xdci_teth_prot, mhip_client, probe_id);
-
-	switch (mhip_client) {
-	case IPA_MPM_MHIP_USB_RMNET:
-		ret = ipa_mpm_reset_dma_mode(IPA_CLIENT_USB_PROD,
-			IPA_CLIENT_MHI_PRIME_RMNET_CONS);
-		if (ret) {
-			IPA_MPM_ERR("failed to reset dma mode\n");
-			return ret;
-		}
-		break;
-	case IPA_MPM_MHIP_TETH:
-		IPA_MPM_DBG("Rndis Disconnect, wait for wan_state ioctl\n");
-		return 0;
-	case IPA_MPM_MHIP_USB_DPL:
-		IPA_MPM_DBG("Teth Disconnecting for DPL\n");
-		ipa_mpm_change_teth_state(probe_id, IPA_MPM_TETH_INIT);
-		ret = ipa_mpm_vote_unvote_pcie_clk(CLK_OFF, probe_id);
-		if (ret)
-			IPA_MPM_ERR("Error cloking off PCIe clk, err = %d\n",
-				ret);
-		ipa_mpm_ctx->md[probe_id].mhip_client = IPA_MPM_MHIP_NONE;
-		return ret;
-	default:
-		IPA_MPM_ERR("mhip_client = %d not supported\n", mhip_client);
-		return 0;
-	}
-
-	status = ipa_mpm_start_stop_mhip_chan(IPA_MPM_MHIP_CHAN_UL,
-		probe_id, MPM_MHIP_STOP);
-
-	switch (status) {
-	case MHIP_STATUS_SUCCESS:
-	case MHIP_STATUS_NO_OP:
-	case MHIP_STATUS_EP_NOT_READY:
-		ipa_mpm_change_teth_state(probe_id, IPA_MPM_TETH_INIT);
-		ipa_mpm_start_stop_ul_mhip_data_path(probe_id, MPM_MHIP_STOP);
-		break;
-	case MHIP_STATUS_FAIL:
-	case MHIP_STATUS_BAD_STATE:
-	case MHIP_STATUS_EP_NOT_FOUND:
-		IPA_MPM_ERR("UL chan cant be started err =%d\n", status);
-		ipa_mpm_vote_unvote_pcie_clk(CLK_OFF, probe_id);
-		return -EFAULT;
-		break;
-	default:
-		IPA_MPM_ERR("Err not found\n");
-		break;
-	}
-
-	ret = ipa_mpm_vote_unvote_pcie_clk(CLK_OFF, probe_id);
-
-	if (ret) {
-		IPA_MPM_ERR("Error cloking off PCIe clk, err = %d\n", ret);
-		return ret;
-	}
-
-	ipa_mpm_ctx->md[probe_id].mhip_client = IPA_MPM_MHIP_NONE;
-
-	return ret;
-}
-
-static int ipa_mpm_populate_smmu_info(struct platform_device *pdev)
-{
-	struct ipa_smmu_in_params smmu_in;
-	struct ipa_smmu_out_params smmu_out;
-	u32 carved_iova_ap_mapping[2];
-	struct ipa_smmu_cb_ctx *cb;
-	struct ipa_smmu_cb_ctx *ap_cb = ipa3_get_smmu_ctx(IPA_SMMU_CB_AP);
-	int ret = 0;
-
-	if (ipa_mpm_ctx->carved_smmu_cb.valid) {
-		IPA_MPM_DBG("SMMU Context allocated, returning ..\n");
-		return ret;
-	}
-
-	cb = &ipa_mpm_ctx->carved_smmu_cb;
-
-	/* get IPA SMMU enabled status */
-	smmu_in.smmu_client = IPA_SMMU_AP_CLIENT;
-	if (ipa_get_smmu_params(&smmu_in, &smmu_out))
-		ipa_mpm_ctx->dev_info.ipa_smmu_enabled = false;
-	else
-		ipa_mpm_ctx->dev_info.ipa_smmu_enabled =
-		smmu_out.smmu_enable;
-
-	/* get cache_coherent enable or not */
-	ipa_mpm_ctx->dev_info.is_cache_coherent = ap_cb->is_cache_coherent;
-	if (of_property_read_u32_array(pdev->dev.of_node, "qcom,iova-mapping",
-		carved_iova_ap_mapping, 2)) {
-		IPA_MPM_ERR("failed to read of_node %s\n",
-			"qcom,mpm-iova-mapping");
-		return -EINVAL;
-	}
-	ipa_mpm_ctx->dev_info.pcie_smmu_enabled = true;
-
-	if (ipa_mpm_ctx->dev_info.ipa_smmu_enabled !=
-		ipa_mpm_ctx->dev_info.pcie_smmu_enabled) {
-		IPA_MPM_DBG("PCIE/IPA SMMU config mismatch\n");
-		return -EINVAL;
-	}
-
-	cb->va_start = carved_iova_ap_mapping[0];
-	cb->va_size = carved_iova_ap_mapping[1];
-	cb->va_end = cb->va_start + cb->va_size;
-
-	if (cb->va_end >= ap_cb->va_start) {
-		IPA_MPM_ERR("MPM iommu and AP overlap addr 0x%lx\n",
-				cb->va_start);
-		ipa_assert();
-		return -EFAULT;
-	}
-
-	cb->dev = ipa_mpm_ctx->dev_info.dev;
-	cb->valid = true;
-	cb->next_addr = cb->va_start;
-
-	if (dma_set_mask_and_coherent(ipa_mpm_ctx->dev_info.dev,
-		DMA_BIT_MASK(64))) {
-		IPA_MPM_ERR("setting DMA mask to 64 failed.\n");
-		return -EINVAL;
-	}
-
-	return ret;
-}
-
-static int ipa_mpm_probe(struct platform_device *pdev)
-{
-	int ret = 0;
-	int i = 0;
-	int idx = 0;
-
-	IPA_MPM_FUNC_ENTRY();
-
-	if (ipa_mpm_ctx) {
-		IPA_MPM_DBG("MPM is already probed, returning\n");
-		return 0;
-	}
-
-	ret = ipa_register_ipa_ready_cb(ipa_mpm_ipa3_ready_cb, (void *)pdev);
-	/*
-	 * If we received -EEXIST, IPA has initialized. So we need
-	 * to continue the probing process.
-	 */
-	if (!ret) {
-		IPA_MPM_DBG("IPA not ready yet, registering callback\n");
-		return ret;
-	}
-	IPA_MPM_DBG("IPA is ready, continue with probe\n");
-
-	ipa_mpm_ctx = kzalloc(sizeof(*ipa_mpm_ctx), GFP_KERNEL);
-
-	if (!ipa_mpm_ctx)
-		return -ENOMEM;
-
-	for (i = 0; i < IPA_MPM_MHIP_CH_ID_MAX; i++) {
-		mutex_init(&ipa_mpm_ctx->md[i].mutex);
-		mutex_init(&ipa_mpm_ctx->md[i].lpm_mutex);
-	}
-
-	ipa_mpm_ctx->dev_info.pdev = pdev;
-	ipa_mpm_ctx->dev_info.dev = &pdev->dev;
-
-	ipa_mpm_init_mhip_channel_info();
-
-	if (of_property_read_u32(pdev->dev.of_node, "qcom,mhi-chdb-base",
-		&ipa_mpm_ctx->dev_info.chdb_base)) {
-		IPA_MPM_ERR("failed to read qcom,mhi-chdb-base\n");
-		goto fail_probe;
-	}
-	IPA_MPM_DBG("chdb-base=0x%x\n", ipa_mpm_ctx->dev_info.chdb_base);
-
-	if (of_property_read_u32(pdev->dev.of_node, "qcom,mhi-erdb-base",
-		&ipa_mpm_ctx->dev_info.erdb_base)) {
-		IPA_MPM_ERR("failed to read qcom,mhi-erdb-base\n");
-		goto fail_probe;
-	}
-	IPA_MPM_DBG("erdb-base=0x%x\n", ipa_mpm_ctx->dev_info.erdb_base);
-
-	ret = ipa_mpm_populate_smmu_info(pdev);
-
-	if (ret) {
-		IPA_MPM_DBG("SMMU Config failed\n");
-		goto fail_probe;
-	}
-
-	atomic_set(&ipa_mpm_ctx->ipa_clk_total_cnt, 0);
-	atomic_set(&ipa_mpm_ctx->pcie_clk_total_cnt, 0);
-
-	for (idx = 0; idx < IPA_MPM_MHIP_CH_ID_MAX; idx++) {
-		ipa_mpm_ctx->md[idx].ul_prod.gsi_state = GSI_INIT;
-		ipa_mpm_ctx->md[idx].dl_cons.gsi_state = GSI_INIT;
-		atomic_set(&ipa_mpm_ctx->md[idx].clk_cnt.ipa_clk_cnt, 0);
-		atomic_set(&ipa_mpm_ctx->md[idx].clk_cnt.pcie_clk_cnt, 0);
-	}
-
-	ret = mhi_driver_register(&mhi_driver);
-	if (ret) {
-		IPA_MPM_ERR("mhi_driver_register failed %d\n", ret);
-		goto fail_probe;
-	}
-	IPA_MPM_FUNC_EXIT();
-	return 0;
-
-fail_probe:
-	kfree(ipa_mpm_ctx);
-	ipa_mpm_ctx = NULL;
-	return -EFAULT;
-}
-
-static int ipa_mpm_remove(struct platform_device *pdev)
-{
-	IPA_MPM_FUNC_ENTRY();
-
-	mhi_driver_unregister(&mhi_driver);
-	IPA_MPM_FUNC_EXIT();
-	return 0;
-}
-
-static const struct of_device_id ipa_mpm_dt_match[] = {
-	{ .compatible = "qcom,ipa-mpm" },
-	{},
-};
-MODULE_DEVICE_TABLE(of, ipa_mpm_dt_match);
-
-static struct platform_driver ipa_ipa_mpm_driver = {
-	.driver = {
-		.name = "ipa_mpm",
-		.of_match_table = ipa_mpm_dt_match,
-	},
-	.probe = ipa_mpm_probe,
-	.remove = ipa_mpm_remove,
-};
-
-/**
- * ipa_mpm_init() - Registers ipa_mpm as a platform device for a APQ
- *
- * This function is called after bootup for APQ device.
- * ipa_mpm will register itself as a platform device, and probe
- * function will get called.
- *
- * Return: None
- */
-static int __init ipa_mpm_init(void)
-{
-	IPA_MPM_DBG("register ipa_mpm platform device\n");
-	return platform_driver_register(&ipa_ipa_mpm_driver);
-}
-
-/**
- * ipa3_is_mhip_offload_enabled() - check if IPA MPM module was initialized
- * successfully. If it is initialized, MHIP is enabled for teth
- *
- * Return value: 1 for yes; 0 for no
- */
-int ipa3_is_mhip_offload_enabled(void)
-{
-	if (ipa_mpm_ctx == NULL)
-		return 0;
-	else
-		return 1;
-}
-
-int ipa_mpm_panic_handler(char *buf, int size)
-{
-	int i;
-	int cnt = 0;
-
-	cnt = scnprintf(buf, size,
-			"\n---- MHIP Active Clients Table ----\n");
-	cnt += scnprintf(buf + cnt, size - cnt,
-			"Total PCIe active clients count: %d\n",
-			atomic_read(&ipa_mpm_ctx->pcie_clk_total_cnt));
-	cnt += scnprintf(buf + cnt, size - cnt,
-			"Total IPA active clients count: %d\n",
-			atomic_read(&ipa_mpm_ctx->ipa_clk_total_cnt));
-
-	for (i = 0; i < IPA_MPM_MHIP_CH_ID_MAX; i++) {
-		cnt += scnprintf(buf + cnt, size - cnt,
-			"client id: %d ipa vote cnt: %d pcie vote cnt\n", i,
-			atomic_read(&ipa_mpm_ctx->md[i].clk_cnt.ipa_clk_cnt),
-			atomic_read(&ipa_mpm_ctx->md[i].clk_cnt.pcie_clk_cnt));
-	}
-	return cnt;
-}
-
-/**
- * ipa3_get_mhip_gsi_stats() - Query MHIP gsi stats from uc
- * @stats:	[inout] stats blob from client populated by driver
- *
- * Returns:	0 on success, negative on failure
- *
- * @note Cannot be called from atomic context
- *
- */
-int ipa3_get_mhip_gsi_stats(struct ipa3_uc_dbg_ring_stats *stats)
-{
-	int i;
-
-	if (!ipa3_ctx->mhip_ctx.dbg_stats.uc_dbg_stats_mmio) {
-		IPAERR("bad parms NULL mhip_gsi_stats_mmio\n");
-		return -EINVAL;
-	}
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-	for (i = 0; i < MAX_MHIP_CHANNELS; i++) {
-		stats->ring[i].ringFull = ioread32(
-			ipa3_ctx->mhip_ctx.dbg_stats.uc_dbg_stats_mmio
-			+ i * IPA3_UC_DEBUG_STATS_OFF +
-			IPA3_UC_DEBUG_STATS_RINGFULL_OFF);
-		stats->ring[i].ringEmpty = ioread32(
-			ipa3_ctx->mhip_ctx.dbg_stats.uc_dbg_stats_mmio
-			+ i * IPA3_UC_DEBUG_STATS_OFF +
-			IPA3_UC_DEBUG_STATS_RINGEMPTY_OFF);
-		stats->ring[i].ringUsageHigh = ioread32(
-			ipa3_ctx->mhip_ctx.dbg_stats.uc_dbg_stats_mmio
-			+ i * IPA3_UC_DEBUG_STATS_OFF +
-			IPA3_UC_DEBUG_STATS_RINGUSAGEHIGH_OFF);
-		stats->ring[i].ringUsageLow = ioread32(
-			ipa3_ctx->mhip_ctx.dbg_stats.uc_dbg_stats_mmio
-			+ i * IPA3_UC_DEBUG_STATS_OFF +
-			IPA3_UC_DEBUG_STATS_RINGUSAGELOW_OFF);
-		stats->ring[i].RingUtilCount = ioread32(
-			ipa3_ctx->mhip_ctx.dbg_stats.uc_dbg_stats_mmio
-			+ i * IPA3_UC_DEBUG_STATS_OFF +
-			IPA3_UC_DEBUG_STATS_RINGUTILCOUNT_OFF);
-	}
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-
-
-	return 0;
-}
-
-
-late_initcall(ipa_mpm_init);
-MODULE_LICENSE("GPL v2");
-MODULE_DESCRIPTION("MHI Proxy Manager Driver");

ipa/ipa_v3/ipa_nat.c

@@ -1,1630 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2012-2019, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/device.h>
-#include <linux/fs.h>
-#include <linux/init.h>
-#include <linux/kernel.h>
-#include <linux/mm.h>
-#include <linux/uaccess.h>
-#include "ipa_i.h"
-#include "ipahal/ipahal.h"
-#include "ipahal/ipahal_nat.h"
-
-/*
- * The following for adding code (ie. for EMULATION) not found on x86.
- */
-#if defined(CONFIG_IPA_EMULATION)
-# include "ipa_emulation_stubs.h"
-#endif
-
-#define IPA_NAT_PHYS_MEM_OFFSET  0
-#define IPA_IPV6CT_PHYS_MEM_OFFSET  0
-#define IPA_NAT_PHYS_MEM_SIZE  IPA_RAM_NAT_SIZE
-#define IPA_IPV6CT_PHYS_MEM_SIZE  IPA_RAM_IPV6CT_SIZE
-
-#define IPA_NAT_IPV6CT_TEMP_MEM_SIZE 128
-
-#define IPA_NAT_MAX_NUM_OF_INIT_CMD_DESC 3
-#define IPA_IPV6CT_MAX_NUM_OF_INIT_CMD_DESC 2
-#define IPA_MAX_NUM_OF_TABLE_DMA_CMD_DESC 4
-
-/*
- * The base table max entries is limited by index into table 13 bits number.
- * Limit the memory size required by user to prevent kernel memory starvation
- */
-#define IPA_TABLE_MAX_ENTRIES 8192
-#define MAX_ALLOC_NAT_SIZE(size) (IPA_TABLE_MAX_ENTRIES * size)
-
-enum ipa_nat_ipv6ct_table_type {
-	IPA_NAT_BASE_TBL = 0,
-	IPA_NAT_EXPN_TBL = 1,
-	IPA_NAT_INDX_TBL = 2,
-	IPA_NAT_INDEX_EXPN_TBL = 3,
-	IPA_IPV6CT_BASE_TBL = 4,
-	IPA_IPV6CT_EXPN_TBL = 5
-};
-
-static int ipa3_nat_ipv6ct_vma_fault_remap(struct vm_fault *vmf)
-{
-	vmf->page = NULL;
-
-	IPADBG("\n");
-	return VM_FAULT_SIGBUS;
-}
-
-/* VMA related file operations functions */
-static const struct vm_operations_struct ipa3_nat_ipv6ct_remap_vm_ops = {
-	.fault = ipa3_nat_ipv6ct_vma_fault_remap,
-};
-
-static int ipa3_nat_ipv6ct_open(struct inode *inode, struct file *filp)
-{
-	struct ipa3_nat_ipv6ct_common_mem *dev;
-
-	IPADBG("\n");
-	dev = container_of(inode->i_cdev,
-		struct ipa3_nat_ipv6ct_common_mem, cdev);
-	filp->private_data = dev;
-	IPADBG("return\n");
-
-	return 0;
-}
-
-static int ipa3_nat_ipv6ct_mmap(struct file *filp, struct vm_area_struct *vma)
-{
-	struct ipa3_nat_ipv6ct_common_mem *dev =
-		(struct ipa3_nat_ipv6ct_common_mem *)filp->private_data;
-	unsigned long vsize = vma->vm_end - vma->vm_start;
-	unsigned long phys_addr;
-	int result = 0;
-	struct ipa_smmu_cb_ctx *cb = ipa3_get_smmu_ctx(IPA_SMMU_CB_AP);
-
-	IPADBG("\n");
-
-	if (!dev->is_dev_init) {
-		IPAERR("attempt to mmap %s before dev init\n", dev->name);
-		return -EPERM;
-	}
-
-	mutex_lock(&dev->lock);
-	if (!dev->is_mem_allocated) {
-		IPAERR_RL("attempt to mmap %s before the memory allocation\n",
-			dev->name);
-		result = -EPERM;
-		goto bail;
-	}
-
-	if (dev->is_sys_mem) {
-		if (dev->is_mapped) {
-			IPAERR("%s already mapped, only 1 mapping supported\n",
-				dev->name);
-			result = -EINVAL;
-			goto bail;
-		}
-	} else {
-		if ((dev->phys_mem_size == 0) || (vsize > dev->phys_mem_size)) {
-			IPAERR_RL("wrong parameters to %s mapping\n",
-				dev->name);
-			result = -EINVAL;
-			goto bail;
-		}
-	}
-	/* check if smmu enable & dma_coherent mode */
-	if (!cb->valid ||
-		!is_device_dma_coherent(cb->dev)) {
-		vma->vm_page_prot =
-		pgprot_noncached(vma->vm_page_prot);
-		IPADBG("App smmu enable in DMA mode\n");
-	}
-
-	if (dev->is_sys_mem) {
-		IPADBG("Mapping system memory\n");
-		IPADBG("map sz=0x%zx\n", dev->size);
-		result =
-			dma_mmap_coherent(
-				ipa3_ctx->pdev, vma,
-				dev->vaddr, dev->dma_handle,
-				dev->size);
-		if (result) {
-			IPAERR("unable to map memory. Err:%d\n", result);
-			goto bail;
-		}
-		dev->base_address = dev->vaddr;
-	} else {
-		IPADBG("Mapping shared(local) memory\n");
-		IPADBG("map sz=0x%lx\n", vsize);
-
-		phys_addr = ipa3_ctx->ipa_wrapper_base +
-			ipa3_ctx->ctrl->ipa_reg_base_ofst +
-			ipahal_get_reg_n_ofst(IPA_SW_AREA_RAM_DIRECT_ACCESS_n,
-				dev->smem_offset);
-
-		if (remap_pfn_range(
-			vma, vma->vm_start,
-			phys_addr >> PAGE_SHIFT, vsize, vma->vm_page_prot)) {
-			IPAERR("remap failed\n");
-			result = -EAGAIN;
-			goto bail;
-		}
-		dev->base_address = (void *)vma->vm_start;
-	}
-	result = 0;
-	vma->vm_ops = &ipa3_nat_ipv6ct_remap_vm_ops;
-	dev->is_mapped = true;
-	IPADBG("return\n");
-bail:
-	mutex_unlock(&dev->lock);
-	return result;
-}
-
-static const struct file_operations ipa3_nat_ipv6ct_fops = {
-	.owner = THIS_MODULE,
-	.open = ipa3_nat_ipv6ct_open,
-	.mmap = ipa3_nat_ipv6ct_mmap
-};
-
-/**
- * ipa3_allocate_nat_ipv6ct_tmp_memory() - Allocates the NAT\IPv6CT temp memory
- */
-static struct ipa3_nat_ipv6ct_tmp_mem *ipa3_nat_ipv6ct_allocate_tmp_memory(void)
-{
-	struct ipa3_nat_ipv6ct_tmp_mem *tmp_mem;
-	gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO;
-
-	IPADBG("\n");
-
-	tmp_mem = kzalloc(sizeof(*tmp_mem), GFP_KERNEL);
-	if (tmp_mem == NULL)
-		return NULL;
-
-	tmp_mem->vaddr =
-		dma_alloc_coherent(ipa3_ctx->pdev, IPA_NAT_IPV6CT_TEMP_MEM_SIZE,
-			&tmp_mem->dma_handle, gfp_flags);
-	if (tmp_mem->vaddr == NULL)
-		goto bail_tmp_mem;
-
-	IPADBG("IPA successfully allocated temp memory\n");
-	return tmp_mem;
-
-bail_tmp_mem:
-	kfree(tmp_mem);
-	return NULL;
-}
-
-static int ipa3_nat_ipv6ct_init_device(
-	struct ipa3_nat_ipv6ct_common_mem *dev,
-	const char *name,
-	u32 phys_mem_size,
-	u32 smem_offset,
-	struct ipa3_nat_ipv6ct_tmp_mem *tmp_mem)
-{
-	int result;
-
-	IPADBG("Init %s\n", name);
-
-	if (strnlen(name, IPA_DEV_NAME_MAX_LEN) == IPA_DEV_NAME_MAX_LEN) {
-		IPAERR("device name is too long\n");
-		return -ENODEV;
-	}
-
-	strlcpy(dev->name, name, IPA_DEV_NAME_MAX_LEN);
-
-	dev->class = class_create(THIS_MODULE, name);
-	if (IS_ERR(dev->class)) {
-		IPAERR("unable to create the class for %s\n", name);
-		return -ENODEV;
-	}
-	result = alloc_chrdev_region(&dev->dev_num, 0, 1, name);
-	if (result) {
-		IPAERR("alloc_chrdev_region err. for %s\n", name);
-		result = -ENODEV;
-		goto alloc_chrdev_region_fail;
-	}
-
-	dev->dev = device_create(dev->class, NULL, dev->dev_num, NULL, name);
-
-	if (IS_ERR(dev->dev)) {
-		IPAERR("device_create err:%ld\n", PTR_ERR(dev->dev));
-		result = -ENODEV;
-		goto device_create_fail;
-	}
-
-	cdev_init(&dev->cdev, &ipa3_nat_ipv6ct_fops);
-	dev->cdev.owner = THIS_MODULE;
-
-	mutex_init(&dev->lock);
-	mutex_lock(&dev->lock);
-
-	result = cdev_add(&dev->cdev, dev->dev_num, 1);
-	if (result) {
-		IPAERR("cdev_add err=%d\n", -result);
-		goto cdev_add_fail;
-	}
-
-	dev->phys_mem_size = phys_mem_size;
-	dev->smem_offset = smem_offset;
-
-	dev->is_dev_init = true;
-	dev->tmp_mem = tmp_mem;
-	mutex_unlock(&dev->lock);
-
-	IPADBG("ipa dev %s added successful. major:%d minor:%d\n", name,
-		MAJOR(dev->dev_num), MINOR(dev->dev_num));
-	return 0;
-
-cdev_add_fail:
-	mutex_unlock(&dev->lock);
-	device_destroy(dev->class, dev->dev_num);
-device_create_fail:
-	unregister_chrdev_region(dev->dev_num, 1);
-alloc_chrdev_region_fail:
-	class_destroy(dev->class);
-	return result;
-}
-
-static void ipa3_nat_ipv6ct_destroy_device(
-	struct ipa3_nat_ipv6ct_common_mem *dev)
-{
-	IPADBG("\n");
-
-	mutex_lock(&dev->lock);
-
-	if (dev->tmp_mem != NULL &&
-		!(ipa3_ctx->nat_mem.is_tmp_mem_allocated)) {
-		dev->tmp_mem = NULL;
-	} else if (dev->tmp_mem != NULL &&
-		ipa3_ctx->nat_mem.is_tmp_mem_allocated) {
-		dma_free_coherent(ipa3_ctx->pdev, IPA_NAT_IPV6CT_TEMP_MEM_SIZE,
-			dev->tmp_mem->vaddr, dev->tmp_mem->dma_handle);
-		kfree(dev->tmp_mem);
-		dev->tmp_mem = NULL;
-		ipa3_ctx->nat_mem.is_tmp_mem_allocated = false;
-	}
-	device_destroy(dev->class, dev->dev_num);
-	unregister_chrdev_region(dev->dev_num, 1);
-	class_destroy(dev->class);
-	dev->is_dev_init = false;
-	mutex_unlock(&dev->lock);
-
-	IPADBG("return\n");
-}
-
-/**
- * ipa3_nat_ipv6ct_init_devices() - Initialize the NAT and IPv6CT devices
- *
- * Called during IPA init to create memory device
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa3_nat_ipv6ct_init_devices(void)
-{
-	struct ipa3_nat_ipv6ct_tmp_mem *tmp_mem;
-	int result;
-
-	IPADBG("\n");
-
-	/*
-	 * Allocate NAT/IPv6CT temporary memory. The memory is never deleted,
-	 * because provided to HW once NAT or IPv6CT table is deleted.
-	 */
-	tmp_mem = ipa3_nat_ipv6ct_allocate_tmp_memory();
-
-	if (tmp_mem == NULL) {
-		IPAERR("unable to allocate tmp_mem\n");
-		return -ENOMEM;
-	}
-	ipa3_ctx->nat_mem.is_tmp_mem_allocated = true;
-
-	if (ipa3_nat_ipv6ct_init_device(
-		&ipa3_ctx->nat_mem.dev,
-		IPA_NAT_DEV_NAME,
-		IPA_NAT_PHYS_MEM_SIZE,
-		IPA_NAT_PHYS_MEM_OFFSET,
-		tmp_mem)) {
-		IPAERR("unable to create nat device\n");
-		result = -ENODEV;
-		goto fail_init_nat_dev;
-	}
-
-	if ((ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0) &&
-		ipa3_nat_ipv6ct_init_device(
-			&ipa3_ctx->ipv6ct_mem.dev,
-			IPA_IPV6CT_DEV_NAME,
-			IPA_IPV6CT_PHYS_MEM_SIZE,
-			IPA_IPV6CT_PHYS_MEM_OFFSET,
-			tmp_mem)) {
-		IPAERR("unable to create IPv6CT device\n");
-		result = -ENODEV;
-		goto fail_init_ipv6ct_dev;
-	}
-
-	return 0;
-
-fail_init_ipv6ct_dev:
-	ipa3_nat_ipv6ct_destroy_device(&ipa3_ctx->nat_mem.dev);
-fail_init_nat_dev:
-	if (tmp_mem != NULL && ipa3_ctx->nat_mem.is_tmp_mem_allocated) {
-		dma_free_coherent(ipa3_ctx->pdev, IPA_NAT_IPV6CT_TEMP_MEM_SIZE,
-			tmp_mem->vaddr, tmp_mem->dma_handle);
-		kfree(tmp_mem);
-		ipa3_ctx->nat_mem.is_tmp_mem_allocated = false;
-	}
-	return result;
-}
-
-/**
- * ipa3_nat_ipv6ct_destroy_devices() - destroy the NAT and IPv6CT devices
- *
- * Called during IPA init to destroy nat device
- */
-void ipa3_nat_ipv6ct_destroy_devices(void)
-{
-	ipa3_nat_ipv6ct_destroy_device(&ipa3_ctx->nat_mem.dev);
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0)
-		ipa3_nat_ipv6ct_destroy_device(&ipa3_ctx->ipv6ct_mem.dev);
-}
-
-static int ipa3_nat_ipv6ct_allocate_mem(struct ipa3_nat_ipv6ct_common_mem *dev,
-	struct ipa_ioc_nat_ipv6ct_table_alloc *table_alloc,
-	enum ipahal_nat_type nat_type)
-{
-	gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO;
-	int result = 0;
-	size_t nat_entry_size;
-
-	IPADBG("passed memory size %zu for %s\n",
-		table_alloc->size, dev->name);
-
-	if (!dev->is_dev_init) {
-		IPAERR("%s hasn't been initialized\n", dev->name);
-		result = -EPERM;
-		goto bail;
-	}
-
-	if (dev->is_mem_allocated) {
-		IPAERR("Memory already allocated\n");
-		result = -EPERM;
-		goto bail;
-	}
-
-	ipahal_nat_entry_size(nat_type, &nat_entry_size);
-	if (table_alloc->size > MAX_ALLOC_NAT_SIZE(nat_entry_size)) {
-		IPAERR("Trying allocate more size = %zu, Max allowed = %zu\n",
-				table_alloc->size,
-				MAX_ALLOC_NAT_SIZE(nat_entry_size));
-		result = -EPERM;
-		goto bail;
-	}
-
-	if (!table_alloc->size) {
-		IPAERR_RL("Invalid Parameters\n");
-		result = -EPERM;
-		goto bail;
-	}
-
-	if (table_alloc->size > IPA_NAT_PHYS_MEM_SIZE) {
-		IPADBG("Allocating system memory\n");
-		dev->is_sys_mem = true;
-		dev->vaddr =
-		   dma_alloc_coherent(ipa3_ctx->pdev, table_alloc->size,
-			   &dev->dma_handle, gfp_flags);
-		if (dev->vaddr == NULL) {
-			IPAERR("memory alloc failed\n");
-			result = -ENOMEM;
-			goto bail;
-		}
-		dev->size = table_alloc->size;
-	} else {
-		IPADBG("using shared(local) memory\n");
-		dev->is_sys_mem = false;
-	}
-
-	IPADBG("return\n");
-
-bail:
-	return result;
-}
-
-/**
- * ipa3_allocate_nat_device() - Allocates memory for the NAT device
- * @mem:	[in/out] memory parameters
- *
- * Called by NAT client driver to allocate memory for the NAT entries. Based on
- * the request size either shared or system memory will be used.
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa3_allocate_nat_device(struct ipa_ioc_nat_alloc_mem *mem)
-{
-	int result;
-	struct ipa_ioc_nat_ipv6ct_table_alloc tmp;
-
-	tmp.size = mem->size;
-	tmp.offset = 0;
-
-	result = ipa3_allocate_nat_table(&tmp);
-	if (result)
-		goto bail;
-
-	mem->offset = tmp.offset;
-
-bail:
-	return result;
-}
-
-/**
- * ipa3_allocate_nat_table() - Allocates memory for the NAT table
- * @table_alloc: [in/out] memory parameters
- *
- * Called by NAT client to allocate memory for the table entries.
- * Based on the request size either shared or system memory will be used.
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa3_allocate_nat_table(struct ipa_ioc_nat_ipv6ct_table_alloc *table_alloc)
-{
-	struct ipa3_nat_mem *nat_ctx = &(ipa3_ctx->nat_mem);
-	gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO;
-	int result;
-
-	IPADBG("\n");
-
-	mutex_lock(&nat_ctx->dev.lock);
-
-	result = ipa3_nat_ipv6ct_allocate_mem(&nat_ctx->dev, table_alloc,
-							IPAHAL_NAT_IPV4);
-	if (result)
-		goto bail;
-
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0) {
-		size_t pdn_entry_size;
-		struct ipa_mem_buffer *pdn_mem = &nat_ctx->pdn_mem;
-
-		ipahal_nat_entry_size(IPAHAL_NAT_IPV4_PDN, &pdn_entry_size);
-		pdn_mem->size = pdn_entry_size * IPA_MAX_PDN_NUM;
-		if (IPA_MEM_PART(pdn_config_size) < pdn_mem->size) {
-			IPAERR(
-				"number of PDN entries exceeds SRAM available space\n");
-			result = -ENOMEM;
-			goto fail_alloc_pdn;
-		}
-
-		pdn_mem->base = dma_zalloc_coherent(ipa3_ctx->pdev,
-			pdn_mem->size,
-			&pdn_mem->phys_base,
-			gfp_flags);
-		if (!pdn_mem->base) {
-			IPAERR("fail to allocate PDN memory\n");
-			result = -ENOMEM;
-			goto fail_alloc_pdn;
-		}
-		IPADBG("IPA NAT dev allocated PDN memory successfully\n");
-	}
-
-	nat_ctx->dev.is_mem_allocated = true;
-	IPADBG("IPA NAT dev init successfully\n");
-	mutex_unlock(&nat_ctx->dev.lock);
-
-	IPADBG("return\n");
-
-	return 0;
-
-fail_alloc_pdn:
-	if (nat_ctx->dev.vaddr) {
-		dma_free_coherent(ipa3_ctx->pdev, table_alloc->size,
-			nat_ctx->dev.vaddr, nat_ctx->dev.dma_handle);
-		nat_ctx->dev.vaddr = NULL;
-	}
-bail:
-	mutex_unlock(&nat_ctx->dev.lock);
-
-	return result;
-}
-
-/**
- * ipa3_allocate_ipv6ct_table() - Allocates memory for the IPv6CT table
- * @table_alloc: [in/out] memory parameters
- *
- * Called by IPv6CT client to allocate memory for the table entries.
- * Based on the request size either shared or system memory will be used.
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa3_allocate_ipv6ct_table(
-	struct ipa_ioc_nat_ipv6ct_table_alloc *table_alloc)
-{
-	int result;
-
-	IPADBG("\n");
-
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_0) {
-		IPAERR_RL("IPv6 connection tracking isn't supported\n");
-		return -EPERM;
-	}
-
-	mutex_lock(&ipa3_ctx->ipv6ct_mem.dev.lock);
-
-	result = ipa3_nat_ipv6ct_allocate_mem(
-		&ipa3_ctx->ipv6ct_mem.dev, table_alloc, IPAHAL_NAT_IPV6CT);
-	if (result)
-		goto bail;
-
-	ipa3_ctx->ipv6ct_mem.dev.is_mem_allocated = true;
-	IPADBG("IPA IPv6CT dev init successfully\n");
-
-bail:
-	mutex_unlock(&ipa3_ctx->ipv6ct_mem.dev.lock);
-	return result;
-}
-
-static int ipa3_nat_ipv6ct_check_table_params(
-	struct ipa3_nat_ipv6ct_common_mem *dev,
-	uint32_t offset, uint16_t entries_num,
-	enum ipahal_nat_type nat_type)
-{
-	int result;
-	size_t entry_size, table_size;
-
-	result = ipahal_nat_entry_size(nat_type, &entry_size);
-	if (result) {
-		IPAERR("Failed to retrieve size of entry for %s\n",
-			ipahal_nat_type_str(nat_type));
-		return result;
-	}
-	table_size = entry_size * entries_num;
-
-	/* check for integer overflow */
-	if (offset > UINT_MAX - table_size) {
-		IPAERR_RL("Detected overflow\n");
-		return -EPERM;
-	}
-
-	/* Check offset is not beyond allocated size */
-	if (dev->size < offset + table_size) {
-		IPAERR_RL("Table offset not valid\n");
-		IPAERR_RL("offset:%d entries:%d table_size:%zu mem_size:%zu\n",
-			offset, entries_num, table_size, dev->size);
-		return -EPERM;
-	}
-
-	if (dev->is_sys_mem && offset > UINT_MAX - dev->dma_handle) {
-		IPAERR_RL("Failed due to integer overflow\n");
-		IPAERR_RL("%s dma_handle: 0x%pa offset: 0x%x\n",
-			dev->name, &dev->dma_handle, offset);
-		return -EPERM;
-	}
-
-	return 0;
-}
-
-static inline void ipa3_nat_ipv6ct_create_init_cmd(
-	struct ipahal_imm_cmd_nat_ipv6ct_init_common *table_init_cmd,
-	bool is_shared,
-	dma_addr_t base_addr,
-	uint8_t tbl_index,
-	uint32_t base_table_offset,
-	uint32_t expn_table_offset,
-	uint16_t table_entries,
-	uint16_t expn_table_entries,
-	const char *table_name)
-{
-	table_init_cmd->base_table_addr_shared = is_shared;
-	table_init_cmd->expansion_table_addr_shared = is_shared;
-
-	table_init_cmd->base_table_addr = base_addr + base_table_offset;
-	IPADBG("%s base table offset:0x%x\n", table_name, base_table_offset);
-
-	table_init_cmd->expansion_table_addr = base_addr + expn_table_offset;
-	IPADBG("%s expn table offset:0x%x\n", table_name, expn_table_offset);
-
-	table_init_cmd->table_index = tbl_index;
-	IPADBG("%s table index:0x%x\n", table_name, tbl_index);
-
-	table_init_cmd->size_base_table = table_entries;
-	IPADBG("%s base table size:0x%x\n", table_name, table_entries);
-
-	table_init_cmd->size_expansion_table = expn_table_entries;
-	IPADBG("%s expansion table size:0x%x\n",
-		table_name, expn_table_entries);
-}
-
-static inline void ipa3_nat_ipv6ct_init_device_structure(
-	struct ipa3_nat_ipv6ct_common_mem *dev,
-	uint32_t base_table_offset,
-	uint32_t expn_table_offset,
-	uint16_t table_entries,
-	uint16_t expn_table_entries)
-{
-	dev->base_table_addr = (char *)dev->base_address + base_table_offset;
-	IPADBG("%s base_table_addr: 0x%pK\n", dev->name, dev->base_table_addr);
-
-	dev->expansion_table_addr =
-		(char *)dev->base_address + expn_table_offset;
-	IPADBG("%s expansion_table_addr: 0x%pK\n",
-		dev->name, dev->expansion_table_addr);
-
-	IPADBG("%s table_entries: %d\n", dev->name, table_entries);
-	dev->table_entries = table_entries;
-
-	IPADBG("%s expn_table_entries: %d\n", dev->name, expn_table_entries);
-	dev->expn_table_entries = expn_table_entries;
-}
-
-static void ipa3_nat_create_init_cmd(
-	struct ipa_ioc_v4_nat_init *init,
-	bool is_shared,
-	dma_addr_t base_addr,
-	struct ipahal_imm_cmd_ip_v4_nat_init *cmd)
-{
-	IPADBG("\n");
-
-	ipa3_nat_ipv6ct_create_init_cmd(
-		&cmd->table_init,
-		is_shared,
-		base_addr,
-		init->tbl_index,
-		init->ipv4_rules_offset,
-		init->expn_rules_offset,
-		init->table_entries,
-		init->expn_table_entries,
-		ipa3_ctx->nat_mem.dev.name);
-
-	cmd->index_table_addr_shared = is_shared;
-	cmd->index_table_expansion_addr_shared = is_shared;
-
-	cmd->index_table_addr =
-		base_addr + init->index_offset;
-	IPADBG("index_offset:0x%x\n", init->index_offset);
-
-	cmd->index_table_expansion_addr =
-		base_addr + init->index_expn_offset;
-	IPADBG("index_expn_offset:0x%x\n", init->index_expn_offset);
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0) {
-		/*
-		 * starting IPAv4.0 public ip field changed to store the
-		 * PDN config table offset in SMEM
-		 */
-		cmd->public_addr_info = IPA_MEM_PART(pdn_config_ofst);
-		IPADBG("pdn config base:0x%x\n", cmd->public_addr_info);
-	} else {
-		cmd->public_addr_info = init->ip_addr;
-		IPADBG("Public IP address:%pI4h\n", &cmd->public_addr_info);
-	}
-
-	IPADBG("return\n");
-}
-
-static void ipa3_nat_create_modify_pdn_cmd(
-	struct ipahal_imm_cmd_dma_shared_mem *mem_cmd, bool zero_mem)
-{
-	size_t pdn_entry_size, mem_size;
-
-	IPADBG("\n");
-
-	ipahal_nat_entry_size(IPAHAL_NAT_IPV4_PDN, &pdn_entry_size);
-	mem_size = pdn_entry_size * IPA_MAX_PDN_NUM;
-
-	if (zero_mem)
-		memset(ipa3_ctx->nat_mem.pdn_mem.base, 0, mem_size);
-
-	/* Copy the PDN config table to SRAM */
-	mem_cmd->is_read = false;
-	mem_cmd->skip_pipeline_clear = false;
-	mem_cmd->pipeline_clear_options = IPAHAL_HPS_CLEAR;
-	mem_cmd->size = mem_size;
-	mem_cmd->system_addr = ipa3_ctx->nat_mem.pdn_mem.phys_base;
-	mem_cmd->local_addr = ipa3_ctx->smem_restricted_bytes +
-		IPA_MEM_PART(pdn_config_ofst);
-
-	IPADBG("return\n");
-}
-
-static int ipa3_nat_send_init_cmd(struct ipahal_imm_cmd_ip_v4_nat_init *cmd,
-	bool zero_pdn_table)
-{
-	struct ipa3_desc desc[IPA_NAT_MAX_NUM_OF_INIT_CMD_DESC];
-	struct ipahal_imm_cmd_pyld *cmd_pyld[IPA_NAT_MAX_NUM_OF_INIT_CMD_DESC];
-	int i, num_cmd = 0, result;
-
-	IPADBG("\n");
-
-	/* NO-OP IC for ensuring that IPA pipeline is empty */
-	cmd_pyld[num_cmd] =
-		ipahal_construct_nop_imm_cmd(false, IPAHAL_HPS_CLEAR, false);
-	if (!cmd_pyld[num_cmd]) {
-		IPAERR("failed to construct NOP imm cmd\n");
-		return -ENOMEM;
-	}
-
-	ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]);
-	++num_cmd;
-
-	cmd_pyld[num_cmd] = ipahal_construct_imm_cmd(
-		IPA_IMM_CMD_IP_V4_NAT_INIT, cmd, false);
-	if (!cmd_pyld[num_cmd]) {
-		IPAERR_RL("fail to construct NAT init imm cmd\n");
-		result = -EPERM;
-		goto destroy_imm_cmd;
-	}
-
-	ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]);
-	++num_cmd;
-
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0) {
-		struct ipahal_imm_cmd_dma_shared_mem mem_cmd = { 0 };
-
-		if (num_cmd >= IPA_NAT_MAX_NUM_OF_INIT_CMD_DESC) {
-			IPAERR("number of commands is out of range\n");
-			result = -ENOBUFS;
-			goto destroy_imm_cmd;
-		}
-
-		/* Copy the PDN config table to SRAM */
-		ipa3_nat_create_modify_pdn_cmd(&mem_cmd, zero_pdn_table);
-		cmd_pyld[num_cmd] = ipahal_construct_imm_cmd(
-			IPA_IMM_CMD_DMA_SHARED_MEM, &mem_cmd, false);
-		if (!cmd_pyld[num_cmd]) {
-			IPAERR(
-				"fail construct dma_shared_mem cmd: for pdn table");
-			result = -ENOMEM;
-			goto destroy_imm_cmd;
-		}
-		ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]);
-		++num_cmd;
-		IPADBG("added PDN table copy cmd\n");
-	}
-
-	result = ipa3_send_cmd(num_cmd, desc);
-	if (result) {
-		IPAERR("fail to send NAT init immediate command\n");
-		goto destroy_imm_cmd;
-	}
-
-	IPADBG("return\n");
-
-destroy_imm_cmd:
-	for (i = 0; i < num_cmd; ++i)
-		ipahal_destroy_imm_cmd(cmd_pyld[i]);
-
-	return result;
-}
-
-static int ipa3_ipv6ct_send_init_cmd(struct ipahal_imm_cmd_ip_v6_ct_init *cmd)
-{
-	struct ipa3_desc desc[IPA_IPV6CT_MAX_NUM_OF_INIT_CMD_DESC];
-	struct ipahal_imm_cmd_pyld
-		*cmd_pyld[IPA_IPV6CT_MAX_NUM_OF_INIT_CMD_DESC];
-	int i, num_cmd = 0, result;
-
-	IPADBG("\n");
-
-	/* NO-OP IC for ensuring that IPA pipeline is empty */
-	cmd_pyld[num_cmd] =
-		ipahal_construct_nop_imm_cmd(false, IPAHAL_HPS_CLEAR, false);
-	if (!cmd_pyld[num_cmd]) {
-		IPAERR("failed to construct NOP imm cmd\n");
-		return -ENOMEM;
-	}
-
-	ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]);
-	++num_cmd;
-
-	if (num_cmd >= IPA_IPV6CT_MAX_NUM_OF_INIT_CMD_DESC) {
-		IPAERR("number of commands is out of range\n");
-		result = -ENOBUFS;
-		goto destroy_imm_cmd;
-	}
-
-	cmd_pyld[num_cmd] = ipahal_construct_imm_cmd(
-		IPA_IMM_CMD_IP_V6_CT_INIT, cmd, false);
-	if (!cmd_pyld[num_cmd]) {
-		IPAERR_RL("fail to construct IPv6CT init imm cmd\n");
-		result = -EPERM;
-		goto destroy_imm_cmd;
-	}
-
-	ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]);
-	++num_cmd;
-
-	result = ipa3_send_cmd(num_cmd, desc);
-	if (result) {
-		IPAERR("Fail to send IPv6CT init immediate command\n");
-		goto destroy_imm_cmd;
-	}
-
-	IPADBG("return\n");
-
-destroy_imm_cmd:
-	for (i = 0; i < num_cmd; ++i)
-		ipahal_destroy_imm_cmd(cmd_pyld[i]);
-
-	return result;
-}
-
-/* IOCTL function handlers */
-/**
- * ipa3_nat_init_cmd() - Post IP_V4_NAT_INIT command to IPA HW
- * @init:	[in] initialization command attributes
- *
- * Called by NAT client driver to post IP_V4_NAT_INIT command to IPA HW
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa3_nat_init_cmd(struct ipa_ioc_v4_nat_init *init)
-{
-	struct ipahal_imm_cmd_ip_v4_nat_init cmd;
-	int result;
-
-	IPADBG("\n");
-
-	if (!ipa3_ctx->nat_mem.dev.is_mapped) {
-		IPAERR_RL("attempt to init %s before mmap\n",
-			ipa3_ctx->nat_mem.dev.name);
-		return -EPERM;
-	}
-
-	if (init->tbl_index >= 1) {
-		IPAERR_RL("Unsupported table index %d\n", init->tbl_index);
-		return -EPERM;
-	}
-
-	if (init->table_entries == 0) {
-		IPAERR_RL("Table entries is zero\n");
-		return -EPERM;
-	}
-
-	result = ipa3_nat_ipv6ct_check_table_params(
-		&ipa3_ctx->nat_mem.dev,
-		init->ipv4_rules_offset,
-		init->table_entries + 1,
-		IPAHAL_NAT_IPV4);
-	if (result) {
-		IPAERR_RL("Bad params for NAT base table\n");
-		return result;
-	}
-
-	result = ipa3_nat_ipv6ct_check_table_params(
-		&ipa3_ctx->nat_mem.dev,
-		init->expn_rules_offset,
-		init->expn_table_entries,
-		IPAHAL_NAT_IPV4);
-	if (result) {
-		IPAERR_RL("Bad params for NAT expansion table\n");
-		return result;
-	}
-
-	result = ipa3_nat_ipv6ct_check_table_params(
-		&ipa3_ctx->nat_mem.dev,
-		init->index_offset,
-		init->table_entries + 1,
-		IPAHAL_NAT_IPV4_INDEX);
-	if (result) {
-		IPAERR_RL("Bad params for index table\n");
-		return result;
-	}
-
-	result = ipa3_nat_ipv6ct_check_table_params(
-		&ipa3_ctx->nat_mem.dev,
-		init->index_expn_offset,
-		init->expn_table_entries,
-		IPAHAL_NAT_IPV4_INDEX);
-	if (result) {
-		IPAERR_RL("Bad params for index expansion table\n");
-		return result;
-	}
-
-	if (ipa3_ctx->nat_mem.dev.is_sys_mem) {
-		IPADBG("using system memory for nat table\n");
-		/*
-		 * Safe to process, since integer overflow was
-		 * checked in ipa3_nat_ipv6ct_check_table_params
-		 */
-		ipa3_nat_create_init_cmd(init, false,
-			ipa3_ctx->nat_mem.dev.dma_handle, &cmd);
-	} else {
-		IPADBG("using shared(local) memory for nat table\n");
-		ipa3_nat_create_init_cmd(init, true, IPA_RAM_NAT_OFST, &cmd);
-	}
-
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0) {
-		struct ipahal_nat_pdn_entry pdn_entry;
-
-		/* store ip in pdn entry cache array */
-		pdn_entry.public_ip = init->ip_addr;
-		pdn_entry.src_metadata = 0;
-		pdn_entry.dst_metadata = 0;
-
-		result = ipahal_nat_construct_entry(
-			IPAHAL_NAT_IPV4_PDN,
-			&pdn_entry,
-			ipa3_ctx->nat_mem.pdn_mem.base);
-		if (result) {
-			IPAERR("Fail to construct NAT pdn entry\n");
-			return result;
-		}
-
-		IPADBG("Public ip address:0x%x\n", init->ip_addr);
-	}
-
-	IPADBG("posting NAT init command\n");
-	result = ipa3_nat_send_init_cmd(&cmd, false);
-	if (result) {
-		IPAERR("Fail to send NAT init immediate command\n");
-		return result;
-	}
-
-	ipa3_nat_ipv6ct_init_device_structure(
-		&ipa3_ctx->nat_mem.dev,
-		init->ipv4_rules_offset,
-		init->expn_rules_offset,
-		init->table_entries,
-		init->expn_table_entries);
-
-	ipa3_ctx->nat_mem.public_ip_addr = init->ip_addr;
-	IPADBG("Public IP address:%pI4h\n", &ipa3_ctx->nat_mem.public_ip_addr);
-
-	ipa3_ctx->nat_mem.index_table_addr =
-		 (char *)ipa3_ctx->nat_mem.dev.base_address +
-		 init->index_offset;
-	IPADBG("index_table_addr: 0x%pK\n",
-				 ipa3_ctx->nat_mem.index_table_addr);
-
-	ipa3_ctx->nat_mem.index_table_expansion_addr =
-	 (char *)ipa3_ctx->nat_mem.dev.base_address + init->index_expn_offset;
-	IPADBG("index_table_expansion_addr: 0x%pK\n",
-				 ipa3_ctx->nat_mem.index_table_expansion_addr);
-
-	ipa3_ctx->nat_mem.dev.is_hw_init = true;
-	IPADBG("return\n");
-	return 0;
-}
-
-/**
- * ipa3_ipv6ct_init_cmd() - Post IP_V6_CONN_TRACK_INIT command to IPA HW
- * @init:	[in] initialization command attributes
- *
- * Called by NAT client driver to post IP_V6_CONN_TRACK_INIT command to IPA HW
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa3_ipv6ct_init_cmd(struct ipa_ioc_ipv6ct_init *init)
-{
-	struct ipahal_imm_cmd_ip_v6_ct_init cmd;
-	int result;
-
-	IPADBG("\n");
-
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_0) {
-		IPAERR_RL("IPv6 connection tracking isn't supported\n");
-		return -EPERM;
-	}
-
-	if (!ipa3_ctx->ipv6ct_mem.dev.is_mapped) {
-		IPAERR_RL("attempt to init %s before mmap\n",
-			ipa3_ctx->ipv6ct_mem.dev.name);
-		return -EPERM;
-	}
-
-	if (init->tbl_index >= 1) {
-		IPAERR_RL("Unsupported table index %d\n", init->tbl_index);
-		return -EPERM;
-	}
-
-	if (init->table_entries == 0) {
-		IPAERR_RL("Table entries is zero\n");
-		return -EPERM;
-	}
-
-	result = ipa3_nat_ipv6ct_check_table_params(
-		&ipa3_ctx->ipv6ct_mem.dev,
-		init->base_table_offset,
-		init->table_entries + 1,
-		IPAHAL_NAT_IPV6CT);
-	if (result) {
-		IPAERR_RL("Bad params for IPv6CT base table\n");
-		return result;
-	}
-
-	result = ipa3_nat_ipv6ct_check_table_params(
-		&ipa3_ctx->ipv6ct_mem.dev,
-		init->expn_table_offset,
-		init->expn_table_entries,
-		IPAHAL_NAT_IPV6CT);
-	if (result) {
-		IPAERR_RL("Bad params for IPv6CT expansion table\n");
-		return result;
-	}
-
-	if (ipa3_ctx->ipv6ct_mem.dev.is_sys_mem) {
-		IPADBG("using system memory for nat table\n");
-		/*
-		 * Safe to process, since integer overflow was
-		 * checked in ipa3_nat_ipv6ct_check_table_params
-		 */
-		ipa3_nat_ipv6ct_create_init_cmd(
-			&cmd.table_init,
-			false,
-			ipa3_ctx->ipv6ct_mem.dev.dma_handle,
-			init->tbl_index,
-			init->base_table_offset,
-			init->expn_table_offset,
-			init->table_entries,
-			init->expn_table_entries,
-			ipa3_ctx->ipv6ct_mem.dev.name);
-	} else {
-		IPADBG("using shared(local) memory for nat table\n");
-		ipa3_nat_ipv6ct_create_init_cmd(
-			&cmd.table_init,
-			true,
-			IPA_RAM_IPV6CT_OFST,
-			init->tbl_index,
-			init->base_table_offset,
-			init->expn_table_offset,
-			init->table_entries,
-			init->expn_table_entries,
-			ipa3_ctx->ipv6ct_mem.dev.name);
-	}
-
-	IPADBG("posting ip_v6_ct_init imm command\n");
-	result = ipa3_ipv6ct_send_init_cmd(&cmd);
-	if (result) {
-		IPAERR("fail to send IPv6CT init immediate command\n");
-		return result;
-	}
-
-	ipa3_nat_ipv6ct_init_device_structure(
-		&ipa3_ctx->ipv6ct_mem.dev,
-		init->base_table_offset,
-		init->expn_table_offset,
-		init->table_entries,
-		init->expn_table_entries);
-
-	ipa3_ctx->ipv6ct_mem.dev.is_hw_init = true;
-	IPADBG("return\n");
-	return 0;
-}
-
-/**
- * ipa3_nat_mdfy_pdn() - Modify a PDN entry in PDN config table in IPA SRAM
- * @mdfy_pdn:	[in] PDN info to be written to SRAM
- *
- * Called by NAT client driver to modify an entry in the PDN config table
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa3_nat_mdfy_pdn(struct ipa_ioc_nat_pdn_entry *mdfy_pdn)
-{
-	struct ipahal_imm_cmd_dma_shared_mem mem_cmd = { 0 };
-	struct ipa3_desc desc;
-	struct ipahal_imm_cmd_pyld *cmd_pyld;
-	int result = 0;
-	struct ipa3_nat_mem *nat_ctx = &(ipa3_ctx->nat_mem);
-	struct ipahal_nat_pdn_entry pdn_fields;
-	size_t entry_size;
-
-	IPADBG("\n");
-
-	mutex_lock(&nat_ctx->dev.lock);
-
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_0) {
-		IPAERR_RL("IPA HW does not support multi PDN\n");
-		result = -EPERM;
-		goto bail;
-	}
-
-	if (!nat_ctx->dev.is_mem_allocated) {
-		IPAERR_RL(
-			"attempt to modify a PDN entry before the PDN table memory allocation\n");
-		result = -EPERM;
-		goto bail;
-	}
-
-	if (mdfy_pdn->pdn_index > (IPA_MAX_PDN_NUM - 1)) {
-		IPAERR_RL("pdn index out of range %d\n", mdfy_pdn->pdn_index);
-		result = -EPERM;
-		goto bail;
-	}
-
-	/* store ip in pdn entry cache array */
-	pdn_fields.public_ip = mdfy_pdn->public_ip;
-	pdn_fields.dst_metadata = mdfy_pdn->dst_metadata;
-	pdn_fields.src_metadata = mdfy_pdn->src_metadata;
-
-	/* mark tethering bit for remote modem */
-	if (ipa3_ctx->ipa_hw_type == IPA_HW_v4_1) {
-		pdn_fields.src_metadata |=
-			IPA_QMAP_TETH_BIT;
-	}
-
-	/* get size of the entry */
-	result = ipahal_nat_entry_size(
-		IPAHAL_NAT_IPV4_PDN,
-		&entry_size);
-	if (result) {
-		IPAERR("Failed to retrieve pdn entry size\n");
-		goto bail;
-	}
-
-	result = ipahal_nat_construct_entry(
-		IPAHAL_NAT_IPV4_PDN,
-		&pdn_fields,
-		(nat_ctx->pdn_mem.base +
-		(mdfy_pdn->pdn_index)*(entry_size)));
-	if (result) {
-		IPAERR("Fail to construct NAT pdn entry\n");
-		goto bail;
-	}
-
-	IPADBG("Modify PDN in index: %d Public ip address:%pI4h\n",
-		mdfy_pdn->pdn_index,
-		&pdn_fields.public_ip);
-	IPADBG("Modify PDN dst metadata: 0x%x src metadata: 0x%x\n",
-		pdn_fields.dst_metadata,
-		pdn_fields.src_metadata);
-
-	/* Copy the PDN config table to SRAM */
-	ipa3_nat_create_modify_pdn_cmd(&mem_cmd, false);
-	cmd_pyld = ipahal_construct_imm_cmd(
-		IPA_IMM_CMD_DMA_SHARED_MEM, &mem_cmd, false);
-	if (!cmd_pyld) {
-		IPAERR(
-			"fail construct dma_shared_mem cmd: for pdn table\n");
-		result = -ENOMEM;
-		goto bail;
-	}
-	ipa3_init_imm_cmd_desc(&desc, cmd_pyld);
-
-	IPADBG("sending PDN table copy cmd\n");
-	result = ipa3_send_cmd(1, &desc);
-	if (result)
-		IPAERR("Fail to send PDN table copy immediate command\n");
-
-	ipahal_destroy_imm_cmd(cmd_pyld);
-
-	IPADBG("return\n");
-
-bail:
-	mutex_unlock(&nat_ctx->dev.lock);
-	return result;
-}
-
-static uint32_t ipa3_nat_ipv6ct_calculate_table_size(uint8_t base_addr)
-{
-	size_t entry_size;
-	u32 entries_num;
-	enum ipahal_nat_type nat_type;
-
-	switch (base_addr) {
-	case IPA_NAT_BASE_TBL:
-		entries_num = ipa3_ctx->nat_mem.dev.table_entries + 1;
-		nat_type = IPAHAL_NAT_IPV4;
-		break;
-	case IPA_NAT_EXPN_TBL:
-		entries_num = ipa3_ctx->nat_mem.dev.expn_table_entries;
-		nat_type = IPAHAL_NAT_IPV4;
-		break;
-	case IPA_NAT_INDX_TBL:
-		entries_num = ipa3_ctx->nat_mem.dev.table_entries + 1;
-		nat_type = IPAHAL_NAT_IPV4_INDEX;
-		break;
-	case IPA_NAT_INDEX_EXPN_TBL:
-		entries_num = ipa3_ctx->nat_mem.dev.expn_table_entries;
-		nat_type = IPAHAL_NAT_IPV4_INDEX;
-		break;
-	case IPA_IPV6CT_BASE_TBL:
-		entries_num = ipa3_ctx->ipv6ct_mem.dev.table_entries + 1;
-		nat_type = IPAHAL_NAT_IPV6CT;
-		break;
-	case IPA_IPV6CT_EXPN_TBL:
-		entries_num = ipa3_ctx->ipv6ct_mem.dev.expn_table_entries;
-		nat_type = IPAHAL_NAT_IPV6CT;
-		break;
-	default:
-		IPAERR_RL("Invalid base_addr %d for table DMA command\n",
-			base_addr);
-		return 0;
-	}
-
-	ipahal_nat_entry_size(nat_type, &entry_size);
-	return entry_size * entries_num;
-}
-
-static int ipa3_table_validate_table_dma_one(struct ipa_ioc_nat_dma_one *param)
-{
-	uint32_t table_size;
-
-	if (param->table_index >= 1) {
-		IPAERR_RL("Unsupported table index %d\n", param->table_index);
-		return -EPERM;
-	}
-
-	switch (param->base_addr) {
-	case IPA_NAT_BASE_TBL:
-	case IPA_NAT_EXPN_TBL:
-	case IPA_NAT_INDX_TBL:
-	case IPA_NAT_INDEX_EXPN_TBL:
-		if (!ipa3_ctx->nat_mem.dev.is_hw_init) {
-			IPAERR_RL("attempt to write to %s before HW int\n",
-				ipa3_ctx->nat_mem.dev.name);
-			return -EPERM;
-		}
-		break;
-	case IPA_IPV6CT_BASE_TBL:
-	case IPA_IPV6CT_EXPN_TBL:
-		if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_0) {
-			IPAERR_RL("IPv6 connection tracking isn't supported\n");
-			return -EPERM;
-		}
-
-		if (!ipa3_ctx->ipv6ct_mem.dev.is_hw_init) {
-			IPAERR_RL("attempt to write to %s before HW int\n",
-				ipa3_ctx->ipv6ct_mem.dev.name);
-			return -EPERM;
-		}
-		break;
-	default:
-		IPAERR_RL("Invalid base_addr %d for table DMA command\n",
-			param->base_addr);
-		return -EPERM;
-	}
-
-	table_size = ipa3_nat_ipv6ct_calculate_table_size(param->base_addr);
-	if (!table_size) {
-		IPAERR_RL("Failed to calculate table size for base_addr %d\n",
-			param->base_addr);
-		return -EPERM;
-	}
-
-	if (param->offset >= table_size) {
-		IPAERR_RL("Invalid offset %d for table DMA command\n",
-			param->offset);
-		IPAERR_RL("table_index %d base addr %d size %d\n",
-			param->table_index, param->base_addr, table_size);
-		return -EPERM;
-	}
-
-	return 0;
-}
-
-
-/**
- * ipa3_table_dma_cmd() - Post TABLE_DMA command to IPA HW
- * @dma:	[in] initialization command attributes
- *
- * Called by NAT/IPv6CT clients to post TABLE_DMA command to IPA HW
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa3_table_dma_cmd(struct ipa_ioc_nat_dma_cmd *dma)
-{
-	struct ipahal_imm_cmd_table_dma cmd;
-	enum ipahal_imm_cmd_name cmd_name = IPA_IMM_CMD_NAT_DMA;
-	struct ipahal_imm_cmd_pyld *cmd_pyld[IPA_MAX_NUM_OF_TABLE_DMA_CMD_DESC];
-	struct ipa3_desc desc[IPA_MAX_NUM_OF_TABLE_DMA_CMD_DESC];
-	uint8_t cnt, num_cmd = 0;
-	int result = 0;
-
-	IPADBG("\n");
-	if (!dma->entries ||
-		dma->entries >= IPA_MAX_NUM_OF_TABLE_DMA_CMD_DESC) {
-		IPAERR_RL("Invalid number of entries %d\n",
-			dma->entries);
-		result = -EPERM;
-		goto bail;
-	}
-
-	if (!ipa3_ctx->nat_mem.dev.is_dev_init) {
-		IPAERR_RL("NAT hasn't been initialized\n");
-		return -EPERM;
-	}
-
-	for (cnt = 0; cnt < dma->entries; ++cnt) {
-		result = ipa3_table_validate_table_dma_one(&dma->dma[cnt]);
-		if (result) {
-			IPAERR_RL("Table DMA command parameter %d is invalid\n",
-				cnt);
-			goto bail;
-		}
-	}
-
-	/* NO-OP IC for ensuring that IPA pipeline is empty */
-	cmd_pyld[num_cmd] =
-		ipahal_construct_nop_imm_cmd(false, IPAHAL_HPS_CLEAR, false);
-	if (!cmd_pyld[num_cmd]) {
-		IPAERR("Failed to construct NOP imm cmd\n");
-		result = -ENOMEM;
-		goto destroy_imm_cmd;
-	}
-	ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]);
-	++num_cmd;
-
-	/* NAT_DMA was renamed to TABLE_DMA starting from IPAv4 */
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0)
-		cmd_name = IPA_IMM_CMD_TABLE_DMA;
-
-	for (cnt = 0; cnt < dma->entries; ++cnt) {
-		cmd.table_index = dma->dma[cnt].table_index;
-		cmd.base_addr = dma->dma[cnt].base_addr;
-		cmd.offset = dma->dma[cnt].offset;
-		cmd.data = dma->dma[cnt].data;
-		cmd_pyld[num_cmd] =
-			ipahal_construct_imm_cmd(cmd_name, &cmd, false);
-		if (!cmd_pyld[num_cmd]) {
-			IPAERR_RL("Fail to construct table_dma imm cmd\n");
-			result = -ENOMEM;
-			goto destroy_imm_cmd;
-		}
-		ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]);
-		++num_cmd;
-	}
-	result = ipa3_send_cmd(num_cmd, desc);
-	if (result)
-		IPAERR("Fail to send table_dma immediate command\n");
-
-	IPADBG("return\n");
-
-destroy_imm_cmd:
-	for (cnt = 0; cnt < num_cmd; ++cnt)
-		ipahal_destroy_imm_cmd(cmd_pyld[cnt]);
-bail:
-	return result;
-}
-
-/**
- * ipa3_nat_dma_cmd() - Post NAT_DMA command to IPA HW
- * @dma:	[in] initialization command attributes
- *
- * Called by NAT client driver to post NAT_DMA command to IPA HW
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa3_nat_dma_cmd(struct ipa_ioc_nat_dma_cmd *dma)
-{
-	return ipa3_table_dma_cmd(dma);
-}
-
-static void ipa3_nat_ipv6ct_free_mem(struct ipa3_nat_ipv6ct_common_mem *dev)
-{
-	IPADBG("\n");
-	if (!dev->is_mem_allocated) {
-		IPADBG("attempt to delete %s before memory allocation\n",
-			dev->name);
-		/* Deletion of partly initialized table is not an error */
-		goto clear;
-	}
-
-	if (dev->is_sys_mem) {
-		IPADBG("freeing the dma memory for %s\n", dev->name);
-		dma_free_coherent(
-			ipa3_ctx->pdev, dev->size,
-			dev->vaddr, dev->dma_handle);
-		dev->size = 0;
-		dev->vaddr = NULL;
-	}
-
-	dev->is_mem_allocated = false;
-
-clear:
-	dev->table_entries = 0;
-	dev->expn_table_entries = 0;
-	dev->base_table_addr = NULL;
-	dev->expansion_table_addr = NULL;
-
-	dev->is_hw_init = false;
-	dev->is_mapped = false;
-	dev->is_sys_mem = false;
-
-	IPADBG("return\n");
-}
-
-static int ipa3_nat_ipv6ct_create_del_table_cmd(
-	uint8_t tbl_index,
-	u32 base_addr,
-	struct ipa3_nat_ipv6ct_common_mem *dev,
-	struct ipahal_imm_cmd_nat_ipv6ct_init_common *table_init_cmd)
-{
-	bool mem_type_shared = true;
-
-	IPADBG("\n");
-
-	if (tbl_index >= 1) {
-		IPAERR_RL("Unsupported table index %d\n", tbl_index);
-		return -EPERM;
-	}
-
-	if (dev->tmp_mem != NULL) {
-		IPADBG("using temp memory during %s del\n", dev->name);
-		mem_type_shared = false;
-		base_addr = dev->tmp_mem->dma_handle;
-	}
-
-	table_init_cmd->table_index = tbl_index;
-	table_init_cmd->base_table_addr = base_addr;
-	table_init_cmd->base_table_addr_shared = mem_type_shared;
-	table_init_cmd->expansion_table_addr = base_addr;
-	table_init_cmd->expansion_table_addr_shared = mem_type_shared;
-	table_init_cmd->size_base_table = 0;
-	table_init_cmd->size_expansion_table = 0;
-	IPADBG("return\n");
-
-	return 0;
-}
-
-static int ipa3_nat_send_del_table_cmd(uint8_t tbl_index)
-{
-	struct ipahal_imm_cmd_ip_v4_nat_init cmd;
-	int result;
-
-	IPADBG("\n");
-
-	result = ipa3_nat_ipv6ct_create_del_table_cmd(
-		tbl_index,
-		IPA_NAT_PHYS_MEM_OFFSET,
-		&ipa3_ctx->nat_mem.dev,
-		&cmd.table_init);
-	if (result) {
-		IPAERR(
-			"Fail to create immediate command to delete NAT table\n");
-		return result;
-	}
-
-	cmd.index_table_addr = cmd.table_init.base_table_addr;
-	cmd.index_table_addr_shared = cmd.table_init.base_table_addr_shared;
-	cmd.index_table_expansion_addr = cmd.index_table_addr;
-	cmd.index_table_expansion_addr_shared = cmd.index_table_addr_shared;
-	cmd.public_addr_info = 0;
-
-	IPADBG("posting NAT delete command\n");
-	result = ipa3_nat_send_init_cmd(&cmd, true);
-	if (result) {
-		IPAERR("Fail to send NAT delete immediate command\n");
-		return result;
-	}
-
-	IPADBG("return\n");
-	return 0;
-}
-
-static int ipa3_ipv6ct_send_del_table_cmd(uint8_t tbl_index)
-{
-	struct ipahal_imm_cmd_ip_v6_ct_init cmd;
-	int result;
-
-	IPADBG("\n");
-
-	result = ipa3_nat_ipv6ct_create_del_table_cmd(
-		tbl_index,
-		IPA_IPV6CT_PHYS_MEM_OFFSET,
-		&ipa3_ctx->ipv6ct_mem.dev,
-		&cmd.table_init);
-	if (result) {
-		IPAERR(
-			"Fail to create immediate command to delete IPv6CT table\n");
-		return result;
-	}
-
-	IPADBG("posting IPv6CT delete command\n");
-	result = ipa3_ipv6ct_send_init_cmd(&cmd);
-	if (result) {
-		IPAERR("Fail to send IPv6CT delete immediate command\n");
-		return result;
-	}
-
-	IPADBG("return\n");
-	return 0;
-}
-
-/**
- * ipa3_nat_del_cmd() - Delete a NAT table
- * @del:	[in] delete table table table parameters
- *
- * Called by NAT client driver to delete the nat table
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa3_nat_del_cmd(struct ipa_ioc_v4_nat_del *del)
-{
-	struct ipa_ioc_nat_ipv6ct_table_del tmp;
-
-	tmp.table_index = del->table_index;
-
-	return ipa3_del_nat_table(&tmp);
-}
-
-/**
- * ipa3_del_nat_table() - Delete the NAT table
- * @del:	[in] delete table parameters
- *
- * Called by NAT client to delete the table
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa3_del_nat_table(struct ipa_ioc_nat_ipv6ct_table_del *del)
-{
-	int result = 0;
-
-	IPADBG("\n");
-	if (!ipa3_ctx->nat_mem.dev.is_dev_init) {
-		IPAERR("NAT hasn't been initialized\n");
-		return -EPERM;
-	}
-
-	mutex_lock(&ipa3_ctx->nat_mem.dev.lock);
-
-	if (ipa3_ctx->nat_mem.dev.is_hw_init) {
-		result = ipa3_nat_send_del_table_cmd(del->table_index);
-		if (result) {
-			IPAERR(
-				"Fail to send immediate command to delete NAT table\n");
-			goto bail;
-		}
-	}
-
-	ipa3_ctx->nat_mem.public_ip_addr = 0;
-	ipa3_ctx->nat_mem.index_table_addr = 0;
-	ipa3_ctx->nat_mem.index_table_expansion_addr = 0;
-
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0 &&
-		ipa3_ctx->nat_mem.dev.is_mem_allocated) {
-		IPADBG("freeing the PDN memory\n");
-		dma_free_coherent(ipa3_ctx->pdev,
-			ipa3_ctx->nat_mem.pdn_mem.size,
-			ipa3_ctx->nat_mem.pdn_mem.base,
-			ipa3_ctx->nat_mem.pdn_mem.phys_base);
-		ipa3_ctx->nat_mem.pdn_mem.base = NULL;
-	}
-
-	ipa3_nat_ipv6ct_free_mem(&ipa3_ctx->nat_mem.dev);
-	IPADBG("return\n");
-
-bail:
-	mutex_unlock(&ipa3_ctx->nat_mem.dev.lock);
-	return result;
-}
-
-/**
- * ipa3_del_ipv6ct_table() - Delete the IPv6CT table
- * @del:	[in] delete table parameters
- *
- * Called by IPv6CT client to delete the table
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa3_del_ipv6ct_table(struct ipa_ioc_nat_ipv6ct_table_del *del)
-{
-	int result = 0;
-
-	IPADBG("\n");
-
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_0) {
-		IPAERR_RL("IPv6 connection tracking isn't supported\n");
-		return -EPERM;
-	}
-
-	if (!ipa3_ctx->ipv6ct_mem.dev.is_dev_init) {
-		IPAERR("IPv6 connection tracking hasn't been initialized\n");
-		return -EPERM;
-	}
-
-	mutex_lock(&ipa3_ctx->ipv6ct_mem.dev.lock);
-
-	if (ipa3_ctx->ipv6ct_mem.dev.is_hw_init) {
-		result = ipa3_ipv6ct_send_del_table_cmd(del->table_index);
-		if (result) {
-			IPAERR(
-				"Fail to send immediate command to delete IPv6CT table\n");
-			goto bail;
-		}
-	}
-
-	ipa3_nat_ipv6ct_free_mem(&ipa3_ctx->ipv6ct_mem.dev);
-	IPADBG("return\n");
-
-bail:
-	mutex_unlock(&ipa3_ctx->ipv6ct_mem.dev.lock);
-	return result;
-}
-

ipa/ipa_v3/ipa_odl.c

@@ -1,729 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
- */
-
-#include "ipa_i.h"
-#include "ipa_odl.h"
-#include <linux/msm_ipa.h>
-#include <linux/sched/signal.h>
-#include <linux/poll.h>
-
-struct ipa_odl_context *ipa3_odl_ctx;
-
-static DECLARE_WAIT_QUEUE_HEAD(odl_ctl_msg_wq);
-
-static void print_ipa_odl_state_bit_mask(void)
-{
-	IPADBG("ipa3_odl_ctx->odl_state.odl_init --> %d\n",
-		ipa3_odl_ctx->odl_state.odl_init);
-	IPADBG("ipa3_odl_ctx->odl_state.odl_open --> %d\n",
-		ipa3_odl_ctx->odl_state.odl_open);
-	IPADBG("ipa3_odl_ctx->odl_state.adpl_open --> %d\n",
-		ipa3_odl_ctx->odl_state.adpl_open);
-	IPADBG("ipa3_odl_ctx->odl_state.aggr_byte_limit_sent --> %d\n",
-		ipa3_odl_ctx->odl_state.aggr_byte_limit_sent);
-	IPADBG("ipa3_odl_ctx->odl_state.odl_ep_setup --> %d\n",
-		ipa3_odl_ctx->odl_state.odl_ep_setup);
-	IPADBG("ipa3_odl_ctx->odl_state.odl_setup_done_sent --> %d\n",
-		ipa3_odl_ctx->odl_state.odl_setup_done_sent);
-	IPADBG("ipa3_odl_ctx->odl_state.odl_ep_info_sent --> %d\n",
-		ipa3_odl_ctx->odl_state.odl_ep_info_sent);
-	IPADBG("ipa3_odl_ctx->odl_state.odl_connected --> %d\n",
-		ipa3_odl_ctx->odl_state.odl_connected);
-	IPADBG("ipa3_odl_ctx->odl_state.odl_disconnected --> %d\n\n",
-		ipa3_odl_ctx->odl_state.odl_disconnected);
-}
-
-static int ipa_odl_ctl_fops_open(struct inode *inode, struct file *filp)
-{
-	int ret = 0;
-
-	if (ipa3_odl_ctx->odl_state.odl_init) {
-		ipa3_odl_ctx->odl_state.odl_open = true;
-	} else {
-		IPAERR("Before odl init trying to open odl ctl pipe\n");
-		print_ipa_odl_state_bit_mask();
-		ret = -ENODEV;
-	}
-
-	return ret;
-}
-
-static int ipa_odl_ctl_fops_release(struct inode *inode, struct file *filp)
-{
-	IPADBG("QTI closed ipa_odl_ctl node\n");
-	ipa3_odl_ctx->odl_state.odl_open = false;
-	return 0;
-}
-
-/**
- * ipa_odl_ctl_fops_read() - read message from IPA ODL device
- * @filp:	[in] file pointer
- * @buf:	[out] buffer to read into
- * @count:	[in] size of above buffer
- * @f_pos:	[inout] file position
- *
- * Uer-space should continuously read from /dev/ipa_odl_ctl,
- * read will block when there are no messages to read.
- * Upon return, user-space should read the u32 data from the
- * start of the buffer.
- *
- * 0 --> ODL disconnected.
- * 1 --> ODL connected.
- *
- * Buffer supplied must be big enough to
- * hold the message of size u32.
- *
- * Returns: how many bytes copied to buffer
- *
- * Note: Should not be called from atomic context
- */
-
-static ssize_t ipa_odl_ctl_fops_read(struct file *filp, char __user *buf,
-			size_t count, loff_t *f_pos)
-{
-	char __user *start;
-	u8 data;
-	int ret = 0;
-	static bool old_state;
-	bool new_state = false;
-
-	start = buf;
-	ipa3_odl_ctx->odl_ctl_msg_wq_flag = false;
-
-	if (!ipa3_odl_ctx->odl_state.adpl_open &&
-			!ipa3_odl_ctx->odl_state.odl_disconnected) {
-		IPADBG("Failed to send data odl pipe already disconnected\n");
-		ret = -EFAULT;
-		goto send_failed;
-	}
-
-	if (ipa3_odl_ctx->odl_state.odl_ep_setup)
-		new_state = true;
-	else if (ipa3_odl_ctx->odl_state.odl_disconnected)
-		new_state = false;
-	else {
-		IPADBG("Failed to send data odl already running\n");
-		ret = -EFAULT;
-		goto send_failed;
-	}
-
-	if (old_state != new_state) {
-		old_state = new_state;
-
-		if (new_state)
-			data = 1;
-		else if (!new_state)
-			data = 0;
-
-		if (copy_to_user(buf, &data,
-					sizeof(data))) {
-			IPADBG("Cpoying data to user failed\n");
-			ret = -EFAULT;
-			goto send_failed;
-		}
-
-		buf += sizeof(data);
-
-		if (data == 1)
-			ipa3_odl_ctx->odl_state.odl_setup_done_sent =
-				true;
-	}
-
-
-	if (start != buf && ret != -EFAULT)
-		ret = buf - start;
-send_failed:
-	return ret;
-}
-
-static unsigned int ipa_odl_ctl_fops_poll(struct file *file, poll_table *wait)
-{
-	unsigned int mask = 0;
-
-	poll_wait(file, &odl_ctl_msg_wq, wait);
-
-	if (ipa3_odl_ctx->odl_ctl_msg_wq_flag) {
-		IPADBG("Sending read mask to odl control pipe\n");
-		mask |= POLLIN | POLLRDNORM;
-	}
-	return mask;
-}
-
-static long ipa_odl_ctl_fops_ioctl(struct file *filp, unsigned int cmd,
-							unsigned long arg)
-{
-	struct ipa_odl_ep_info ep_info = {0};
-	struct ipa_odl_modem_config status;
-	int retval = 0;
-
-	IPADBG("Calling odl ioctl cmd = %d\n", cmd);
-	if (!ipa3_odl_ctx->odl_state.odl_setup_done_sent) {
-		IPAERR("Before complete the odl setup trying calling ioctl\n");
-		print_ipa_odl_state_bit_mask();
-		retval = -ENODEV;
-		goto fail;
-	}
-
-	switch (cmd) {
-	case IPA_IOC_ODL_QUERY_ADAPL_EP_INFO:
-		/* Send ep_info to user APP */
-		ep_info.ep_type = ODL_EP_TYPE_HSUSB;
-		ep_info.peripheral_iface_id = ODL_EP_PERIPHERAL_IFACE_ID;
-		ep_info.cons_pipe_num = -1;
-		ep_info.prod_pipe_num =
-			ipa3_odl_ctx->odl_client_hdl;
-		if (copy_to_user((void __user *)arg, &ep_info,
-					sizeof(ep_info))) {
-			retval = -EFAULT;
-			goto fail;
-		}
-		ipa3_odl_ctx->odl_state.odl_ep_info_sent = true;
-		break;
-	case IPA_IOC_ODL_QUERY_MODEM_CONFIG:
-		IPADBG("Received the IPA_IOC_ODL_QUERY_MODEM_CONFIG :\n");
-		if (copy_from_user(&status, (const void __user *)arg,
-			sizeof(status))) {
-			retval = -EFAULT;
-			break;
-		}
-		if (status.config_status == CONFIG_SUCCESS)
-			ipa3_odl_ctx->odl_state.odl_connected = true;
-		IPADBG("status.config_status = %d odl_connected = %d\n",
-		status.config_status, ipa3_odl_ctx->odl_state.odl_connected);
-		break;
-	default:
-		retval = -ENOIOCTLCMD;
-		break;
-	}
-
-fail:
-	return retval;
-}
-
-static void delete_first_node(void)
-{
-	struct ipa3_push_msg_odl *msg;
-
-	if (!list_empty(&ipa3_odl_ctx->adpl_msg_list)) {
-		msg = list_first_entry(&ipa3_odl_ctx->adpl_msg_list,
-				struct ipa3_push_msg_odl, link);
-		if (msg) {
-			list_del(&msg->link);
-			kfree(msg->buff);
-			kfree(msg);
-			ipa3_odl_ctx->stats.odl_drop_pkt++;
-			if (atomic_read(&ipa3_odl_ctx->stats.numer_in_queue))
-				atomic_dec(&ipa3_odl_ctx->stats.numer_in_queue);
-		}
-	} else {
-		IPADBG("List Empty\n");
-	}
-}
-
-int ipa3_send_adpl_msg(unsigned long skb_data)
-{
-	struct ipa3_push_msg_odl *msg;
-	struct sk_buff *skb = (struct sk_buff *)skb_data;
-	void *data;
-
-	IPADBG_LOW("Processing DPL data\n");
-	msg = kzalloc(sizeof(struct ipa3_push_msg_odl), GFP_KERNEL);
-	if (msg == NULL) {
-		IPADBG("Memory allocation failed\n");
-		return -ENOMEM;
-	}
-
-	data = kmemdup(skb->data, skb->len, GFP_KERNEL);
-	if (data == NULL) {
-		kfree(msg);
-		return -ENOMEM;
-	}
-	memcpy(data, skb->data, skb->len);
-	msg->buff = data;
-	msg->len = skb->len;
-	mutex_lock(&ipa3_odl_ctx->adpl_msg_lock);
-	if (atomic_read(&ipa3_odl_ctx->stats.numer_in_queue) >=
-						MAX_QUEUE_TO_ODL)
-		delete_first_node();
-	list_add_tail(&msg->link, &ipa3_odl_ctx->adpl_msg_list);
-	atomic_inc(&ipa3_odl_ctx->stats.numer_in_queue);
-	mutex_unlock(&ipa3_odl_ctx->adpl_msg_lock);
-	IPA_STATS_INC_CNT(ipa3_odl_ctx->stats.odl_rx_pkt);
-
-	return 0;
-}
-
-/**
- * odl_ipa_packet_receive_notify() - Rx notify
- *
- * @priv: driver context
- * @evt: event type
- * @data: data provided with event
- *
- * IPA will pass a packet to the Linux network stack with skb->data
- */
-static void odl_ipa_packet_receive_notify(void *priv,
-		enum ipa_dp_evt_type evt,
-		unsigned long data)
-{
-	IPADBG_LOW("Rx packet was received\n");
-	if (evt == IPA_RECEIVE)
-		ipa3_send_adpl_msg(data);
-	else
-		IPAERR("Invalid evt %d received in wan_ipa_receive\n", evt);
-}
-
-int ipa_setup_odl_pipe(void)
-{
-	struct ipa_sys_connect_params *ipa_odl_ep_cfg;
-	int ret;
-
-	ipa_odl_ep_cfg = &ipa3_odl_ctx->odl_sys_param;
-
-	IPADBG("Setting up the odl endpoint\n");
-	ipa_odl_ep_cfg->ipa_ep_cfg.cfg.cs_offload_en = IPA_ENABLE_CS_OFFLOAD_DL;
-
-	ipa_odl_ep_cfg->ipa_ep_cfg.aggr.aggr_en = IPA_ENABLE_AGGR;
-	ipa_odl_ep_cfg->ipa_ep_cfg.aggr.aggr_hard_byte_limit_en = 1;
-	ipa_odl_ep_cfg->ipa_ep_cfg.aggr.aggr = IPA_GENERIC;
-	ipa_odl_ep_cfg->ipa_ep_cfg.aggr.aggr_byte_limit =
-						IPA_ODL_AGGR_BYTE_LIMIT;
-	ipa_odl_ep_cfg->ipa_ep_cfg.aggr.aggr_pkt_limit = 0;
-
-	ipa_odl_ep_cfg->ipa_ep_cfg.hdr.hdr_len = 4;
-	ipa_odl_ep_cfg->ipa_ep_cfg.hdr.hdr_ofst_metadata_valid = 1;
-	ipa_odl_ep_cfg->ipa_ep_cfg.hdr.hdr_ofst_metadata = 1;
-	ipa_odl_ep_cfg->ipa_ep_cfg.hdr.hdr_ofst_pkt_size_valid = 1;
-	ipa_odl_ep_cfg->ipa_ep_cfg.hdr.hdr_ofst_pkt_size = 2;
-
-	ipa_odl_ep_cfg->ipa_ep_cfg.hdr_ext.hdr_total_len_or_pad_valid = true;
-	ipa_odl_ep_cfg->ipa_ep_cfg.hdr_ext.hdr_total_len_or_pad = 0;
-	ipa_odl_ep_cfg->ipa_ep_cfg.hdr_ext.hdr_payload_len_inc_padding = true;
-	ipa_odl_ep_cfg->ipa_ep_cfg.hdr_ext.hdr_total_len_or_pad_offset = 0;
-	ipa_odl_ep_cfg->ipa_ep_cfg.hdr_ext.hdr_little_endian = 0;
-	ipa_odl_ep_cfg->ipa_ep_cfg.metadata_mask.metadata_mask = 0xFF000000;
-
-	ipa_odl_ep_cfg->client = IPA_CLIENT_ODL_DPL_CONS;
-	ipa_odl_ep_cfg->notify = odl_ipa_packet_receive_notify;
-
-	ipa_odl_ep_cfg->napi_obj = NULL;
-	ipa_odl_ep_cfg->desc_fifo_sz = IPA_ODL_RX_RING_SIZE *
-						IPA_FIFO_ELEMENT_SIZE;
-	ipa3_odl_ctx->odl_client_hdl = -1;
-	ret = ipa3_setup_sys_pipe(ipa_odl_ep_cfg,
-			&ipa3_odl_ctx->odl_client_hdl);
-	return ret;
-
-}
-
-/**
- * ipa3_odl_register_pm - Register odl client for PM
- *
- * This function will register 1 client with IPA PM to represent odl
- * in clock scaling calculation:
- *	- "ODL" - this client will be activated when pipe connected
- */
-static int ipa3_odl_register_pm(void)
-{
-	int result = 0;
-	struct ipa_pm_register_params pm_reg;
-
-	memset(&pm_reg, 0, sizeof(pm_reg));
-	pm_reg.name = "ODL";
-	pm_reg.group = IPA_PM_GROUP_DEFAULT;
-	pm_reg.skip_clk_vote = true;
-	result = ipa_pm_register(&pm_reg, &ipa3_odl_ctx->odl_pm_hdl);
-	if (result) {
-		IPAERR("failed to create IPA PM client %d\n", result);
-		return result;
-	}
-	return result;
-}
-
-int ipa3_odl_pipe_open(void)
-{
-	int ret = 0;
-	struct ipa_ep_cfg_holb holb_cfg;
-
-	if (!ipa3_odl_ctx->odl_state.adpl_open) {
-		IPAERR("adpl pipe not configured\n");
-		return 0;
-	}
-
-	memset(&holb_cfg, 0, sizeof(holb_cfg));
-	holb_cfg.tmr_val = 0;
-	holb_cfg.en = 1;
-
-	ipa3_cfg_ep_holb_by_client(IPA_CLIENT_USB_DPL_CONS, &holb_cfg);
-	ret = ipa_setup_odl_pipe();
-	if (ret) {
-		IPAERR(" Setup endpoint config failed\n");
-		ipa3_odl_ctx->odl_state.adpl_open = false;
-		goto fail;
-	}
-	ipa3_cfg_ep_holb_by_client(IPA_CLIENT_ODL_DPL_CONS, &holb_cfg);
-	ipa3_odl_ctx->odl_state.odl_ep_setup = true;
-	IPADBG("Setup endpoint config success\n");
-
-	ipa3_odl_ctx->stats.odl_drop_pkt = 0;
-	atomic_set(&ipa3_odl_ctx->stats.numer_in_queue, 0);
-	ipa3_odl_ctx->stats.odl_rx_pkt = 0;
-	ipa3_odl_ctx->stats.odl_tx_diag_pkt = 0;
-	/*
-	 * Send signal to ipa_odl_ctl_fops_read,
-	 * to send ODL ep open notification
-	 */
-	ipa3_odl_ctx->odl_ctl_msg_wq_flag = true;
-	IPADBG("Wake up odl ctl\n");
-	wake_up_interruptible(&odl_ctl_msg_wq);
-	if (ipa3_odl_ctx->odl_state.odl_disconnected)
-		ipa3_odl_ctx->odl_state.odl_disconnected = false;
-fail:
-	return ret;
-
-}
-static int ipa_adpl_open(struct inode *inode, struct file *filp)
-{
-	int ret = 0;
-
-	IPADBG("Called the function :\n");
-	if (ipa3_odl_ctx->odl_state.odl_init &&
-				!ipa3_odl_ctx->odl_state.adpl_open) {
-		/* Activate ipa_pm*/
-		ret = ipa_pm_activate_sync(ipa3_odl_ctx->odl_pm_hdl);
-		if (ret)
-			IPAERR("failed to activate pm\n");
-		ipa3_odl_ctx->odl_state.adpl_open = true;
-		ret = ipa3_odl_pipe_open();
-	} else {
-		IPAERR("Before odl init trying to open adpl pipe\n");
-		print_ipa_odl_state_bit_mask();
-		ret = -ENODEV;
-	}
-
-	return ret;
-}
-
-static int ipa_adpl_release(struct inode *inode, struct file *filp)
-{
-	int ret = 0;
-	/* Deactivate ipa_pm */
-	ret = ipa_pm_deactivate_sync(ipa3_odl_ctx->odl_pm_hdl);
-	if (ret)
-		IPAERR("failed to activate pm\n");
-	ipa3_odl_pipe_cleanup(false);
-	return ret;
-}
-
-void ipa3_odl_pipe_cleanup(bool is_ssr)
-{
-	bool ipa_odl_opened = false;
-	struct ipa_ep_cfg_holb holb_cfg;
-
-	if (!ipa3_odl_ctx->odl_state.adpl_open) {
-		IPAERR("adpl pipe not configured\n");
-		return;
-	}
-	if (ipa3_odl_ctx->odl_state.odl_open)
-		ipa_odl_opened = true;
-
-	memset(&ipa3_odl_ctx->odl_state, 0, sizeof(ipa3_odl_ctx->odl_state));
-
-	/*Since init will not be done again*/
-	ipa3_odl_ctx->odl_state.odl_init = true;
-	memset(&holb_cfg, 0, sizeof(holb_cfg));
-	holb_cfg.tmr_val = 0;
-	holb_cfg.en = 0;
-
-	ipa3_cfg_ep_holb_by_client(IPA_CLIENT_USB_DPL_CONS, &holb_cfg);
-
-	ipa3_teardown_sys_pipe(ipa3_odl_ctx->odl_client_hdl);
-	ipa3_odl_ctx->odl_client_hdl = -1;
-	/*Assume QTI will never close this node once opened*/
-	if (ipa_odl_opened)
-		ipa3_odl_ctx->odl_state.odl_open = true;
-
-	/*Assume DIAG will not close this node in SSR case*/
-	if (is_ssr)
-		ipa3_odl_ctx->odl_state.adpl_open = true;
-	else
-		ipa3_odl_ctx->odl_state.adpl_open = false;
-
-	ipa3_odl_ctx->odl_state.odl_disconnected = true;
-	ipa3_odl_ctx->odl_state.odl_ep_setup = false;
-	ipa3_odl_ctx->odl_state.aggr_byte_limit_sent = false;
-	ipa3_odl_ctx->odl_state.odl_connected = false;
-	/*
-	 * Send signal to ipa_odl_ctl_fops_read,
-	 * to send ODL ep close notification
-	 */
-	ipa3_odl_ctx->odl_ctl_msg_wq_flag = true;
-	ipa3_odl_ctx->stats.odl_drop_pkt = 0;
-	atomic_set(&ipa3_odl_ctx->stats.numer_in_queue, 0);
-	ipa3_odl_ctx->stats.odl_rx_pkt = 0;
-	ipa3_odl_ctx->stats.odl_tx_diag_pkt = 0;
-	IPADBG("Wake up odl ctl\n");
-	wake_up_interruptible(&odl_ctl_msg_wq);
-
-}
-
-/**
- * ipa_adpl_read() - read message from IPA device
- * @filp:	[in] file pointer
- * @buf:	[out] buffer to read into
- * @count:	[in] size of above buffer
- * @f_pos:	[inout] file position
- *
- * User-space should continually read from /dev/ipa_adpl,
- * read will block when there are no messages to read.
- * Upon return, user-space should read
- * Buffer supplied must be big enough to
- * hold the data.
- *
- * Returns:	how many bytes copied to buffer
- *
- * Note:	Should not be called from atomic context
- */
-static ssize_t ipa_adpl_read(struct file *filp, char __user *buf, size_t count,
-		  loff_t *f_pos)
-{
-	int ret =  0;
-	char __user *start = buf;
-	struct ipa3_push_msg_odl *msg;
-
-	while (1) {
-		IPADBG_LOW("Writing message to adpl pipe\n");
-		if (!ipa3_odl_ctx->odl_state.odl_open)
-			break;
-
-		mutex_lock(&ipa3_odl_ctx->adpl_msg_lock);
-		msg = NULL;
-		if (!list_empty(&ipa3_odl_ctx->adpl_msg_list)) {
-			msg = list_first_entry(&ipa3_odl_ctx->adpl_msg_list,
-					struct ipa3_push_msg_odl, link);
-			list_del(&msg->link);
-			if (atomic_read(&ipa3_odl_ctx->stats.numer_in_queue))
-				atomic_dec(&ipa3_odl_ctx->stats.numer_in_queue);
-		}
-
-		mutex_unlock(&ipa3_odl_ctx->adpl_msg_lock);
-
-		if (msg != NULL) {
-			if (msg->len > count) {
-				IPAERR("Message length greater than count\n");
-				kfree(msg->buff);
-				kfree(msg);
-				msg = NULL;
-				ret = -EAGAIN;
-				break;
-			}
-
-			if (msg->buff) {
-				if (copy_to_user(buf, msg->buff,
-							msg->len)) {
-					ret = -EFAULT;
-					kfree(msg->buff);
-					kfree(msg);
-					msg = NULL;
-					ret = -EAGAIN;
-					break;
-				}
-				buf += msg->len;
-				count -= msg->len;
-				kfree(msg->buff);
-			}
-			IPA_STATS_INC_CNT(ipa3_odl_ctx->stats.odl_tx_diag_pkt);
-			kfree(msg);
-			msg = NULL;
-		} else {
-			ret = -EAGAIN;
-			break;
-		}
-
-		ret = -EAGAIN;
-		if (filp->f_flags & O_NONBLOCK)
-			break;
-
-		ret = -EINTR;
-		if (signal_pending(current))
-			break;
-
-		if (start != buf)
-			break;
-
-	}
-
-	if (start != buf && ret != -EFAULT)
-		ret = buf - start;
-
-	return ret;
-}
-
-static long ipa_adpl_ioctl(struct file *filp,
-	unsigned int cmd, unsigned long arg)
-{
-	struct odl_agg_pipe_info odl_pipe_info;
-	int retval = 0;
-
-	if (!ipa3_odl_ctx->odl_state.odl_connected) {
-		IPAERR("ODL config in progress not allowed ioctl\n");
-		print_ipa_odl_state_bit_mask();
-		retval = -ENODEV;
-		goto fail;
-	}
-	IPADBG("Calling adpl ioctl\n");
-
-	switch (cmd) {
-	case IPA_IOC_ODL_GET_AGG_BYTE_LIMIT:
-		odl_pipe_info.agg_byte_limit =
-		ipa3_odl_ctx->odl_sys_param.ipa_ep_cfg.aggr.aggr_byte_limit;
-		if (copy_to_user((void __user *)arg, &odl_pipe_info,
-					sizeof(odl_pipe_info))) {
-			retval = -EFAULT;
-			goto fail;
-		}
-		ipa3_odl_ctx->odl_state.aggr_byte_limit_sent = true;
-		break;
-	default:
-		retval = -ENOIOCTLCMD;
-		print_ipa_odl_state_bit_mask();
-		break;
-	}
-
-fail:
-	return retval;
-}
-
-static const struct file_operations ipa_odl_ctl_fops = {
-	.owner = THIS_MODULE,
-	.open = ipa_odl_ctl_fops_open,
-	.release = ipa_odl_ctl_fops_release,
-	.read = ipa_odl_ctl_fops_read,
-	.unlocked_ioctl = ipa_odl_ctl_fops_ioctl,
-	.poll = ipa_odl_ctl_fops_poll,
-};
-
-static const struct file_operations ipa_adpl_fops = {
-	.owner = THIS_MODULE,
-	.open = ipa_adpl_open,
-	.release = ipa_adpl_release,
-	.read = ipa_adpl_read,
-	.unlocked_ioctl = ipa_adpl_ioctl,
-};
-
-int ipa_odl_init(void)
-{
-	int result = 0;
-	struct cdev *cdev;
-	int loop = 0;
-	struct ipa3_odl_char_device_context *odl_cdev;
-
-	ipa3_odl_ctx = kzalloc(sizeof(*ipa3_odl_ctx), GFP_KERNEL);
-	if (!ipa3_odl_ctx) {
-		result = -ENOMEM;
-		goto fail_mem_ctx;
-	}
-
-	odl_cdev = ipa3_odl_ctx->odl_cdev;
-	INIT_LIST_HEAD(&ipa3_odl_ctx->adpl_msg_list);
-	mutex_init(&ipa3_odl_ctx->adpl_msg_lock);
-
-	odl_cdev[loop].class = class_create(THIS_MODULE, "ipa_adpl");
-
-	if (IS_ERR(odl_cdev[loop].class)) {
-		IPAERR("Error: odl_cdev->class NULL\n");
-		result = -ENODEV;
-		goto create_char_dev0_fail;
-	}
-
-	result = alloc_chrdev_region(&odl_cdev[loop].dev_num, 0, 1, "ipa_adpl");
-	if (result) {
-		IPAERR("alloc_chrdev_region error for ipa adpl pipe\n");
-		result = -ENODEV;
-		goto alloc_chrdev0_region_fail;
-	}
-
-	odl_cdev[loop].dev = device_create(odl_cdev[loop].class, NULL,
-		 odl_cdev[loop].dev_num, ipa3_ctx, "ipa_adpl");
-	if (IS_ERR(odl_cdev[loop].dev)) {
-		IPAERR("device_create err:%ld\n", PTR_ERR(odl_cdev[loop].dev));
-		result = PTR_ERR(odl_cdev[loop].dev);
-		goto device0_create_fail;
-	}
-
-	cdev = &odl_cdev[loop].cdev;
-	cdev_init(cdev, &ipa_adpl_fops);
-	cdev->owner = THIS_MODULE;
-	cdev->ops = &ipa_adpl_fops;
-
-	result = cdev_add(cdev, odl_cdev[loop].dev_num, 1);
-	if (result) {
-		IPAERR("cdev_add err=%d\n", -result);
-		goto cdev0_add_fail;
-	}
-
-	loop++;
-
-	odl_cdev[loop].class = class_create(THIS_MODULE, "ipa_odl_ctl");
-
-	if (IS_ERR(odl_cdev[loop].class)) {
-		IPAERR("Error: odl_cdev->class NULL\n");
-		result =  -ENODEV;
-		goto create_char_dev1_fail;
-	}
-
-	result = alloc_chrdev_region(&odl_cdev[loop].dev_num, 0, 1,
-							"ipa_odl_ctl");
-	if (result) {
-		IPAERR("alloc_chrdev_region error for ipa odl ctl pipe\n");
-		goto alloc_chrdev1_region_fail;
-	}
-
-	odl_cdev[loop].dev = device_create(odl_cdev[loop].class, NULL,
-		 odl_cdev[loop].dev_num, ipa3_ctx, "ipa_odl_ctl");
-	if (IS_ERR(odl_cdev[loop].dev)) {
-		IPAERR("device_create err:%ld\n", PTR_ERR(odl_cdev[loop].dev));
-		result = PTR_ERR(odl_cdev[loop].dev);
-		goto device1_create_fail;
-	}
-
-	cdev = &odl_cdev[loop].cdev;
-	cdev_init(cdev, &ipa_odl_ctl_fops);
-	cdev->owner = THIS_MODULE;
-	cdev->ops = &ipa_odl_ctl_fops;
-
-	result = cdev_add(cdev, odl_cdev[loop].dev_num, 1);
-	if (result) {
-		IPAERR(":cdev_add err=%d\n", -result);
-		goto cdev1_add_fail;
-	}
-
-	ipa3_odl_ctx->odl_state.odl_init = true;
-
-	/* register ipa_pm */
-	result = ipa3_odl_register_pm();
-	if (result) {
-		IPAWANERR("ipa3_odl_register_pm failed, ret: %d\n",
-				result);
-	}
-	return 0;
-cdev1_add_fail:
-	device_destroy(odl_cdev[1].class, odl_cdev[1].dev_num);
-device1_create_fail:
-	unregister_chrdev_region(odl_cdev[1].dev_num, 1);
-alloc_chrdev1_region_fail:
-	class_destroy(odl_cdev[1].class);
-create_char_dev1_fail:
-cdev0_add_fail:
-	device_destroy(odl_cdev[0].class, odl_cdev[0].dev_num);
-device0_create_fail:
-	unregister_chrdev_region(odl_cdev[0].dev_num, 1);
-alloc_chrdev0_region_fail:
-	class_destroy(odl_cdev[0].class);
-create_char_dev0_fail:
-	kfree(ipa3_odl_ctx);
-fail_mem_ctx:
-	return result;
-}

ipa/ipa_v3/ipa_odl.h

@@ -1,74 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
- */
-
-#ifndef _IPA3_ODL_H_
-#define _IPA3_ODL_H_
-
-#define IPA_ODL_AGGR_BYTE_LIMIT (15 * 1024)
-#define IPA_ODL_RX_RING_SIZE 192
-#define MAX_QUEUE_TO_ODL 1024
-#define CONFIG_SUCCESS 1
-#define ODL_EP_TYPE_HSUSB 2
-#define ODL_EP_PERIPHERAL_IFACE_ID 3
-
-struct ipa3_odlstats {
-	u32 odl_rx_pkt;
-	u32 odl_tx_diag_pkt;
-	u32 odl_drop_pkt;
-	atomic_t numer_in_queue;
-};
-
-struct odl_state_bit_mask {
-	u32 odl_init:1;
-	u32 odl_open:1;
-	u32 adpl_open:1;
-	u32 aggr_byte_limit_sent:1;
-	u32 odl_ep_setup:1;
-	u32 odl_setup_done_sent:1;
-	u32 odl_ep_info_sent:1;
-	u32 odl_connected:1;
-	u32 odl_disconnected:1;
-	u32:0;
-};
-
-/**
- * struct ipa3_odl_char_device_context - IPA ODL character device
- * @class: pointer to the struct class
- * @dev_num: device number
- * @dev: the dev_t of the device
- * @cdev: cdev of the device
- */
-struct ipa3_odl_char_device_context {
-	struct class *class;
-	dev_t dev_num;
-	struct device *dev;
-	struct cdev cdev;
-};
-
-struct ipa_odl_context {
-	struct ipa3_odl_char_device_context odl_cdev[2];
-	struct list_head adpl_msg_list;
-	struct mutex adpl_msg_lock;
-	struct ipa_sys_connect_params odl_sys_param;
-	u32 odl_client_hdl;
-	struct odl_state_bit_mask odl_state;
-	bool odl_ctl_msg_wq_flag;
-	struct ipa3_odlstats stats;
-	u32 odl_pm_hdl;
-};
-
-struct ipa3_push_msg_odl {
-	void *buff;
-	int len;
-	struct list_head link;
-};
-
-extern struct ipa_odl_context *ipa3_odl_ctx;
-
-int ipa_odl_init(void);
-void ipa3_odl_pipe_cleanup(bool is_ssr);
-int ipa3_odl_pipe_open(void);
-
-#endif /* _IPA3_ODL_H_ */

ipa/ipa_v3/ipa_pm.c

@@ -1,1419 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2017-2019, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/debugfs.h>
-#include "ipa_pm.h"
-#include "ipa_i.h"
-
-
-#define IPA_PM_DRV_NAME "ipa_pm"
-
-#define IPA_PM_DBG(fmt, args...) \
-	do { \
-		pr_debug(IPA_PM_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-			IPA_PM_DRV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			IPA_PM_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-#define IPA_PM_DBG_LOW(fmt, args...) \
-	do { \
-		pr_debug(IPA_PM_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			IPA_PM_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-#define IPA_PM_ERR(fmt, args...) \
-	do { \
-		pr_err(IPA_PM_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-			IPA_PM_DRV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			IPA_PM_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-#define IPA_PM_DBG_STATE(hdl, name, state) \
-	IPA_PM_DBG_LOW("Client[%d] %s: %s\n", hdl, name, \
-		client_state_to_str[state])
-
-
-#if IPA_PM_MAX_CLIENTS > 32
-#error max client greater than 32 all bitmask types should be changed
-#endif
-
-/*
- * struct ipa_pm_exception_list - holds information about an exception
- * @pending: number of clients in exception that have not yet been adctivated
- * @bitmask: bitmask of the clients in the exception based on handle
- * @threshold: the threshold values for the exception
- */
-struct ipa_pm_exception_list {
-	char clients[IPA_PM_MAX_EX_CL];
-	int pending;
-	u32 bitmask;
-	int threshold[IPA_PM_THRESHOLD_MAX];
-};
-
-/*
- * struct clk_scaling_db - holds information about threshholds and exceptions
- * @lock: lock the bitmasks and thresholds
- * @exception_list: pointer to the list of exceptions
- * @work: work for clock scaling algorithm
- * @active_client_bitmask: the bits represent handles in the clients array that
- * contain non-null client
- * @threshold_size: size of the throughput threshold
- * @exception_size: size of the exception list
- * @cur_vote: idx of the threshold
- * @default_threshold: the thresholds used if no exception passes
- * @current_threshold: the current threshold of the clock plan
- */
-struct clk_scaling_db {
-	spinlock_t lock;
-	struct ipa_pm_exception_list exception_list[IPA_PM_EXCEPTION_MAX];
-	struct work_struct work;
-	u32 active_client_bitmask;
-	int threshold_size;
-	int exception_size;
-	int cur_vote;
-	int default_threshold[IPA_PM_THRESHOLD_MAX];
-	int *current_threshold;
-};
-
-/*
- * ipa_pm state names
- *
- * Timer free states:
- * @IPA_PM_DEACTIVATED: client starting state when registered
- * @IPA_PM_DEACTIVATE_IN_PROGRESS: deactivate was called in progress of a client
- *				   activating
- * @IPA_PM_ACTIVATE_IN_PROGRESS: client is being activated by work_queue
- * @IPA_PM_ACTIVATED: client is activated without any timers
- *
- * Timer set states:
- * @IPA_PM_ACTIVATED_PENDING_DEACTIVATION: moves to deactivate once timer pass
- * @IPA_PM_ACTIVATED_TIMER_SET: client was activated while timer was set, so
- *			 when the timer pass, client will still be activated
- *@IPA_PM_ACTIVATED_PENDING_RESCHEDULE: state signifying extended timer when
- *             a client is deferred_deactivated when a time ris still active
- */
-enum ipa_pm_state {
-	IPA_PM_DEACTIVATED,
-	IPA_PM_DEACTIVATE_IN_PROGRESS,
-	IPA_PM_ACTIVATE_IN_PROGRESS,
-	IPA_PM_ACTIVATED,
-	IPA_PM_ACTIVATED_PENDING_DEACTIVATION,
-	IPA_PM_ACTIVATED_TIMER_SET,
-	IPA_PM_ACTIVATED_PENDING_RESCHEDULE,
-	IPA_PM_STATE_MAX
-};
-
-#define IPA_PM_STATE_ACTIVE(state) \
-	(state == IPA_PM_ACTIVATED ||\
-		state == IPA_PM_ACTIVATED_PENDING_DEACTIVATION ||\
-		state  == IPA_PM_ACTIVATED_TIMER_SET ||\
-		state == IPA_PM_ACTIVATED_PENDING_RESCHEDULE)
-
-#define IPA_PM_STATE_IN_PROGRESS(state) \
-	(state == IPA_PM_ACTIVATE_IN_PROGRESS \
-		|| state == IPA_PM_DEACTIVATE_IN_PROGRESS)
-
-/*
- * struct ipa_pm_client - holds information about a specific IPA client
- * @name: string name of the client
- * @callback: pointer to the client's callback function
- * @callback_params: pointer to the client's callback parameters
- * @state: Activation state of the client
- * @skip_clk_vote: 0 if client votes for clock when activated, 1 if no vote
- * @group: the ipa_pm_group the client belongs to
- * @hdl: handle of the client
- * @throughput: the throughput of the client for clock scaling
- * @state_lock: spinlock to lock the pm_states
- * @activate_work: work for activate (blocking case)
- * @deactivate work: delayed work for deferred_deactivate function
- * @complete: generic wait-for-completion handler
- * @wlock: wake source to prevent AP suspend
- */
-struct ipa_pm_client {
-	char name[IPA_PM_MAX_EX_CL];
-	void (*callback)(void *user_data, enum ipa_pm_cb_event);
-	void *callback_params;
-	enum ipa_pm_state state;
-	bool skip_clk_vote;
-	int group;
-	int hdl;
-	int throughput;
-	spinlock_t state_lock;
-	struct work_struct activate_work;
-	struct delayed_work deactivate_work;
-	struct completion complete;
-	struct wakeup_source wlock;
-};
-
-/*
- * struct ipa_pm_ctx - global ctx that will hold the client arrays and tput info
- * @clients: array to the clients with the handle as its index
- * @clients_by_pipe: array to the clients with endpoint as the index
- * @wq: work queue for deferred deactivate, activate, and clk_scaling work
- 8 @clk_scaling: pointer to clock scaling database
- * @client_mutex: global mutex to  lock the client arrays
- * @aggragated_tput: aggragated tput value of all valid activated clients
- * @group_tput: combined throughput for the groups
- */
-struct ipa_pm_ctx {
-	struct ipa_pm_client *clients[IPA_PM_MAX_CLIENTS];
-	struct ipa_pm_client *clients_by_pipe[IPA3_MAX_NUM_PIPES];
-	struct workqueue_struct *wq;
-	struct clk_scaling_db clk_scaling;
-	struct mutex client_mutex;
-	int aggregated_tput;
-	int group_tput[IPA_PM_GROUP_MAX];
-};
-
-static struct ipa_pm_ctx *ipa_pm_ctx;
-
-static const char *client_state_to_str[IPA_PM_STATE_MAX] = {
-	__stringify(IPA_PM_DEACTIVATED),
-	__stringify(IPA_PM_DEACTIVATE_IN_PROGRESS),
-	__stringify(IPA_PM_ACTIVATE_IN_PROGRESS),
-	__stringify(IPA_PM_ACTIVATED),
-	__stringify(IPA_PM_ACTIVATED_PENDING_DEACTIVATION),
-	__stringify(IPA_PM_ACTIVATED_TIMER_SET),
-	__stringify(IPA_PM_ACTIVATED_PENDING_RESCHEDULE),
-};
-
-static const char *ipa_pm_group_to_str[IPA_PM_GROUP_MAX] = {
-	__stringify(IPA_PM_GROUP_DEFAULT),
-	__stringify(IPA_PM_GROUP_APPS),
-	__stringify(IPA_PM_GROUP_MODEM),
-};
-
-/**
- * pop_max_from_array() -pop the max and move the last element to where the
- * max was popped
- * @arr: array to be searched for max
- * @n: size of the array
- *
- * Returns: max value of the array
- */
-static int pop_max_from_array(int *arr, int *n)
-{
-	int i;
-	int max, max_idx;
-
-	max_idx = *n - 1;
-	max = 0;
-
-	if (*n == 0)
-		return 0;
-
-	for (i = 0; i < *n; i++) {
-		if (arr[i] > max) {
-			max = arr[i];
-			max_idx = i;
-		}
-	}
-	(*n)--;
-	arr[max_idx] = arr[*n];
-
-	return max;
-}
-
-/**
- * calculate_throughput() - calculate the aggregated throughput
- * based on active clients
- *
- * Returns: aggregated tput value
- */
-static int calculate_throughput(void)
-{
-	int client_tput[IPA_PM_MAX_CLIENTS] = { 0 };
-	bool group_voted[IPA_PM_GROUP_MAX] = { false };
-	int i, n;
-	int max, second_max, aggregated_tput;
-	struct ipa_pm_client *client;
-
-	/* Create a basic array to hold throughputs*/
-	for (i = 1, n = 0; i < IPA_PM_MAX_CLIENTS; i++) {
-		client = ipa_pm_ctx->clients[i];
-		if (client != NULL && IPA_PM_STATE_ACTIVE(client->state)) {
-			/* default case */
-			if (client->group == IPA_PM_GROUP_DEFAULT) {
-				client_tput[n++] = client->throughput;
-			} else if (!group_voted[client->group]) {
-				client_tput[n++] = ipa_pm_ctx->group_tput
-					[client->group];
-				group_voted[client->group] = true;
-			}
-		}
-	}
-	/*the array will only use n+1 spots. n will be the last index used*/
-
-	aggregated_tput = 0;
-
-	/**
-	 * throughput algorithm:
-	 * 1) pop the max and second_max
-	 * 2) add the 2nd max to aggregated tput
-	 * 3) insert the value of max - 2nd max
-	 * 4) repeat until array is of size 1
-	 */
-	while (n > 1) {
-		max = pop_max_from_array(client_tput, &n);
-		second_max = pop_max_from_array(client_tput, &n);
-		client_tput[n++] = max - second_max;
-		aggregated_tput += second_max;
-	}
-
-	IPA_PM_DBG_LOW("Aggregated throughput: %d\n", aggregated_tput);
-
-	return aggregated_tput;
-}
-
-/**
- * deactivate_client() - turn off the bit in the active client bitmask based on
- * the handle passed in
- * @hdl: The index of the client to be deactivated
- */
-static void deactivate_client(u32 hdl)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&ipa_pm_ctx->clk_scaling.lock, flags);
-	ipa_pm_ctx->clk_scaling.active_client_bitmask &= ~(1 << hdl);
-	spin_unlock_irqrestore(&ipa_pm_ctx->clk_scaling.lock, flags);
-	IPA_PM_DBG_LOW("active bitmask: %x\n",
-		ipa_pm_ctx->clk_scaling.active_client_bitmask);
-}
-
-/**
- * activate_client() - turn on the bit in the active client bitmask based on
- * the handle passed in
- * @hdl: The index of the client to be activated
- */
-static void activate_client(u32 hdl)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&ipa_pm_ctx->clk_scaling.lock, flags);
-	ipa_pm_ctx->clk_scaling.active_client_bitmask |= (1 << hdl);
-	spin_unlock_irqrestore(&ipa_pm_ctx->clk_scaling.lock, flags);
-	IPA_PM_DBG_LOW("active bitmask: %x\n",
-		ipa_pm_ctx->clk_scaling.active_client_bitmask);
-}
-
-/**
- * deactivate_client() - get threshold
- *
- * Returns: threshold of the exception that passes or default if none pass
- */
-static void set_current_threshold(void)
-{
-	int i;
-	struct clk_scaling_db *clk;
-	struct ipa_pm_exception_list *exception;
-	unsigned long flags;
-
-	clk = &ipa_pm_ctx->clk_scaling;
-
-	spin_lock_irqsave(&ipa_pm_ctx->clk_scaling.lock, flags);
-	for (i = 0; i < clk->exception_size; i++) {
-		exception = &clk->exception_list[i];
-		if (exception->pending == 0 && (exception->bitmask
-			& ~clk->active_client_bitmask) == 0) {
-			spin_unlock_irqrestore(&ipa_pm_ctx->clk_scaling.lock,
-				 flags);
-			clk->current_threshold = exception->threshold;
-			IPA_PM_DBG("Exception %d set\n", i);
-			return;
-		}
-	}
-	clk->current_threshold = clk->default_threshold;
-	spin_unlock_irqrestore(&ipa_pm_ctx->clk_scaling.lock, flags);
-}
-
-/**
- * do_clk_scaling() - set the clock based on the activated clients
- *
- * Returns: 0 if success, negative otherwise
- */
-static int do_clk_scaling(void)
-{
-	int i, tput;
-	int new_th_idx = 1;
-	struct clk_scaling_db *clk_scaling;
-
-	if (atomic_read(&ipa3_ctx->ipa_clk_vote) == 0) {
-		IPA_PM_DBG("IPA clock is gated\n");
-		return 0;
-	}
-
-	clk_scaling = &ipa_pm_ctx->clk_scaling;
-
-	mutex_lock(&ipa_pm_ctx->client_mutex);
-	IPA_PM_DBG_LOW("clock scaling started\n");
-	tput = calculate_throughput();
-	ipa_pm_ctx->aggregated_tput = tput;
-	set_current_threshold();
-
-	mutex_unlock(&ipa_pm_ctx->client_mutex);
-
-	for (i = 0; i < clk_scaling->threshold_size; i++) {
-		if (tput >= clk_scaling->current_threshold[i])
-			new_th_idx++;
-	}
-
-	IPA_PM_DBG_LOW("old idx was at %d\n", ipa_pm_ctx->clk_scaling.cur_vote);
-
-
-	if (ipa_pm_ctx->clk_scaling.cur_vote != new_th_idx) {
-		ipa_pm_ctx->clk_scaling.cur_vote = new_th_idx;
-		ipa3_set_clock_plan_from_pm(ipa_pm_ctx->clk_scaling.cur_vote);
-	}
-
-	IPA_PM_DBG_LOW("new idx is at %d\n", ipa_pm_ctx->clk_scaling.cur_vote);
-
-	return 0;
-}
-
-/**
- * clock_scaling_func() - set the clock on a work queue
- */
-static void clock_scaling_func(struct work_struct *work)
-{
-	do_clk_scaling();
-}
-
-/**
- * activate_work_func - activate a client and vote for clock on a work queue
- */
-static void activate_work_func(struct work_struct *work)
-{
-	struct ipa_pm_client *client;
-	bool dec_clk = false;
-	unsigned long flags;
-
-	client = container_of(work, struct ipa_pm_client, activate_work);
-	if (!client->skip_clk_vote) {
-		IPA_ACTIVE_CLIENTS_INC_SPECIAL(client->name);
-		if (client->group == IPA_PM_GROUP_APPS)
-			__pm_stay_awake(&client->wlock);
-	}
-
-	spin_lock_irqsave(&client->state_lock, flags);
-	IPA_PM_DBG_STATE(client->hdl, client->name, client->state);
-	if (client->state == IPA_PM_ACTIVATE_IN_PROGRESS) {
-		client->state = IPA_PM_ACTIVATED;
-	} else if (client->state == IPA_PM_DEACTIVATE_IN_PROGRESS) {
-		client->state = IPA_PM_DEACTIVATED;
-		dec_clk = true;
-	} else {
-		IPA_PM_ERR("unexpected state %d\n", client->state);
-		WARN_ON(1);
-	}
-	spin_unlock_irqrestore(&client->state_lock, flags);
-
-	complete_all(&client->complete);
-
-	if (dec_clk) {
-		if (!client->skip_clk_vote) {
-			IPA_ACTIVE_CLIENTS_DEC_SPECIAL(client->name);
-			if (client->group == IPA_PM_GROUP_APPS)
-				__pm_relax(&client->wlock);
-		}
-
-		IPA_PM_DBG_STATE(client->hdl, client->name, client->state);
-		return;
-	}
-
-	activate_client(client->hdl);
-
-	mutex_lock(&ipa_pm_ctx->client_mutex);
-	if (client->callback) {
-		client->callback(client->callback_params,
-			IPA_PM_CLIENT_ACTIVATED);
-	} else {
-		IPA_PM_ERR("client has no callback");
-		WARN_ON(1);
-	}
-	mutex_unlock(&ipa_pm_ctx->client_mutex);
-
-	IPA_PM_DBG_STATE(client->hdl, client->name, client->state);
-	do_clk_scaling();
-}
-
-/**
- * delayed_deferred_deactivate_work_func - deferred deactivate on a work queue
- */
-static void delayed_deferred_deactivate_work_func(struct work_struct *work)
-{
-	struct delayed_work *dwork;
-	struct ipa_pm_client *client;
-	unsigned long flags;
-	unsigned long delay;
-
-	dwork = container_of(work, struct delayed_work, work);
-	client = container_of(dwork, struct ipa_pm_client, deactivate_work);
-
-	spin_lock_irqsave(&client->state_lock, flags);
-	IPA_PM_DBG_STATE(client->hdl, client->name, client->state);
-	switch (client->state) {
-	case IPA_PM_ACTIVATED_TIMER_SET:
-		client->state = IPA_PM_ACTIVATED;
-		goto bail;
-	case IPA_PM_ACTIVATED_PENDING_RESCHEDULE:
-		delay = IPA_PM_DEFERRED_TIMEOUT;
-		if (ipa3_ctx->ipa3_hw_mode == IPA_HW_MODE_VIRTUAL ||
-			ipa3_ctx->ipa3_hw_mode == IPA_HW_MODE_EMULATION)
-			delay *= 5;
-
-		queue_delayed_work(ipa_pm_ctx->wq, &client->deactivate_work,
-			msecs_to_jiffies(delay));
-		client->state = IPA_PM_ACTIVATED_PENDING_DEACTIVATION;
-		goto bail;
-	case IPA_PM_ACTIVATED_PENDING_DEACTIVATION:
-		client->state = IPA_PM_DEACTIVATED;
-		IPA_PM_DBG_STATE(client->hdl, client->name, client->state);
-		spin_unlock_irqrestore(&client->state_lock, flags);
-		if (!client->skip_clk_vote) {
-			IPA_ACTIVE_CLIENTS_DEC_SPECIAL(client->name);
-			if (client->group == IPA_PM_GROUP_APPS)
-				__pm_relax(&client->wlock);
-		}
-
-		deactivate_client(client->hdl);
-		do_clk_scaling();
-		return;
-	default:
-		IPA_PM_ERR("unexpected state %d\n", client->state);
-		WARN_ON(1);
-		goto bail;
-	}
-
-bail:
-	IPA_PM_DBG_STATE(client->hdl, client->name, client->state);
-	spin_unlock_irqrestore(&client->state_lock, flags);
-}
-
-static int find_next_open_array_element(const char *name)
-{
-	int i, n;
-
-	n = -ENOBUFS;
-
-	/* 0 is not a valid handle */
-	for (i = IPA_PM_MAX_CLIENTS - 1; i >= 1; i--) {
-		if (ipa_pm_ctx->clients[i] == NULL) {
-			n = i;
-			continue;
-		}
-
-		if (strlen(name) == strlen(ipa_pm_ctx->clients[i]->name))
-			if (!strcmp(name, ipa_pm_ctx->clients[i]->name))
-				return -EEXIST;
-	}
-	return n;
-}
-
-/**
- * add_client_to_exception_list() - add client to the exception list and
- * update pending if necessary
- * @hdl: index of the IPA client
- *
- * Returns: 0 if success, negative otherwise
- */
-static int add_client_to_exception_list(u32 hdl)
-{
-	int i, len = 0;
-	struct ipa_pm_exception_list *exception;
-
-	mutex_lock(&ipa_pm_ctx->client_mutex);
-	len = strlen(ipa_pm_ctx->clients[hdl]->name);
-	for (i = 0; i < ipa_pm_ctx->clk_scaling.exception_size; i++) {
-		exception = &ipa_pm_ctx->clk_scaling.exception_list[i];
-		if (strnstr(exception->clients, ipa_pm_ctx->clients[hdl]->name,
-			len) && (strlen(exception->clients)
-			== len)) {
-			exception->pending--;
-			IPA_PM_DBG("Pending: %d\n",
-			exception->pending);
-
-			if (exception->pending < 0) {
-				WARN_ON(1);
-				exception->pending = 0;
-				mutex_unlock(&ipa_pm_ctx->client_mutex);
-				return -EPERM;
-			}
-			exception->bitmask |= (1 << hdl);
-		}
-	}
-	IPA_PM_DBG("%s added to exception list\n",
-		ipa_pm_ctx->clients[hdl]->name);
-	mutex_unlock(&ipa_pm_ctx->client_mutex);
-
-	return 0;
-}
-
-/**
- * remove_client_to_exception_list() - remove client from the exception list and
- * update pending if necessary
- * @hdl: index of the IPA client
- *
- * Returns: 0 if success, negative otherwise
- */
-static int remove_client_from_exception_list(u32 hdl)
-{
-	int i;
-	struct ipa_pm_exception_list *exception;
-
-	for (i = 0; i < ipa_pm_ctx->clk_scaling.exception_size; i++) {
-		exception = &ipa_pm_ctx->clk_scaling.exception_list[i];
-		if (exception->bitmask & (1 << hdl)) {
-			exception->pending++;
-			IPA_PM_DBG("Pending: %d\n",
-			exception->pending);
-			exception->bitmask &= ~(1 << hdl);
-		}
-	}
-	IPA_PM_DBG("Client %d removed from exception list\n", hdl);
-
-	return 0;
-}
-
-/**
- * ipa_pm_init() - initialize  IPA PM Components
- * @ipa_pm_init_params: parameters needed to fill exceptions and thresholds
- *
- * Returns: 0 on success, negative on failure
- */
-int ipa_pm_init(struct ipa_pm_init_params *params)
-{
-	int i, j;
-	struct clk_scaling_db *clk_scaling;
-
-	if (params == NULL) {
-		IPA_PM_ERR("Invalid Params\n");
-		return -EINVAL;
-	}
-
-	if (params->threshold_size <= 0
-		|| params->threshold_size > IPA_PM_THRESHOLD_MAX) {
-		IPA_PM_ERR("Invalid threshold size\n");
-		return -EINVAL;
-	}
-
-	if (params->exception_size < 0
-		|| params->exception_size > IPA_PM_EXCEPTION_MAX) {
-		IPA_PM_ERR("Invalid exception size\n");
-		return -EINVAL;
-	}
-
-	IPA_PM_DBG("IPA PM initialization started\n");
-
-	if (ipa_pm_ctx != NULL) {
-		IPA_PM_ERR("Already initialized\n");
-		return -EPERM;
-	}
-
-
-	ipa_pm_ctx = kzalloc(sizeof(*ipa_pm_ctx), GFP_KERNEL);
-	if (!ipa_pm_ctx) {
-		IPA_PM_ERR(":kzalloc err.\n");
-		return -ENOMEM;
-	}
-
-	ipa_pm_ctx->wq = create_singlethread_workqueue("ipa_pm_activate");
-	if (!ipa_pm_ctx->wq) {
-		IPA_PM_ERR("create workqueue failed\n");
-		kfree(ipa_pm_ctx);
-		return -ENOMEM;
-	}
-
-	mutex_init(&ipa_pm_ctx->client_mutex);
-
-	/* Populate and init locks in clk_scaling_db */
-	clk_scaling = &ipa_pm_ctx->clk_scaling;
-	spin_lock_init(&clk_scaling->lock);
-	clk_scaling->threshold_size = params->threshold_size;
-	clk_scaling->exception_size = params->exception_size;
-	INIT_WORK(&clk_scaling->work, clock_scaling_func);
-
-	for (i = 0; i < params->threshold_size; i++)
-		clk_scaling->default_threshold[i] =
-			params->default_threshold[i];
-
-	/* Populate exception list*/
-	for (i = 0; i < params->exception_size; i++) {
-		strlcpy(clk_scaling->exception_list[i].clients,
-			params->exceptions[i].usecase, IPA_PM_MAX_EX_CL);
-		IPA_PM_DBG("Usecase: %s\n", params->exceptions[i].usecase);
-
-		/* Parse the commas to count the size of the clients */
-		for (j = 0; j < IPA_PM_MAX_EX_CL &&
-			clk_scaling->exception_list[i].clients[j]; j++) {
-			if (clk_scaling->exception_list[i].clients[j] == ',')
-				clk_scaling->exception_list[i].pending++;
-		}
-
-		/* for the first client */
-		clk_scaling->exception_list[i].pending++;
-		IPA_PM_DBG("Pending: %d\n",
-			clk_scaling->exception_list[i].pending);
-
-		/* populate the threshold */
-		for (j = 0; j < params->threshold_size; j++) {
-			clk_scaling->exception_list[i].threshold[j]
-			= params->exceptions[i].threshold[j];
-		}
-
-	}
-	IPA_PM_DBG("initialization success");
-
-	return 0;
-}
-
-int ipa_pm_destroy(void)
-{
-	IPA_PM_DBG("IPA PM destroy started\n");
-
-	if (ipa_pm_ctx == NULL) {
-		IPA_PM_ERR("Already destroyed\n");
-		return -EPERM;
-	}
-
-	destroy_workqueue(ipa_pm_ctx->wq);
-
-	kfree(ipa_pm_ctx);
-	ipa_pm_ctx = NULL;
-
-	return 0;
-}
-
-/**
- * ipa_pm_register() - register an IPA PM client with the PM
- * @register_params: params for a client like throughput, callback, etc.
- * @hdl: int pointer that will be used as an index to access the client
- *
- * Returns: 0 on success, negative on failure
- *
- * Side effects: *hdl is replaced with the client index or -EEXIST if
- * client is already registered
- */
-int ipa_pm_register(struct ipa_pm_register_params *params, u32 *hdl)
-{
-	struct ipa_pm_client *client;
-	struct wakeup_source *wlock;
-	int elem;
-
-	if (ipa_pm_ctx == NULL) {
-		IPA_PM_ERR("PM_ctx is null\n");
-		return -EINVAL;
-	}
-
-	if (params == NULL || hdl == NULL || params->name == NULL) {
-		IPA_PM_ERR("Invalid Params\n");
-		return -EINVAL;
-	}
-
-	IPA_PM_DBG("IPA PM registering client\n");
-
-	mutex_lock(&ipa_pm_ctx->client_mutex);
-
-	elem = find_next_open_array_element(params->name);
-	*hdl = elem;
-	if (elem < 0 || elem > IPA_PM_MAX_CLIENTS) {
-		mutex_unlock(&ipa_pm_ctx->client_mutex);
-		IPA_PM_ERR("client already registered or full array elem=%d\n",
-			elem);
-		return elem;
-	}
-
-	ipa_pm_ctx->clients[*hdl] = kzalloc(sizeof
-		(struct ipa_pm_client), GFP_KERNEL);
-	if (!ipa_pm_ctx->clients[*hdl]) {
-		mutex_unlock(&ipa_pm_ctx->client_mutex);
-		IPA_PM_ERR(":kzalloc err.\n");
-		return -ENOMEM;
-	}
-	mutex_unlock(&ipa_pm_ctx->client_mutex);
-
-	client = ipa_pm_ctx->clients[*hdl];
-
-	spin_lock_init(&client->state_lock);
-
-	INIT_DELAYED_WORK(&client->deactivate_work,
-		delayed_deferred_deactivate_work_func);
-
-	INIT_WORK(&client->activate_work, activate_work_func);
-
-	/* populate fields */
-	strlcpy(client->name, params->name, IPA_PM_MAX_EX_CL);
-	client->callback = params->callback;
-	client->callback_params = params->user_data;
-	client->group = params->group;
-	client->hdl = *hdl;
-	client->skip_clk_vote = params->skip_clk_vote;
-	wlock = &client->wlock;
-	wakeup_source_init(wlock, client->name);
-
-	init_completion(&client->complete);
-
-	/* add client to exception list */
-	if (add_client_to_exception_list(*hdl)) {
-		ipa_pm_deregister(*hdl);
-		IPA_PM_ERR("Fail to add client to exception_list\n");
-		return -EPERM;
-	}
-
-	IPA_PM_DBG("IPA PM client registered with handle %d\n", *hdl);
-	return 0;
-}
-
-/**
- * ipa_pm_deregister() - deregister IPA client from the PM
- * @hdl: index of the client in the array
- *
- * Returns: 0 on success, negative on failure
- */
-int ipa_pm_deregister(u32 hdl)
-{
-	struct ipa_pm_client *client;
-	int i;
-	unsigned long flags;
-
-	if (ipa_pm_ctx == NULL) {
-		IPA_PM_ERR("PM_ctx is null\n");
-		return -EINVAL;
-	}
-
-	if (hdl >= IPA_PM_MAX_CLIENTS) {
-		IPA_PM_ERR("Invalid Param\n");
-		return -EINVAL;
-	}
-
-	if (ipa_pm_ctx->clients[hdl] == NULL) {
-		IPA_PM_ERR("Client is Null\n");
-		return -EINVAL;
-	}
-
-	IPA_PM_DBG("IPA PM deregistering client\n");
-
-	client = ipa_pm_ctx->clients[hdl];
-	spin_lock_irqsave(&client->state_lock, flags);
-	if (IPA_PM_STATE_IN_PROGRESS(client->state)) {
-		spin_unlock_irqrestore(&client->state_lock, flags);
-		wait_for_completion(&client->complete);
-		spin_lock_irqsave(&client->state_lock, flags);
-	}
-
-	if (IPA_PM_STATE_ACTIVE(client->state)) {
-		IPA_PM_DBG("Activated clients cannot be deregistered");
-		spin_unlock_irqrestore(&client->state_lock, flags);
-		return -EPERM;
-	}
-	spin_unlock_irqrestore(&client->state_lock, flags);
-
-	mutex_lock(&ipa_pm_ctx->client_mutex);
-
-	/* nullify pointers in pipe array */
-	for (i = 0; i < IPA3_MAX_NUM_PIPES; i++) {
-		if (ipa_pm_ctx->clients_by_pipe[i] == ipa_pm_ctx->clients[hdl])
-			ipa_pm_ctx->clients_by_pipe[i] = NULL;
-	}
-	wakeup_source_trash(&client->wlock);
-	kfree(client);
-	ipa_pm_ctx->clients[hdl] = NULL;
-
-	remove_client_from_exception_list(hdl);
-	IPA_PM_DBG("IPA PM client %d deregistered\n", hdl);
-	mutex_unlock(&ipa_pm_ctx->client_mutex);
-
-	return 0;
-}
-
-/**
- * ipa_pm_associate_ipa_cons_to_client() - add mapping to pipe with ipa cllent
- * @hdl: index of the client to be mapped
- * @consumer: the pipe/consumer name to be pipped to the client
- *
- * Returns: 0 on success, negative on failure
- *
- * Side effects: multiple pipes are allowed to be mapped to a single client
- */
-int ipa_pm_associate_ipa_cons_to_client(u32 hdl, enum ipa_client_type consumer)
-{
-	int idx;
-
-	if (ipa_pm_ctx == NULL) {
-		IPA_PM_ERR("PM_ctx is null\n");
-		return -EINVAL;
-	}
-
-	if (hdl >= IPA_PM_MAX_CLIENTS || consumer < 0 ||
-		consumer >= IPA_CLIENT_MAX) {
-		IPA_PM_ERR("invalid params\n");
-		return -EINVAL;
-	}
-
-	mutex_lock(&ipa_pm_ctx->client_mutex);
-	if (ipa_pm_ctx->clients[hdl] == NULL) {
-		mutex_unlock(&ipa_pm_ctx->client_mutex);
-		IPA_PM_ERR("Client is NULL\n");
-		return -EPERM;
-	}
-
-	idx = ipa_get_ep_mapping(consumer);
-
-	if (idx < 0) {
-		mutex_unlock(&ipa_pm_ctx->client_mutex);
-		IPA_PM_DBG("Pipe is not used\n");
-		return 0;
-	}
-
-	IPA_PM_DBG("Mapping pipe %d to client %d\n", idx, hdl);
-
-	if (ipa_pm_ctx->clients_by_pipe[idx] != NULL) {
-		mutex_unlock(&ipa_pm_ctx->client_mutex);
-		IPA_PM_ERR("Pipe is already mapped\n");
-		return -EPERM;
-	}
-	ipa_pm_ctx->clients_by_pipe[idx] = ipa_pm_ctx->clients[hdl];
-	mutex_unlock(&ipa_pm_ctx->client_mutex);
-
-	IPA_PM_DBG("Pipe %d is mapped to client %d\n", idx, hdl);
-
-	return 0;
-}
-
-static int ipa_pm_activate_helper(struct ipa_pm_client *client, bool sync)
-{
-	struct ipa_active_client_logging_info log_info;
-	int result = 0;
-	unsigned long flags;
-
-	spin_lock_irqsave(&client->state_lock, flags);
-	IPA_PM_DBG_STATE(client->hdl, client->name, client->state);
-
-	if (IPA_PM_STATE_IN_PROGRESS(client->state)) {
-		if (sync) {
-			spin_unlock_irqrestore(&client->state_lock, flags);
-			wait_for_completion(&client->complete);
-			spin_lock_irqsave(&client->state_lock, flags);
-		} else {
-			client->state = IPA_PM_ACTIVATE_IN_PROGRESS;
-			spin_unlock_irqrestore(&client->state_lock, flags);
-			return -EINPROGRESS;
-		}
-	}
-
-	switch (client->state) {
-	case IPA_PM_ACTIVATED_PENDING_RESCHEDULE:
-	case IPA_PM_ACTIVATED_PENDING_DEACTIVATION:
-		client->state = IPA_PM_ACTIVATED_TIMER_SET;
-	case IPA_PM_ACTIVATED:
-	case IPA_PM_ACTIVATED_TIMER_SET:
-		spin_unlock_irqrestore(&client->state_lock, flags);
-		return 0;
-	case IPA_PM_DEACTIVATED:
-		break;
-	default:
-		IPA_PM_ERR("Invalid State\n");
-		spin_unlock_irqrestore(&client->state_lock, flags);
-		return -EPERM;
-	}
-	IPA_PM_DBG_STATE(client->hdl, client->name, client->state);
-
-	IPA_ACTIVE_CLIENTS_PREP_SPECIAL(log_info, client->name);
-	if (!client->skip_clk_vote) {
-		if (sync) {
-			client->state = IPA_PM_ACTIVATE_IN_PROGRESS;
-			spin_unlock_irqrestore(&client->state_lock, flags);
-			IPA_ACTIVE_CLIENTS_INC_SPECIAL(client->name);
-			spin_lock_irqsave(&client->state_lock, flags);
-		} else
-			result = ipa3_inc_client_enable_clks_no_block
-				 (&log_info);
-	}
-
-	/* we got the clocks */
-	if (result == 0) {
-		client->state = IPA_PM_ACTIVATED;
-		if (client->group == IPA_PM_GROUP_APPS)
-			__pm_stay_awake(&client->wlock);
-		spin_unlock_irqrestore(&client->state_lock, flags);
-		activate_client(client->hdl);
-		if (sync)
-			do_clk_scaling();
-		else
-			queue_work(ipa_pm_ctx->wq,
-				   &ipa_pm_ctx->clk_scaling.work);
-		IPA_PM_DBG_STATE(client->hdl, client->name, client->state);
-		return 0;
-	}
-
-	client->state = IPA_PM_ACTIVATE_IN_PROGRESS;
-	reinit_completion(&client->complete);
-	queue_work(ipa_pm_ctx->wq, &client->activate_work);
-	spin_unlock_irqrestore(&client->state_lock, flags);
-	IPA_PM_DBG_STATE(client->hdl, client->name, client->state);
-	return -EINPROGRESS;
-}
-
-/**
- * ipa_pm_activate(): activate ipa client to vote for clock(). Can be called
- * from atomic context and returns -EINPROGRESS if cannot be done synchronously
- * @hdl: index of the client in the array
- *
- * Returns: 0 on success, -EINPROGRESS if operation cannot be done synchronously
- * and other negatives on failure
- */
-int ipa_pm_activate(u32 hdl)
-{
-	if (ipa_pm_ctx == NULL) {
-		IPA_PM_ERR("PM_ctx is null\n");
-		return -EINVAL;
-	}
-
-	if (hdl >= IPA_PM_MAX_CLIENTS || ipa_pm_ctx->clients[hdl] == NULL) {
-		IPA_PM_ERR("Invalid Param\n");
-		return -EINVAL;
-	}
-
-	return ipa_pm_activate_helper(ipa_pm_ctx->clients[hdl], false);
-}
-
-/**
- * ipa_pm_activate(): activate ipa client to vote for clock synchronously.
- * Cannot be called from an atomic contex.
- * @hdl: index of the client in the array
- *
- * Returns: 0 on success, negative on failure
- */
-int ipa_pm_activate_sync(u32 hdl)
-{
-	if (ipa_pm_ctx == NULL) {
-		IPA_PM_ERR("PM_ctx is null\n");
-		return -EINVAL;
-	}
-
-	if (hdl >= IPA_PM_MAX_CLIENTS || ipa_pm_ctx->clients[hdl] == NULL) {
-		IPA_PM_ERR("Invalid Param\n");
-		return -EINVAL;
-	}
-
-	return ipa_pm_activate_helper(ipa_pm_ctx->clients[hdl], true);
-}
-
-/**
- * ipa_pm_deferred_deactivate(): schedule a timer to deactivate client and
- * devote clock. Can be called from atomic context (asynchronously)
- * @hdl: index of the client in the array
- *
- * Returns: 0 on success, negative on failure
- */
-int ipa_pm_deferred_deactivate(u32 hdl)
-{
-	struct ipa_pm_client *client;
-	unsigned long flags;
-	unsigned long delay;
-
-	if (ipa_pm_ctx == NULL) {
-		IPA_PM_ERR("PM_ctx is null\n");
-		return -EINVAL;
-	}
-
-	if (hdl >= IPA_PM_MAX_CLIENTS || ipa_pm_ctx->clients[hdl] == NULL) {
-		IPA_PM_ERR("Invalid Param\n");
-		return -EINVAL;
-	}
-
-	client = ipa_pm_ctx->clients[hdl];
-	IPA_PM_DBG_STATE(hdl, client->name, client->state);
-
-	spin_lock_irqsave(&client->state_lock, flags);
-	switch (client->state) {
-	case IPA_PM_ACTIVATE_IN_PROGRESS:
-		client->state = IPA_PM_DEACTIVATE_IN_PROGRESS;
-	case IPA_PM_DEACTIVATED:
-		IPA_PM_DBG_STATE(hdl, client->name, client->state);
-		spin_unlock_irqrestore(&client->state_lock, flags);
-		return 0;
-	case IPA_PM_ACTIVATED:
-		delay = IPA_PM_DEFERRED_TIMEOUT;
-		if (ipa3_ctx->ipa3_hw_mode == IPA_HW_MODE_VIRTUAL ||
-			ipa3_ctx->ipa3_hw_mode == IPA_HW_MODE_EMULATION)
-			delay *= 5;
-
-		client->state = IPA_PM_ACTIVATED_PENDING_DEACTIVATION;
-		queue_delayed_work(ipa_pm_ctx->wq, &client->deactivate_work,
-			msecs_to_jiffies(delay));
-		break;
-	case IPA_PM_ACTIVATED_TIMER_SET:
-	case IPA_PM_ACTIVATED_PENDING_DEACTIVATION:
-		client->state = IPA_PM_ACTIVATED_PENDING_RESCHEDULE;
-	case IPA_PM_DEACTIVATE_IN_PROGRESS:
-	case IPA_PM_ACTIVATED_PENDING_RESCHEDULE:
-		break;
-	case IPA_PM_STATE_MAX:
-	default:
-		IPA_PM_ERR("Bad State");
-		spin_unlock_irqrestore(&client->state_lock, flags);
-		return -EINVAL;
-	}
-	IPA_PM_DBG_STATE(hdl, client->name, client->state);
-	spin_unlock_irqrestore(&client->state_lock, flags);
-
-	return 0;
-}
-
-/**
- * ipa_pm_deactivate_all_deferred(): Cancel the deferred deactivation timer and
- * immediately devotes for IPA clocks
- *
- * Returns: 0 on success, negative on failure
- */
-int ipa_pm_deactivate_all_deferred(void)
-{
-	int i;
-	bool run_algorithm = false;
-	struct ipa_pm_client *client;
-	unsigned long flags;
-
-	if (ipa_pm_ctx == NULL) {
-		IPA_PM_ERR("PM_ctx is null\n");
-		return -EINVAL;
-	}
-
-	for (i = 1; i < IPA_PM_MAX_CLIENTS; i++) {
-		client = ipa_pm_ctx->clients[i];
-
-		if (client == NULL)
-			continue;
-
-		cancel_delayed_work_sync(&client->deactivate_work);
-
-		if (IPA_PM_STATE_IN_PROGRESS(client->state)) {
-			wait_for_completion(&client->complete);
-			continue;
-		}
-
-		spin_lock_irqsave(&client->state_lock, flags);
-		IPA_PM_DBG_STATE(client->hdl, client->name, client->state);
-
-		if (client->state == IPA_PM_ACTIVATED_TIMER_SET) {
-			client->state = IPA_PM_ACTIVATED;
-			IPA_PM_DBG_STATE(client->hdl, client->name,
-				client->state);
-			spin_unlock_irqrestore(&client->state_lock, flags);
-		} else if (client->state ==
-				IPA_PM_ACTIVATED_PENDING_DEACTIVATION ||
-			client->state ==
-				IPA_PM_ACTIVATED_PENDING_RESCHEDULE) {
-			run_algorithm = true;
-			client->state = IPA_PM_DEACTIVATED;
-			IPA_PM_DBG_STATE(client->hdl, client->name,
-				client->state);
-			spin_unlock_irqrestore(&client->state_lock, flags);
-			if (!client->skip_clk_vote) {
-				IPA_ACTIVE_CLIENTS_DEC_SPECIAL(client->name);
-				if (client->group == IPA_PM_GROUP_APPS)
-					__pm_relax(&client->wlock);
-			}
-			deactivate_client(client->hdl);
-		} else /* if activated or deactivated, we do nothing */
-			spin_unlock_irqrestore(&client->state_lock, flags);
-	}
-
-	if (run_algorithm)
-		do_clk_scaling();
-
-	return 0;
-}
-
-/**
- * ipa_pm_deactivate_sync(): deactivate ipa client and devote clock. Cannot be
- * called from atomic context.
- * @hdl: index of the client in the array
- *
- * Returns: 0 on success, negative on failure
- */
-int ipa_pm_deactivate_sync(u32 hdl)
-{
-	struct ipa_pm_client *client;
-	unsigned long flags;
-
-	if (ipa_pm_ctx == NULL) {
-		IPA_PM_ERR("PM_ctx is null\n");
-		return -EINVAL;
-	}
-
-	if (hdl >= IPA_PM_MAX_CLIENTS || ipa_pm_ctx->clients[hdl] == NULL) {
-		IPA_PM_ERR("Invalid Param\n");
-		return -EINVAL;
-	}
-	client = ipa_pm_ctx->clients[hdl];
-
-	cancel_delayed_work_sync(&client->deactivate_work);
-
-	if (IPA_PM_STATE_IN_PROGRESS(client->state))
-		wait_for_completion(&client->complete);
-
-	spin_lock_irqsave(&client->state_lock, flags);
-	IPA_PM_DBG_STATE(hdl, client->name, client->state);
-
-	if (client->state == IPA_PM_DEACTIVATED) {
-		spin_unlock_irqrestore(&client->state_lock, flags);
-		return 0;
-	}
-
-	spin_unlock_irqrestore(&client->state_lock, flags);
-
-	/* else case (Deactivates all Activated cases)*/
-	if (!client->skip_clk_vote) {
-		IPA_ACTIVE_CLIENTS_DEC_SPECIAL(client->name);
-		if (client->group == IPA_PM_GROUP_APPS)
-			__pm_relax(&client->wlock);
-	}
-
-	spin_lock_irqsave(&client->state_lock, flags);
-	client->state = IPA_PM_DEACTIVATED;
-	IPA_PM_DBG_STATE(hdl, client->name, client->state);
-	spin_unlock_irqrestore(&client->state_lock, flags);
-	deactivate_client(hdl);
-	do_clk_scaling();
-
-	return 0;
-}
-
-/**
- * ipa_pm_handle_suspend(): calls the callbacks of suspended clients to wake up
- * @pipe_bitmask: the bits represent the indexes of the clients to be woken up
- *
- * Returns: 0 on success, negative on failure
- */
-int ipa_pm_handle_suspend(u32 pipe_bitmask)
-{
-	int i;
-	struct ipa_pm_client *client;
-	bool client_notified[IPA_PM_MAX_CLIENTS] = { false };
-
-	if (ipa_pm_ctx == NULL) {
-		IPA_PM_ERR("PM_ctx is null\n");
-		return -EINVAL;
-	}
-
-	IPA_PM_DBG_LOW("bitmask: %d",  pipe_bitmask);
-
-	if (pipe_bitmask == 0)
-		return 0;
-
-	mutex_lock(&ipa_pm_ctx->client_mutex);
-	for (i = 0; i < IPA3_MAX_NUM_PIPES; i++) {
-		if (pipe_bitmask & (1 << i)) {
-			client = ipa_pm_ctx->clients_by_pipe[i];
-			if (client && !client_notified[client->hdl]) {
-				if (client->callback) {
-					client->callback(client->callback_params
-						, IPA_PM_REQUEST_WAKEUP);
-					client_notified[client->hdl] = true;
-				} else {
-					IPA_PM_ERR("client has no callback");
-					WARN_ON(1);
-				}
-			}
-		}
-	}
-	mutex_unlock(&ipa_pm_ctx->client_mutex);
-	return 0;
-}
-
-/**
- * ipa_pm_set_throughput(): Adds/changes the throughput requirement to IPA PM
- * to be used for clock scaling
- * @hdl: index of the client in the array
- * @throughput: the new throughput value to be set for that client
- *
- * Returns: 0 on success, negative on failure
- */
-int ipa_pm_set_throughput(u32 hdl, int throughput)
-{
-	struct ipa_pm_client *client;
-	unsigned long flags;
-
-	if (ipa_pm_ctx == NULL) {
-		IPA_PM_ERR("PM_ctx is null\n");
-		return -EINVAL;
-	}
-
-	if (hdl >= IPA_PM_MAX_CLIENTS || ipa_pm_ctx->clients[hdl] == NULL
-		|| throughput < 0) {
-		IPA_PM_ERR("Invalid Params\n");
-		return -EINVAL;
-	}
-	client = ipa_pm_ctx->clients[hdl];
-
-	mutex_lock(&ipa_pm_ctx->client_mutex);
-	if (client->group == IPA_PM_GROUP_DEFAULT)
-		IPA_PM_DBG_LOW("Old throughput: %d\n",  client->throughput);
-	else
-		IPA_PM_DBG_LOW("old Group %d throughput: %d\n",
-			client->group, ipa_pm_ctx->group_tput[client->group]);
-
-	if (client->group == IPA_PM_GROUP_DEFAULT)
-		client->throughput = throughput;
-	else
-		ipa_pm_ctx->group_tput[client->group] = throughput;
-
-	if (client->group == IPA_PM_GROUP_DEFAULT)
-		IPA_PM_DBG_LOW("New throughput: %d\n",  client->throughput);
-	else
-		IPA_PM_DBG_LOW("New Group %d throughput: %d\n",
-			client->group, ipa_pm_ctx->group_tput[client->group]);
-	mutex_unlock(&ipa_pm_ctx->client_mutex);
-
-	spin_lock_irqsave(&client->state_lock, flags);
-	if (IPA_PM_STATE_ACTIVE(client->state) || (client->group !=
-			IPA_PM_GROUP_DEFAULT)) {
-		spin_unlock_irqrestore(&client->state_lock, flags);
-		do_clk_scaling();
-		return 0;
-	}
-	spin_unlock_irqrestore(&client->state_lock, flags);
-
-	return 0;
-}
-
-void ipa_pm_set_clock_index(int index)
-{
-	if (ipa_pm_ctx && index >= 0)
-		ipa_pm_ctx->clk_scaling.cur_vote = index;
-
-	IPA_PM_DBG("Setting pm clock vote to %d\n", index);
-}
-
-/**
- * ipa_pm_stat() - print PM stat
- * @buf: [in] The user buff used to print
- * @size: [in] The size of buf
- * Returns: number of bytes used on success, negative on failure
- *
- * This function is called by ipa_debugfs in order to receive
- * a picture of the clients in the PM and the throughput, threshold and cur vote
- */
-int ipa_pm_stat(char *buf, int size)
-{
-	struct ipa_pm_client *client;
-	struct clk_scaling_db *clk = &ipa_pm_ctx->clk_scaling;
-	int i, j, tput, cnt = 0, result = 0;
-	unsigned long flags;
-
-	if (!buf || size < 0)
-		return -EINVAL;
-
-	mutex_lock(&ipa_pm_ctx->client_mutex);
-
-	result = scnprintf(buf + cnt, size - cnt, "\n\nCurrent threshold: [");
-	cnt += result;
-
-	for (i = 0; i < clk->threshold_size; i++) {
-		result = scnprintf(buf + cnt, size - cnt,
-			"%d, ", clk->current_threshold[i]);
-		cnt += result;
-	}
-
-	result = scnprintf(buf + cnt, size - cnt, "\b\b]\n");
-	cnt += result;
-
-	result = scnprintf(buf + cnt, size - cnt,
-		"Aggregated tput: %d, Cur vote: %d",
-		ipa_pm_ctx->aggregated_tput, clk->cur_vote);
-	cnt += result;
-
-	result = scnprintf(buf + cnt, size - cnt, "\n\nRegistered Clients:\n");
-	cnt += result;
-
-
-	for (i = 1; i < IPA_PM_MAX_CLIENTS; i++) {
-		client = ipa_pm_ctx->clients[i];
-
-		if (client == NULL)
-			continue;
-
-		spin_lock_irqsave(&client->state_lock, flags);
-		if (client->group == IPA_PM_GROUP_DEFAULT)
-			tput = client->throughput;
-		else
-			tput = ipa_pm_ctx->group_tput[client->group];
-
-		result = scnprintf(buf + cnt, size - cnt,
-		"Client[%d]: %s State:%s\nGroup: %s Throughput: %d Pipes: ",
-			i, client->name, client_state_to_str[client->state],
-			ipa_pm_group_to_str[client->group], tput);
-		cnt += result;
-
-		for (j = 0; j < IPA3_MAX_NUM_PIPES; j++) {
-			if (ipa_pm_ctx->clients_by_pipe[j] == client) {
-				result = scnprintf(buf + cnt, size - cnt,
-					"%d, ", j);
-				cnt += result;
-			}
-		}
-
-		result = scnprintf(buf + cnt, size - cnt, "\b\b\n\n");
-		cnt += result;
-		spin_unlock_irqrestore(&client->state_lock, flags);
-	}
-	mutex_unlock(&ipa_pm_ctx->client_mutex);
-
-	return cnt;
-}
-
-/**
- * ipa_pm_exceptions_stat() - print PM exceptions stat
- * @buf: [in] The user buff used to print
- * @size: [in] The size of buf
- * Returns: number of bytes used on success, negative on failure
- *
- * This function is called by ipa_debugfs in order to receive
- * a full picture of the exceptions in the PM
- */
-int ipa_pm_exceptions_stat(char *buf, int size)
-{
-	int i, j, cnt = 0, result = 0;
-	struct ipa_pm_exception_list *exception;
-
-	if (!buf || size < 0)
-		return -EINVAL;
-
-	result = scnprintf(buf + cnt, size - cnt, "\n");
-	cnt += result;
-
-	mutex_lock(&ipa_pm_ctx->client_mutex);
-	for (i = 0; i < ipa_pm_ctx->clk_scaling.exception_size; i++) {
-		exception = &ipa_pm_ctx->clk_scaling.exception_list[i];
-		if (exception == NULL) {
-			result = scnprintf(buf + cnt, size - cnt,
-			"Exception %d is NULL\n\n", i);
-			cnt += result;
-			continue;
-		}
-
-		result = scnprintf(buf + cnt, size - cnt,
-			"Exception %d: %s\nPending: %d Bitmask: %d Threshold: ["
-			, i, exception->clients, exception->pending,
-			exception->bitmask);
-		cnt += result;
-		for (j = 0; j < ipa_pm_ctx->clk_scaling.threshold_size; j++) {
-			result = scnprintf(buf + cnt, size - cnt,
-				"%d, ", exception->threshold[j]);
-			cnt += result;
-		}
-		result = scnprintf(buf + cnt, size - cnt, "\b\b]\n\n");
-		cnt += result;
-	}
-	mutex_unlock(&ipa_pm_ctx->client_mutex);
-
-	return cnt;
-}

ipa/ipa_v3/ipa_pm.h

@@ -1,181 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2017-2019, The Linux Foundation. All rights reserved.
- */
-
-#ifndef _IPA_PM_H_
-#define _IPA_PM_H_
-
-#include <linux/msm_ipa.h>
-
-/* internal to ipa */
-#define IPA_PM_MAX_CLIENTS 32 /* actual max is value -1 since we start from 1*/
-#define IPA_PM_MAX_EX_CL 64
-#define IPA_PM_THRESHOLD_MAX 5
-#define IPA_PM_EXCEPTION_MAX 5
-#define IPA_PM_DEFERRED_TIMEOUT 10
-
-/*
- * ipa_pm group names
- *
- * Default stands for individual clients while other groups share one throughput
- * Some groups also have special flags like modem which do not vote for clock
- * but is accounted for in clock scaling while activated
- */
-enum ipa_pm_group {
-	IPA_PM_GROUP_DEFAULT,
-	IPA_PM_GROUP_APPS,
-	IPA_PM_GROUP_MODEM,
-	IPA_PM_GROUP_MAX,
-};
-
-/*
- * ipa_pm_cb_event
- *
- * specifies what kind of callback is being called.
- * IPA_PM_CLIENT_ACTIVATED: the client has completed asynchronous activation
- * IPA_PM_REQUEST_WAKEUP: wake up the client after it has been suspended
- */
-enum ipa_pm_cb_event {
-	IPA_PM_CLIENT_ACTIVATED,
-	IPA_PM_REQUEST_WAKEUP,
-	IPA_PM_CB_EVENT_MAX,
-};
-
-/*
- * struct ipa_pm_exception - clients included in exception and its threshold
- * @usecase: comma separated client names
- * @threshold: the threshold values for the exception
- */
-struct ipa_pm_exception {
-	const char *usecase;
-	int threshold[IPA_PM_THRESHOLD_MAX];
-};
-
-/*
- * struct ipa_pm_init_params - parameters needed for initializng the pm
- * @default_threshold: the thresholds used if no exception passes
- * @threshold_size: size of the threshold
- * @exceptions: list of exceptions  for the pm
- * @exception_size: size of the exception_list
- */
-struct ipa_pm_init_params {
-	int default_threshold[IPA_PM_THRESHOLD_MAX];
-	int threshold_size;
-	struct ipa_pm_exception exceptions[IPA_PM_EXCEPTION_MAX];
-	int exception_size;
-};
-
-/*
- * struct ipa_pm_register_params - parameters needed to register a client
- * @name: name of the client
- * @callback: pointer to the client's callback function
- * @user_data: pointer to the client's callback parameters
- * @group: group number of the client
- * @skip_clk_vote: 0 if client votes for clock when activated, 1 if no vote
- */
-struct ipa_pm_register_params {
-	const char *name;
-	void (*callback)(void *user_data, enum ipa_pm_cb_event);
-	void *user_data;
-	enum ipa_pm_group group;
-	bool skip_clk_vote;
-};
-
-#ifdef CONFIG_IPA3
-
-int ipa_pm_register(struct ipa_pm_register_params *params, u32 *hdl);
-int ipa_pm_associate_ipa_cons_to_client(u32 hdl, enum ipa_client_type consumer);
-int ipa_pm_activate(u32 hdl);
-int ipa_pm_activate_sync(u32 hdl);
-int ipa_pm_deferred_deactivate(u32 hdl);
-int ipa_pm_deactivate_sync(u32 hdl);
-int ipa_pm_set_throughput(u32 hdl, int throughput);
-int ipa_pm_deregister(u32 hdl);
-
-/* IPA Internal Functions */
-int ipa_pm_init(struct ipa_pm_init_params *params);
-int ipa_pm_destroy(void);
-int ipa_pm_handle_suspend(u32 pipe_bitmask);
-int ipa_pm_deactivate_all_deferred(void);
-int ipa_pm_stat(char *buf, int size);
-int ipa_pm_exceptions_stat(char *buf, int size);
-void ipa_pm_set_clock_index(int index);
-
-#else
-
-static inline int ipa_pm_register(
-	struct ipa_pm_register_params *params, u32 *hdl)
-{
-	return -EPERM;
-}
-
-static inline int ipa_pm_associate_ipa_cons_to_client(
-	u32 hdl, enum ipa_client_type consumer)
-{
-	return -EPERM;
-}
-
-static inline int ipa_pm_activate(u32 hdl)
-{
-	return -EPERM;
-}
-
-static inline int ipa_pm_activate_sync(u32 hdl)
-{
-	return -EPERM;
-}
-
-static inline int ipa_pm_deferred_deactivate(u32 hdl)
-{
-	return -EPERM;
-}
-
-static inline int ipa_pm_deactivate_sync(u32 hdl)
-{
-	return -EPERM;
-}
-
-static inline int ipa_pm_set_throughput(u32 hdl, int throughput)
-{
-	return -EPERM;
-}
-
-static inline int ipa_pm_deregister(u32 hdl)
-{
-	return -EPERM;
-}
-
-/* IPA Internal Functions */
-static inline int ipa_pm_init(struct ipa_pm_init_params *params)
-{
-	return -EPERM;
-}
-
-static inline int ipa_pm_destroy(void)
-{
-	return -EPERM;
-}
-
-static inline int ipa_pm_handle_suspend(u32 pipe_bitmask)
-{
-	return -EPERM;
-}
-
-static inline int ipa_pm_deactivate_all_deferred(void)
-{
-	return -EPERM;
-}
-
-static inline int ipa_pm_stat(char *buf, int size)
-{
-	return -EPERM;
-}
-
-static inline int ipa_pm_exceptions_stat(char *buf, int size)
-{
-	return -EPERM;
-}
-#endif
-
-#endif /* _IPA_PM_H_ */

ipa/ipa_v3/ipa_qmi_service.c

@@ -1,2248 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2013-2019, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/module.h>
-#include <linux/slab.h>
-#include <linux/errno.h>
-#include <linux/delay.h>
-#include <linux/debugfs.h>
-#include <linux/delay.h>
-#include <linux/uaccess.h>
-#include <soc/qcom/subsystem_restart.h>
-#include <linux/ipa.h>
-#include <linux/vmalloc.h>
-
-#include "ipa_qmi_service.h"
-#include "ipa_mhi_proxy.h"
-
-#define IPA_Q6_SVC_VERS 1
-#define IPA_A5_SVC_VERS 1
-#define Q6_QMI_COMPLETION_TIMEOUT (60*HZ)
-
-#define IPA_A5_SERVICE_SVC_ID 0x31
-#define IPA_A5_SERVICE_INS_ID 1
-#define IPA_Q6_SERVICE_SVC_ID 0x31
-#define IPA_Q6_SERVICE_INS_ID 2
-
-#define QMI_SEND_STATS_REQ_TIMEOUT_MS 5000
-#define QMI_SEND_REQ_TIMEOUT_MS 60000
-#define QMI_MHI_SEND_REQ_TIMEOUT_MS 1000
-
-#define QMI_IPA_FORCE_CLEAR_DATAPATH_TIMEOUT_MS 1000
-
-static struct qmi_handle *ipa3_svc_handle;
-static struct workqueue_struct *ipa_clnt_req_workqueue;
-static bool ipa3_qmi_modem_init_fin, ipa3_qmi_indication_fin;
-static struct work_struct ipa3_qmi_service_init_work;
-static uint32_t ipa_wan_platform;
-struct ipa3_qmi_context *ipa3_qmi_ctx;
-static bool workqueues_stopped;
-static bool ipa3_modem_init_cmplt;
-static bool first_time_handshake;
-static bool send_qmi_init_q6;
-struct mutex ipa3_qmi_lock;
-struct ipa_msg_desc {
-	uint16_t msg_id;
-	int max_msg_len;
-	struct qmi_elem_info *ei_array;
-};
-
-static struct ipa_mhi_prime_aggr_info_req_msg_v01 aggr_req = {
-	.aggr_info_valid = 1,
-	.aggr_info_len = 5,
-	.aggr_info[0] = {
-		.ic_type = DATA_IC_TYPE_MHI_PRIME_V01,
-		.ep_type = DATA_EP_DESC_TYPE_DPL_PROD_V01,
-		.bytes_count = 16,
-	},
-	.aggr_info[1] = {
-		.ic_type = DATA_IC_TYPE_MHI_PRIME_V01,
-		.ep_type = DATA_EP_DESC_TYPE_TETH_CONS_V01,
-		.bytes_count = 24,
-		.aggr_type = DATA_AGGR_TYPE_QMAPv5_V01,
-	},
-	.aggr_info[2] = {
-		.ic_type = DATA_IC_TYPE_MHI_PRIME_V01,
-		.ep_type = DATA_EP_DESC_TYPE_TETH_PROD_V01,
-		.bytes_count = 16,
-		.aggr_type = DATA_AGGR_TYPE_QMAPv5_V01,
-	},
-	.aggr_info[3] = {
-		.ic_type = DATA_IC_TYPE_MHI_PRIME_V01,
-		.ep_type = DATA_EP_DESC_TYPE_TETH_RMNET_CONS_V01,
-		.bytes_count = 31,
-		.aggr_type = DATA_AGGR_TYPE_QMAPv5_V01,
-	},
-	.aggr_info[4] = {
-		.ic_type = DATA_IC_TYPE_MHI_PRIME_V01,
-		.ep_type = DATA_EP_DESC_TYPE_TETH_RMNET_PROD_V01,
-		.bytes_count = 31,
-		.aggr_type = DATA_AGGR_TYPE_QMAPv5_V01,
-	},
-};
-
-/* QMI A5 service */
-
-static void ipa3_handle_indication_req(struct qmi_handle *qmi_handle,
-	struct sockaddr_qrtr *sq,
-	struct qmi_txn *txn,
-	const void *decoded_msg)
-{
-	struct ipa_indication_reg_req_msg_v01 *indication_req;
-	struct ipa_indication_reg_resp_msg_v01 resp;
-	struct ipa_master_driver_init_complt_ind_msg_v01 ind;
-	int rc;
-
-	indication_req = (struct ipa_indication_reg_req_msg_v01 *)decoded_msg;
-	IPAWANDBG("Received INDICATION Request\n");
-
-	/* cache the client sq */
-	memcpy(&ipa3_qmi_ctx->client_sq, sq, sizeof(*sq));
-
-	memset(&resp, 0, sizeof(struct ipa_indication_reg_resp_msg_v01));
-	resp.resp.result = IPA_QMI_RESULT_SUCCESS_V01;
-
-	IPAWANDBG("qmi_snd_rsp: result %d, err %d\n",
-		resp.resp.result, resp.resp.error);
-	rc = qmi_send_response(qmi_handle, sq, txn,
-		QMI_IPA_INDICATION_REGISTER_RESP_V01,
-		QMI_IPA_INDICATION_REGISTER_RESP_MAX_MSG_LEN_V01,
-		ipa3_indication_reg_resp_msg_data_v01_ei,
-		&resp);
-
-	if (rc < 0) {
-		IPAWANERR("send response for Indication register failed\n");
-		return;
-	}
-
-	ipa3_qmi_indication_fin = true;
-
-	/* check if need sending indication to modem */
-	if (ipa3_qmi_modem_init_fin)	{
-		IPAWANDBG("send indication to modem (%d)\n",
-		ipa3_qmi_modem_init_fin);
-		memset(&ind, 0, sizeof(struct
-				ipa_master_driver_init_complt_ind_msg_v01));
-		ind.master_driver_init_status.result =
-			IPA_QMI_RESULT_SUCCESS_V01;
-
-		rc = qmi_send_indication(qmi_handle,
-			&(ipa3_qmi_ctx->client_sq),
-			QMI_IPA_MASTER_DRIVER_INIT_COMPLETE_IND_V01,
-			QMI_IPA_MASTER_DRIVER_INIT_COMPLETE_IND_MAX_MSG_LEN_V01,
-			ipa3_master_driver_init_complt_ind_msg_data_v01_ei,
-			&ind);
-
-		if (rc < 0) {
-			IPAWANERR("send indication failed\n");
-			ipa3_qmi_indication_fin = false;
-		}
-	} else {
-		IPAWANERR("not send indication\n");
-	}
-}
-
-static void ipa3_handle_install_filter_rule_req(struct qmi_handle *qmi_handle,
-	struct sockaddr_qrtr *sq,
-	struct qmi_txn *txn,
-	const void *decoded_msg)
-{
-	struct ipa_install_fltr_rule_req_msg_v01 *rule_req;
-	struct ipa_install_fltr_rule_resp_msg_v01 resp;
-	uint32_t rule_hdl[MAX_NUM_Q6_RULE];
-	int rc = 0, i;
-
-	rule_req = (struct ipa_install_fltr_rule_req_msg_v01 *)decoded_msg;
-	memset(rule_hdl, 0, sizeof(rule_hdl));
-	memset(&resp, 0, sizeof(struct ipa_install_fltr_rule_resp_msg_v01));
-	IPAWANDBG("Received install filter Request\n");
-
-	rc = ipa3_copy_ul_filter_rule_to_ipa((struct
-		ipa_install_fltr_rule_req_msg_v01*)decoded_msg);
-
-	if (rc) {
-		IPAWANERR("copy UL rules from modem is failed\n");
-		return;
-	}
-
-	resp.resp.result = IPA_QMI_RESULT_SUCCESS_V01;
-	if (rule_req->filter_spec_ex_list_valid == true) {
-		resp.rule_id_valid = 1;
-		if (rule_req->filter_spec_ex_list_len > MAX_NUM_Q6_RULE) {
-			resp.rule_id_len = MAX_NUM_Q6_RULE;
-			IPAWANERR("installed (%d) max Q6-UL rules ",
-			MAX_NUM_Q6_RULE);
-			IPAWANERR("but modem gives total (%u)\n",
-			rule_req->filter_spec_ex_list_len);
-		} else {
-			resp.rule_id_len =
-				rule_req->filter_spec_ex_list_len;
-		}
-	} else {
-		resp.rule_id_valid = 0;
-		resp.rule_id_len = 0;
-	}
-
-	/* construct UL filter rules response to Modem*/
-	for (i = 0; i < resp.rule_id_len; i++) {
-		resp.rule_id[i] =
-			rule_req->filter_spec_ex_list[i].rule_id;
-	}
-
-	IPAWANDBG("qmi_snd_rsp: result %d, err %d\n",
-		resp.resp.result, resp.resp.error);
-	rc = qmi_send_response(qmi_handle, sq, txn,
-		QMI_IPA_INSTALL_FILTER_RULE_RESP_V01,
-		QMI_IPA_INSTALL_FILTER_RULE_RESP_MAX_MSG_LEN_V01,
-		ipa3_install_fltr_rule_resp_msg_data_v01_ei,
-		&resp);
-
-	if (rc < 0)
-		IPAWANERR("install filter rules failed\n");
-	else
-		IPAWANDBG("Replied to install filter request\n");
-}
-
-static void ipa3_handle_filter_installed_notify_req(
-	struct qmi_handle *qmi_handle,
-	struct sockaddr_qrtr *sq,
-	struct qmi_txn *txn,
-	const void *decoded_msg)
-{
-	struct ipa_fltr_installed_notif_resp_msg_v01 resp;
-	int rc = 0;
-
-	memset(&resp, 0, sizeof(struct ipa_fltr_installed_notif_resp_msg_v01));
-	IPAWANDBG("Received filter_install_notify Request\n");
-	resp.resp.result = IPA_QMI_RESULT_SUCCESS_V01;
-
-	IPAWANDBG("qmi_snd_rsp: result %d, err %d\n",
-		resp.resp.result, resp.resp.error);
-	rc = qmi_send_response(qmi_handle, sq, txn,
-		QMI_IPA_FILTER_INSTALLED_NOTIF_RESP_V01,
-		QMI_IPA_FILTER_INSTALLED_NOTIF_RESP_MAX_MSG_LEN_V01,
-		ipa3_fltr_installed_notif_resp_msg_data_v01_ei,
-		&resp);
-
-	if (rc < 0)
-		IPAWANERR("handle filter rules failed\n");
-	else
-		IPAWANDBG("Responsed filter_install_notify Request\n");
-}
-
-static void handle_ipa_config_req(struct qmi_handle *qmi_handle,
-	struct sockaddr_qrtr *sq,
-	struct qmi_txn *txn,
-	const void *decoded_msg)
-{
-	struct ipa_config_resp_msg_v01 resp;
-	int rc;
-
-	memset(&resp, 0, sizeof(struct ipa_config_resp_msg_v01));
-	resp.resp.result = IPA_QMI_RESULT_SUCCESS_V01;
-	IPAWANDBG("Received IPA CONFIG Request\n");
-	rc = ipa_mhi_handle_ipa_config_req(
-		(struct ipa_config_req_msg_v01 *)decoded_msg);
-	if (rc) {
-		IPAERR("ipa3_mhi_handle_ipa_config_req failed %d\n", rc);
-		resp.resp.result = IPA_QMI_RESULT_FAILURE_V01;
-	}
-
-	IPAWANDBG("qmi_snd_rsp: result %d, err %d\n",
-		resp.resp.result, resp.resp.error);
-	rc = qmi_send_response(qmi_handle, sq, txn,
-		QMI_IPA_CONFIG_RESP_V01,
-		QMI_IPA_CONFIG_RESP_MAX_MSG_LEN_V01,
-		ipa3_config_resp_msg_data_v01_ei,
-		&resp);
-
-	if (rc < 0)
-		IPAWANERR("QMI_IPA_CONFIG_RESP_V01 failed\n");
-	else
-		IPAWANDBG("Responsed QMI_IPA_CONFIG_RESP_V01\n");
-}
-
-static void ipa3_handle_modem_init_cmplt_req(struct qmi_handle *qmi_handle,
-	struct sockaddr_qrtr *sq,
-	struct qmi_txn *txn,
-	const void *decoded_msg)
-{
-	struct ipa_init_modem_driver_cmplt_req_msg_v01 *cmplt_req;
-	struct ipa_init_modem_driver_cmplt_resp_msg_v01 resp;
-	int rc;
-
-	IPAWANDBG("Received QMI_IPA_INIT_MODEM_DRIVER_CMPLT_REQ_V01\n");
-	cmplt_req = (struct ipa_init_modem_driver_cmplt_req_msg_v01 *)
-		decoded_msg;
-
-	if (!ipa3_modem_init_cmplt) {
-		ipa3_modem_init_cmplt = true;
-	}
-
-	memset(&resp, 0, sizeof(resp));
-	resp.resp.result = IPA_QMI_RESULT_SUCCESS_V01;
-
-	IPAWANDBG("qmi_snd_rsp: result %d, err %d\n",
-		resp.resp.result, resp.resp.error);
-	rc = qmi_send_response(qmi_handle, sq, txn,
-		QMI_IPA_INIT_MODEM_DRIVER_CMPLT_RESP_V01,
-		QMI_IPA_INIT_MODEM_DRIVER_CMPLT_RESP_MAX_MSG_LEN_V01,
-		ipa3_init_modem_driver_cmplt_resp_msg_data_v01_ei,
-		&resp);
-
-
-	if (rc < 0)
-		IPAWANERR("QMI_IPA_INIT_MODEM_DRIVER_CMPLT_RESP_V01 failed\n");
-	else
-		IPAWANDBG("Sent QMI_IPA_INIT_MODEM_DRIVER_CMPLT_RESP_V01\n");
-}
-
-static void ipa3_handle_mhi_alloc_channel_req(struct qmi_handle *qmi_handle,
-	struct sockaddr_qrtr *sq,
-	struct qmi_txn *txn,
-	const void *decoded_msg)
-{
-	struct ipa_mhi_alloc_channel_req_msg_v01 *ch_alloc_req;
-	struct ipa_mhi_alloc_channel_resp_msg_v01 *resp = NULL;
-	int rc;
-
-	IPAWANDBG("Received QMI_IPA_MHI_ALLOC_CHANNEL_REQ_V01\n");
-	ch_alloc_req = (struct ipa_mhi_alloc_channel_req_msg_v01 *)decoded_msg;
-
-	resp = imp_handle_allocate_channel_req(ch_alloc_req);
-	if (!resp) {
-		IPAWANERR("imp handle allocate channel req fails\n");
-		return;
-	}
-
-	IPAWANDBG("qmi_snd_rsp: result %d, err %d, arr_vald: %d, arr_len %d\n",
-		resp->resp.result, resp->resp.error, resp->alloc_resp_arr_valid,
-		resp->alloc_resp_arr_len);
-	rc = qmi_send_response(qmi_handle, sq, txn,
-		QMI_IPA_MHI_ALLOC_CHANNEL_RESP_V01,
-		IPA_MHI_ALLOC_CHANNEL_RESP_MSG_V01_MAX_MSG_LEN,
-		ipa_mhi_alloc_channel_resp_msg_v01_ei,
-		resp);
-
-	if (rc < 0)
-		IPAWANERR("QMI_IPA_MHI_ALLOC_CHANNEL_RESP_V01 failed\n");
-	else
-		IPAWANDBG("Sent QMI_IPA_MHI_ALLOC_CHANNEL_RESP_V01\n");
-}
-
-static void ipa3_handle_mhi_vote_req(struct qmi_handle *qmi_handle,
-	struct sockaddr_qrtr *sq,
-	struct qmi_txn *txn,
-	const void *decoded_msg)
-{
-	struct ipa_mhi_clk_vote_req_msg_v01 *vote_req;
-	struct ipa_mhi_clk_vote_resp_msg_v01 *resp = NULL, resp2;
-	int rc;
-	uint32_t bw_mbps = 0;
-
-	vote_req = (struct ipa_mhi_clk_vote_req_msg_v01 *)decoded_msg;
-	IPAWANDBG("Received QMI_IPA_MHI_CLK_VOTE_REQ_V01(%d)\n",
-		vote_req->mhi_vote);
-
-	memset(&resp2, 0, sizeof(struct ipa_mhi_clk_vote_resp_msg_v01));
-
-	/* for mpm used for ipa clk voting */
-	if (ipa3_is_apq()) {
-		IPAWANDBG("Throughput(%d:%d) clk-rate(%d:%d)\n",
-			vote_req->tput_value_valid,
-			vote_req->tput_value,
-			vote_req->clk_rate_valid,
-			vote_req->clk_rate);
-		if (vote_req->clk_rate_valid) {
-			switch (vote_req->clk_rate) {
-			case QMI_IPA_CLOCK_RATE_LOW_SVS_V01:
-				bw_mbps = 0;
-				break;
-			case QMI_IPA_CLOCK_RATE_SVS_V01:
-				bw_mbps = 350;
-				break;
-			case QMI_IPA_CLOCK_RATE_NOMINAL_V01:
-				bw_mbps = 690;
-				break;
-			case QMI_IPA_CLOCK_RATE_TURBO_V01:
-				bw_mbps = 1200;
-				break;
-			default:
-				IPAWANERR("Note supported clk_rate (%d)\n",
-				vote_req->clk_rate);
-				bw_mbps = 0;
-				resp2.resp.result = IPA_QMI_RESULT_FAILURE_V01;
-				resp2.resp.error =
-					IPA_QMI_ERR_NOT_SUPPORTED_V01;
-				break;
-			}
-			if (ipa3_vote_for_bus_bw(&bw_mbps)) {
-				IPAWANERR("Failed to vote BW (%u)\n", bw_mbps);
-				resp2.resp.result = IPA_QMI_RESULT_FAILURE_V01;
-				resp2.resp.error =
-					IPA_QMI_ERR_NOT_SUPPORTED_V01;
-			}
-			resp = &resp2;
-		}
-	} else {
-		resp = imp_handle_vote_req(vote_req->mhi_vote);
-		if (!resp) {
-			IPAWANERR("imp handle allocate channel req fails");
-			return;
-		}
-		IPAWANDBG("start sending QMI_IPA_MHI_CLK_VOTE_RESP_V01\n");
-	}
-
-	IPAWANDBG("qmi_snd_rsp: result %d, err %d\n",
-		resp->resp.result, resp->resp.error);
-	rc = qmi_send_response(qmi_handle, sq, txn,
-		QMI_IPA_MHI_CLK_VOTE_RESP_V01,
-		IPA_MHI_CLK_VOTE_RESP_MSG_V01_MAX_MSG_LEN,
-		ipa_mhi_clk_vote_resp_msg_v01_ei,
-		resp);
-
-	if (rc < 0)
-		IPAWANERR("QMI_IPA_MHI_CLK_VOTE_RESP_V01 failed\n");
-	else
-		IPAWANDBG("Finished senting QMI_IPA_MHI_CLK_VOTE_RESP_V01\n");
-}
-
-static void ipa3_a5_svc_disconnect_cb(struct qmi_handle *qmi,
-	unsigned int node, unsigned int port)
-{
-	IPAWANDBG_LOW("Received QMI client disconnect\n");
-}
-
-/****************************************************/
-/*                 QMI A5 client ->Q6               */
-/****************************************************/
-static void ipa3_q6_clnt_svc_arrive(struct work_struct *work);
-static DECLARE_DELAYED_WORK(ipa3_work_svc_arrive, ipa3_q6_clnt_svc_arrive);
-static void ipa3_q6_clnt_svc_exit(struct work_struct *work);
-static DECLARE_DELAYED_WORK(ipa3_work_svc_exit, ipa3_q6_clnt_svc_exit);
-/* Test client port for IPC Router */
-static struct qmi_handle *ipa_q6_clnt;
-
-static int ipa3_check_qmi_response(int rc,
-				  int req_id,
-				  enum ipa_qmi_result_type_v01 result,
-				  enum ipa_qmi_error_type_v01 error,
-				  char *resp_type)
-{
-	if (rc < 0) {
-		if (rc == -ETIMEDOUT && ipa3_rmnet_ctx.ipa_rmnet_ssr) {
-			IPAWANERR(
-			"Timeout for qmi request id %d\n", req_id);
-			return rc;
-		}
-		if ((rc == -ENETRESET) || (rc == -ENODEV)) {
-			IPAWANERR(
-			"SSR while waiting for qmi request id %d\n", req_id);
-			return rc;
-		}
-		IPAWANERR("Error sending qmi request id %d, rc = %d\n",
-			req_id, rc);
-		return rc;
-	}
-	if (result != IPA_QMI_RESULT_SUCCESS_V01 &&
-	    ipa3_rmnet_ctx.ipa_rmnet_ssr) {
-		IPAWANERR(
-		"Got bad response %d from request id %d (error %d)\n",
-		req_id, result, error);
-		return result;
-	}
-	IPAWANDBG_LOW("Received %s successfully\n", resp_type);
-	return 0;
-}
-
-static int ipa3_qmi_send_req_wait(struct qmi_handle *client_handle,
-	struct ipa_msg_desc *req_desc, void *req,
-	struct ipa_msg_desc *resp_desc, void *resp,
-	unsigned long timeout_ms)
-{
-	struct qmi_txn txn;
-	int ret;
-
-	ret = qmi_txn_init(client_handle, &txn, resp_desc->ei_array, resp);
-
-	if (ret < 0) {
-		IPAWANERR("QMI txn init failed, ret= %d\n", ret);
-		return ret;
-	}
-
-	ret = qmi_send_request(client_handle,
-		&ipa3_qmi_ctx->server_sq,
-		&txn,
-		req_desc->msg_id,
-		req_desc->max_msg_len,
-		req_desc->ei_array,
-		req);
-
-	if (ret < 0) {
-		qmi_txn_cancel(&txn);
-		return ret;
-	}
-	ret = qmi_txn_wait(&txn, msecs_to_jiffies(timeout_ms));
-
-	return ret;
-}
-
-static int ipa3_qmi_init_modem_send_sync_msg(void)
-{
-	struct ipa_init_modem_driver_req_msg_v01 req;
-	struct ipa_init_modem_driver_resp_msg_v01 resp;
-	struct ipa_msg_desc req_desc, resp_desc;
-	int rc;
-	u16 smem_restr_bytes = ipa3_get_smem_restr_bytes();
-	int wan_cons_ep;
-
-	memset(&req, 0, sizeof(struct ipa_init_modem_driver_req_msg_v01));
-	memset(&resp, 0, sizeof(struct ipa_init_modem_driver_resp_msg_v01));
-
-	req.platform_type_valid = true;
-	req.platform_type = ipa_wan_platform;
-
-	req.hdr_tbl_info_valid = (IPA_MEM_PART(modem_hdr_size) != 0);
-	req.hdr_tbl_info.modem_offset_start =
-		IPA_MEM_PART(modem_hdr_ofst) + smem_restr_bytes;
-	req.hdr_tbl_info.modem_offset_end = IPA_MEM_PART(modem_hdr_ofst) +
-		smem_restr_bytes + IPA_MEM_PART(modem_hdr_size) - 1;
-
-	req.v4_route_tbl_info_valid = true;
-	req.v4_route_tbl_info.route_tbl_start_addr =
-		IPA_MEM_PART(v4_rt_nhash_ofst) + smem_restr_bytes;
-	req.v4_route_tbl_info.num_indices =
-		IPA_MEM_PART(v4_modem_rt_index_hi);
-	req.v6_route_tbl_info_valid = true;
-
-	req.v6_route_tbl_info.route_tbl_start_addr =
-		IPA_MEM_PART(v6_rt_nhash_ofst) + smem_restr_bytes;
-	req.v6_route_tbl_info.num_indices =
-		IPA_MEM_PART(v6_modem_rt_index_hi);
-
-	req.v4_filter_tbl_start_addr_valid = true;
-	req.v4_filter_tbl_start_addr =
-		IPA_MEM_PART(v4_flt_nhash_ofst) + smem_restr_bytes;
-
-	req.v6_filter_tbl_start_addr_valid = true;
-	req.v6_filter_tbl_start_addr =
-		IPA_MEM_PART(v6_flt_nhash_ofst) + smem_restr_bytes;
-
-	req.modem_mem_info_valid = (IPA_MEM_PART(modem_size) != 0);
-	req.modem_mem_info.block_start_addr =
-		IPA_MEM_PART(modem_ofst) + smem_restr_bytes;
-	req.modem_mem_info.size = IPA_MEM_PART(modem_size);
-
-	wan_cons_ep = ipa_get_ep_mapping(IPA_CLIENT_APPS_WAN_CONS);
-	if (wan_cons_ep == IPA_EP_NOT_ALLOCATED) {
-		IPAWANDBG("APPS_WAN_CONS is not valid\n");
-		req.ctrl_comm_dest_end_pt_valid = false;
-		req.ctrl_comm_dest_end_pt = 0;
-	} else {
-		req.ctrl_comm_dest_end_pt_valid = true;
-		req.ctrl_comm_dest_end_pt =
-			ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_CONS);
-	}
-
-	req.hdr_proc_ctx_tbl_info_valid =
-		(IPA_MEM_PART(modem_hdr_proc_ctx_size) != 0);
-	req.hdr_proc_ctx_tbl_info.modem_offset_start =
-		IPA_MEM_PART(modem_hdr_proc_ctx_ofst) + smem_restr_bytes;
-	req.hdr_proc_ctx_tbl_info.modem_offset_end =
-		IPA_MEM_PART(modem_hdr_proc_ctx_ofst) +
-		IPA_MEM_PART(modem_hdr_proc_ctx_size) + smem_restr_bytes - 1;
-
-	req.zip_tbl_info_valid = (IPA_MEM_PART(modem_comp_decomp_size) != 0);
-	req.zip_tbl_info.modem_offset_start =
-		IPA_MEM_PART(modem_comp_decomp_size) + smem_restr_bytes;
-	req.zip_tbl_info.modem_offset_end =
-		IPA_MEM_PART(modem_comp_decomp_ofst) +
-		IPA_MEM_PART(modem_comp_decomp_size) + smem_restr_bytes - 1;
-
-	/* if hashing not supported, Modem filter/routing hash
-	 * tables should not fill with valid data.
-	 */
-	if (!ipa3_ctx->ipa_fltrt_not_hashable) {
-		req.v4_hash_route_tbl_info_valid = true;
-		req.v4_hash_route_tbl_info.route_tbl_start_addr =
-			IPA_MEM_PART(v4_rt_hash_ofst) + smem_restr_bytes;
-		req.v4_hash_route_tbl_info.num_indices =
-			IPA_MEM_PART(v4_modem_rt_index_hi);
-
-		req.v6_hash_route_tbl_info_valid = true;
-		req.v6_hash_route_tbl_info.route_tbl_start_addr =
-			IPA_MEM_PART(v6_rt_hash_ofst) + smem_restr_bytes;
-		req.v6_hash_route_tbl_info.num_indices =
-			IPA_MEM_PART(v6_modem_rt_index_hi);
-
-		req.v4_hash_filter_tbl_start_addr_valid = true;
-		req.v4_hash_filter_tbl_start_addr =
-			IPA_MEM_PART(v4_flt_hash_ofst) + smem_restr_bytes;
-
-		req.v6_hash_filter_tbl_start_addr_valid = true;
-		req.v6_hash_filter_tbl_start_addr =
-			IPA_MEM_PART(v6_flt_hash_ofst) + smem_restr_bytes;
-	}
-	req.hw_stats_quota_base_addr_valid = true;
-	req.hw_stats_quota_base_addr =
-		IPA_MEM_PART(stats_quota_ofst) + smem_restr_bytes;
-
-	req.hw_stats_quota_size_valid = true;
-	req.hw_stats_quota_size = IPA_MEM_PART(stats_quota_size);
-
-	req.hw_drop_stats_base_addr_valid = true;
-	req.hw_drop_stats_base_addr =
-		IPA_MEM_PART(stats_drop_ofst) + smem_restr_bytes;
-
-	req.hw_drop_stats_table_size_valid = true;
-	req.hw_drop_stats_table_size = IPA_MEM_PART(stats_drop_size);
-
-	if (!ipa3_uc_loaded_check()) {  /* First time boot */
-		req.is_ssr_bootup_valid = false;
-		req.is_ssr_bootup = 0;
-	} else {  /* After SSR boot */
-		req.is_ssr_bootup_valid = true;
-		req.is_ssr_bootup = 1;
-	}
-
-	IPAWANDBG("platform_type %d\n", req.platform_type);
-	IPAWANDBG("hdr_tbl_info.modem_offset_start %d\n",
-			req.hdr_tbl_info.modem_offset_start);
-	IPAWANDBG("hdr_tbl_info.modem_offset_end %d\n",
-			req.hdr_tbl_info.modem_offset_end);
-	IPAWANDBG("v4_route_tbl_info.route_tbl_start_addr %d\n",
-			req.v4_route_tbl_info.route_tbl_start_addr);
-	IPAWANDBG("v4_route_tbl_info.num_indices %d\n",
-			req.v4_route_tbl_info.num_indices);
-	IPAWANDBG("v6_route_tbl_info.route_tbl_start_addr %d\n",
-			req.v6_route_tbl_info.route_tbl_start_addr);
-	IPAWANDBG("v6_route_tbl_info.num_indices %d\n",
-			req.v6_route_tbl_info.num_indices);
-	IPAWANDBG("v4_filter_tbl_start_addr %d\n",
-			req.v4_filter_tbl_start_addr);
-	IPAWANDBG("v6_filter_tbl_start_addr %d\n",
-			req.v6_filter_tbl_start_addr);
-	IPAWANDBG("modem_mem_info.block_start_addr %d\n",
-			req.modem_mem_info.block_start_addr);
-	IPAWANDBG("modem_mem_info.size %d\n",
-			req.modem_mem_info.size);
-	IPAWANDBG("ctrl_comm_dest_end_pt %d\n",
-			req.ctrl_comm_dest_end_pt);
-	IPAWANDBG("is_ssr_bootup %d\n",
-			req.is_ssr_bootup);
-	IPAWANDBG("v4_hash_route_tbl_info.route_tbl_start_addr %d\n",
-		req.v4_hash_route_tbl_info.route_tbl_start_addr);
-	IPAWANDBG("v4_hash_route_tbl_info.num_indices %d\n",
-		req.v4_hash_route_tbl_info.num_indices);
-	IPAWANDBG("v6_hash_route_tbl_info.route_tbl_start_addr %d\n",
-		req.v6_hash_route_tbl_info.route_tbl_start_addr);
-	IPAWANDBG("v6_hash_route_tbl_info.num_indices %d\n",
-		req.v6_hash_route_tbl_info.num_indices);
-	IPAWANDBG("v4_hash_filter_tbl_start_addr %d\n",
-		req.v4_hash_filter_tbl_start_addr);
-	IPAWANDBG("v6_hash_filter_tbl_start_addr %d\n",
-		req.v6_hash_filter_tbl_start_addr);
-
-	req_desc.max_msg_len = QMI_IPA_INIT_MODEM_DRIVER_REQ_MAX_MSG_LEN_V01;
-	req_desc.msg_id = QMI_IPA_INIT_MODEM_DRIVER_REQ_V01;
-	req_desc.ei_array = ipa3_init_modem_driver_req_msg_data_v01_ei;
-
-	resp_desc.max_msg_len = QMI_IPA_INIT_MODEM_DRIVER_RESP_MAX_MSG_LEN_V01;
-	resp_desc.msg_id = QMI_IPA_INIT_MODEM_DRIVER_RESP_V01;
-	resp_desc.ei_array = ipa3_init_modem_driver_resp_msg_data_v01_ei;
-
-	pr_info("Sending QMI_IPA_INIT_MODEM_DRIVER_REQ_V01\n");
-	if (unlikely(!ipa_q6_clnt))
-		return -ETIMEDOUT;
-	rc = ipa3_qmi_send_req_wait(ipa_q6_clnt,
-		&req_desc, &req,
-		&resp_desc, &resp,
-		QMI_SEND_REQ_TIMEOUT_MS);
-
-	if (rc < 0) {
-		IPAWANERR("QMI send Req %d failed, rc= %d\n",
-			QMI_IPA_INIT_MODEM_DRIVER_REQ_V01,
-			rc);
-		return rc;
-	}
-
-	pr_info("QMI_IPA_INIT_MODEM_DRIVER_REQ_V01 response received\n");
-	return ipa3_check_qmi_response(rc,
-		QMI_IPA_INIT_MODEM_DRIVER_REQ_V01, resp.resp.result,
-		resp.resp.error, "ipa_init_modem_driver_resp_msg_v01");
-}
-
-/* sending filter-install-request to modem*/
-int ipa3_qmi_filter_request_send(struct ipa_install_fltr_rule_req_msg_v01 *req)
-{
-	struct ipa_install_fltr_rule_resp_msg_v01 resp;
-	struct ipa_msg_desc req_desc, resp_desc;
-	int rc;
-	int i;
-
-	/* check if modem up */
-	if (!ipa3_qmi_indication_fin ||
-		!ipa3_qmi_modem_init_fin ||
-		!ipa_q6_clnt) {
-		IPAWANDBG("modem QMI haven't up yet\n");
-		return -EINVAL;
-	}
-
-	/* check if the filter rules from IPACM is valid */
-	if (req->filter_spec_list_len == 0) {
-		IPAWANDBG("IPACM pass zero rules to Q6\n");
-	} else {
-		IPAWANDBG("IPACM pass %u rules to Q6\n",
-		req->filter_spec_list_len);
-	}
-
-	if (req->filter_spec_list_len >= QMI_IPA_MAX_FILTERS_V01) {
-		IPAWANDBG(
-		"IPACM passes the number of filtering rules exceed limit\n");
-		return -EINVAL;
-	} else if (req->source_pipe_index_valid != 0) {
-		IPAWANDBG(
-		"IPACM passes source_pipe_index_valid not zero 0 != %d\n",
-			req->source_pipe_index_valid);
-		return -EINVAL;
-	} else if (req->source_pipe_index >= ipa3_ctx->ipa_num_pipes) {
-		IPAWANDBG(
-		"IPACM passes source pipe index not valid ID = %d\n",
-		req->source_pipe_index);
-		return -EINVAL;
-	}
-	for (i = 0; i < req->filter_spec_list_len; i++) {
-		if ((req->filter_spec_list[i].ip_type !=
-			QMI_IPA_IP_TYPE_V4_V01) &&
-			(req->filter_spec_list[i].ip_type !=
-			QMI_IPA_IP_TYPE_V6_V01))
-			return -EINVAL;
-		if (req->filter_spec_list[i].is_mux_id_valid == false)
-			return -EINVAL;
-		if (req->filter_spec_list[i].is_routing_table_index_valid
-			== false)
-			return -EINVAL;
-		if ((req->filter_spec_list[i].filter_action <=
-			QMI_IPA_FILTER_ACTION_INVALID_V01) ||
-			(req->filter_spec_list[i].filter_action >
-			QMI_IPA_FILTER_ACTION_EXCEPTION_V01))
-			return -EINVAL;
-	}
-
-	mutex_lock(&ipa3_qmi_lock);
-	if (ipa3_qmi_ctx != NULL) {
-		/* cache the qmi_filter_request */
-		memcpy(&(ipa3_qmi_ctx->ipa_install_fltr_rule_req_msg_cache[
-			ipa3_qmi_ctx->num_ipa_install_fltr_rule_req_msg]),
-			req,
-			sizeof(struct ipa_install_fltr_rule_req_msg_v01));
-		ipa3_qmi_ctx->num_ipa_install_fltr_rule_req_msg++;
-		ipa3_qmi_ctx->num_ipa_install_fltr_rule_req_msg %= 10;
-	}
-	mutex_unlock(&ipa3_qmi_lock);
-
-	req_desc.max_msg_len = QMI_IPA_INSTALL_FILTER_RULE_REQ_MAX_MSG_LEN_V01;
-	req_desc.msg_id = QMI_IPA_INSTALL_FILTER_RULE_REQ_V01;
-	req_desc.ei_array = ipa3_install_fltr_rule_req_msg_data_v01_ei;
-
-	memset(&resp, 0, sizeof(struct ipa_install_fltr_rule_resp_msg_v01));
-	resp_desc.max_msg_len =
-		QMI_IPA_INSTALL_FILTER_RULE_RESP_MAX_MSG_LEN_V01;
-	resp_desc.msg_id = QMI_IPA_INSTALL_FILTER_RULE_RESP_V01;
-	resp_desc.ei_array = ipa3_install_fltr_rule_resp_msg_data_v01_ei;
-
-	if (unlikely(!ipa_q6_clnt))
-		return -ETIMEDOUT;
-	rc = ipa3_qmi_send_req_wait(ipa_q6_clnt,
-		&req_desc, req,
-		&resp_desc, &resp,
-		QMI_SEND_REQ_TIMEOUT_MS);
-
-	if (rc < 0) {
-		IPAWANERR("QMI send Req %d failed, rc= %d\n",
-			QMI_IPA_INSTALL_FILTER_RULE_REQ_V01,
-			rc);
-		return rc;
-	}
-
-	return ipa3_check_qmi_response(rc,
-		QMI_IPA_INSTALL_FILTER_RULE_REQ_V01, resp.resp.result,
-		resp.resp.error, "ipa_install_filter");
-}
-
-/* sending filter-install-request to modem*/
-int ipa3_qmi_filter_request_ex_send(
-	struct ipa_install_fltr_rule_req_ex_msg_v01 *req)
-{
-	struct ipa_install_fltr_rule_resp_ex_msg_v01 resp;
-	struct ipa_msg_desc req_desc, resp_desc;
-	int rc;
-	int i;
-
-	/* check if modem up */
-	if (!ipa3_qmi_indication_fin ||
-		!ipa3_qmi_modem_init_fin ||
-		!ipa_q6_clnt) {
-		IPAWANDBG("modem QMI haven't up yet\n");
-		return -EINVAL;
-	}
-
-	/* check if the filter rules from IPACM is valid */
-	if (req->filter_spec_ex_list_len == 0) {
-		IPAWANDBG("IPACM pass zero rules to Q6\n");
-	} else {
-		IPAWANDBG("IPACM pass %u rules to Q6\n",
-		req->filter_spec_ex_list_len);
-	}
-
-	if (req->filter_spec_ex_list_len >= QMI_IPA_MAX_FILTERS_EX_V01) {
-		IPAWANDBG(
-		"IPACM pass the number of filtering rules exceed limit\n");
-		return -EINVAL;
-	} else if (req->source_pipe_index_valid != 0) {
-		IPAWANDBG(
-		"IPACM passes source_pipe_index_valid not zero 0 != %d\n",
-			req->source_pipe_index_valid);
-		return -EINVAL;
-	}
-
-	for (i = 0; i < req->filter_spec_ex_list_len; i++) {
-		if ((req->filter_spec_ex_list[i].ip_type !=
-			QMI_IPA_IP_TYPE_V4_V01) &&
-			(req->filter_spec_ex_list[i].ip_type !=
-			QMI_IPA_IP_TYPE_V6_V01))
-			return -EINVAL;
-		if (req->filter_spec_ex_list[i].is_mux_id_valid == false)
-			return -EINVAL;
-		if (req->filter_spec_ex_list[i].is_routing_table_index_valid
-			== false)
-			return -EINVAL;
-		if ((req->filter_spec_ex_list[i].filter_action <=
-			QMI_IPA_FILTER_ACTION_INVALID_V01) ||
-			(req->filter_spec_ex_list[i].filter_action >
-			QMI_IPA_FILTER_ACTION_EXCEPTION_V01))
-			return -EINVAL;
-	}
-	mutex_lock(&ipa3_qmi_lock);
-	if (ipa3_qmi_ctx != NULL) {
-		/* cache the qmi_filter_request */
-		memcpy(&(ipa3_qmi_ctx->ipa_install_fltr_rule_req_ex_msg_cache[
-			ipa3_qmi_ctx->num_ipa_install_fltr_rule_req_ex_msg]),
-			req,
-			sizeof(struct ipa_install_fltr_rule_req_ex_msg_v01));
-		ipa3_qmi_ctx->num_ipa_install_fltr_rule_req_ex_msg++;
-		ipa3_qmi_ctx->num_ipa_install_fltr_rule_req_ex_msg %= 10;
-	}
-	mutex_unlock(&ipa3_qmi_lock);
-
-	req_desc.max_msg_len =
-		QMI_IPA_INSTALL_FILTER_RULE_EX_REQ_MAX_MSG_LEN_V01;
-	req_desc.msg_id = QMI_IPA_INSTALL_FILTER_RULE_EX_REQ_V01;
-	req_desc.ei_array = ipa3_install_fltr_rule_req_ex_msg_data_v01_ei;
-
-	memset(&resp, 0, sizeof(struct ipa_install_fltr_rule_resp_ex_msg_v01));
-	resp_desc.max_msg_len =
-		QMI_IPA_INSTALL_FILTER_RULE_EX_RESP_MAX_MSG_LEN_V01;
-	resp_desc.msg_id = QMI_IPA_INSTALL_FILTER_RULE_EX_RESP_V01;
-	resp_desc.ei_array = ipa3_install_fltr_rule_resp_ex_msg_data_v01_ei;
-
-	rc = ipa3_qmi_send_req_wait(ipa_q6_clnt,
-		&req_desc, req,
-		&resp_desc, &resp,
-		QMI_SEND_REQ_TIMEOUT_MS);
-
-	if (rc < 0) {
-		IPAWANERR("QMI send Req %d failed, rc= %d\n",
-			QMI_IPA_INSTALL_FILTER_RULE_EX_REQ_V01,
-			rc);
-		return rc;
-	}
-
-	return ipa3_check_qmi_response(rc,
-		QMI_IPA_INSTALL_FILTER_RULE_EX_REQ_V01, resp.resp.result,
-		resp.resp.error, "ipa_install_filter");
-}
-
-/* sending add offload-connection-request to modem*/
-int ipa3_qmi_add_offload_request_send(
-	struct ipa_add_offload_connection_req_msg_v01 *req)
-{
-	struct ipa_add_offload_connection_resp_msg_v01 resp;
-	struct ipa_msg_desc req_desc, resp_desc;
-	int rc = 0;
-	int i, j;
-	uint32_t id;
-
-	/* check if modem up */
-	if (!ipa3_qmi_modem_init_fin ||
-		!ipa_q6_clnt) {
-		IPAWANDBG("modem QMI haven't up yet\n");
-		return -EINVAL;
-	}
-
-	/* check if the filter rules from IPACM is valid */
-	if (req->filter_spec_ex2_list_len == 0) {
-		IPAWANDBG("IPACM pass zero rules to Q6\n");
-	} else {
-		IPAWANDBG("IPACM pass %u rules to Q6\n",
-		req->filter_spec_ex2_list_len);
-	}
-
-	/* currently set total max to 64 */
-	if (req->filter_spec_ex2_list_len +
-		ipa3_qmi_ctx->num_ipa_offload_connection
-		>= QMI_IPA_MAX_FILTERS_V01) {
-		IPAWANDBG(
-		"cur(%d), req(%d), exceed limit (%d)\n",
-			ipa3_qmi_ctx->num_ipa_offload_connection,
-			req->filter_spec_ex2_list_len,
-			QMI_IPA_MAX_FILTERS_V01);
-		return -EINVAL;
-	}
-
-	for (i = 0; i < req->filter_spec_ex2_list_len; i++) {
-		if ((req->filter_spec_ex2_list[i].ip_type !=
-			QMI_IPA_IP_TYPE_V4_V01) &&
-			(req->filter_spec_ex2_list[i].ip_type !=
-			QMI_IPA_IP_TYPE_V6_V01))
-			return -EINVAL;
-		if (req->filter_spec_ex2_list[i].is_mux_id_valid == false)
-			return -EINVAL;
-		if ((req->filter_spec_ex2_list[i].filter_action <=
-			QMI_IPA_FILTER_ACTION_INVALID_V01) ||
-			(req->filter_spec_ex2_list[i].filter_action >
-			QMI_IPA_FILTER_ACTION_EXCEPTION_V01))
-			return -EINVAL;
-	}
-
-	req_desc.max_msg_len =
-		IPA_ADD_OFFLOAD_CONNECTION_REQ_MSG_V01_MAX_MSG_LEN;
-	req_desc.msg_id = QMI_IPA_ADD_OFFLOAD_CONNECTION_REQ_V01;
-	req_desc.ei_array = ipa_add_offload_connection_req_msg_v01_ei;
-
-	memset(&resp, 0, sizeof(struct
-		ipa_add_offload_connection_resp_msg_v01));
-	resp_desc.max_msg_len =
-		IPA_ADD_OFFLOAD_CONNECTION_RESP_MSG_V01_MAX_MSG_LEN;
-	resp_desc.msg_id = QMI_IPA_ADD_OFFLOAD_CONNECTION_RESP_V01;
-	resp_desc.ei_array = ipa_add_offload_connection_resp_msg_v01_ei;
-
-	rc = ipa3_qmi_send_req_wait(ipa_q6_clnt,
-		&req_desc, req,
-		&resp_desc, &resp,
-		QMI_SEND_REQ_TIMEOUT_MS);
-
-	if (rc < 0) {
-		IPAWANERR("QMI send Req %d failed, rc= %d\n",
-			QMI_IPA_ADD_OFFLOAD_CONNECTION_REQ_V01,
-			rc);
-		return rc;
-	}
-
-	rc = ipa3_check_qmi_response(rc,
-		QMI_IPA_ADD_OFFLOAD_CONNECTION_REQ_V01, resp.resp.result,
-		resp.resp.error, "ipa_add_offload_connection");
-
-	if (rc) {
-		IPAWANERR("QMI get Response %d failed, rc= %d\n",
-			QMI_IPA_ADD_OFFLOAD_CONNECTION_REQ_V01,
-			rc);
-		return rc;
-	}
-
-	/* Check & copy rule-handle */
-	if (!resp.filter_handle_list_valid) {
-		IPAWANERR("QMI resp invalid %d failed\n",
-			resp.filter_handle_list_valid);
-		return -ERANGE;
-	}
-
-	if (resp.filter_handle_list_len !=
-		req->filter_spec_ex2_list_len) {
-		IPAWANERR("QMI resp invalid size %d req %d\n",
-			resp.filter_handle_list_len,
-			req->filter_spec_ex2_list_len);
-		return -ERANGE;
-	}
-
-	mutex_lock(&ipa3_qmi_lock);
-	for (i = 0; i < req->filter_spec_ex2_list_len; i++) {
-		id = resp.filter_handle_list[i].filter_spec_identifier;
-		/* check rule-id matched or not */
-		if (req->filter_spec_ex2_list[i].rule_id !=
-			id) {
-			IPAWANERR("QMI error (%d)st-(%d) rule-id (%d)\n",
-				i,
-				id,
-				req->filter_spec_ex2_list[i].rule_id);
-			mutex_unlock(&ipa3_qmi_lock);
-			return -EINVAL;
-		}
-		/* find free spot*/
-		for (j = 0; j < QMI_IPA_MAX_FILTERS_V01; j++) {
-			if (!ipa3_qmi_ctx->ipa_offload_cache[j].valid)
-				break;
-		}
-
-		if (j == QMI_IPA_MAX_FILTERS_V01) {
-			IPAWANERR("can't find free spot for rule-id %d\n",
-				id);
-			mutex_unlock(&ipa3_qmi_lock);
-			return -EINVAL;
-		}
-
-		/* save rule-id handle to cache */
-		ipa3_qmi_ctx->ipa_offload_cache[j].rule_id =
-			resp.filter_handle_list[i].filter_spec_identifier;
-		ipa3_qmi_ctx->ipa_offload_cache[j].rule_hdl =
-			resp.filter_handle_list[i].filter_handle;
-		ipa3_qmi_ctx->ipa_offload_cache[j].valid = true;
-		ipa3_qmi_ctx->ipa_offload_cache[j].ip_type =
-			req->filter_spec_ex2_list[i].ip_type;
-		ipa3_qmi_ctx->num_ipa_offload_connection++;
-	}
-	mutex_unlock(&ipa3_qmi_lock);
-	IPAWANDBG("Update cached conntrack entries (%d)\n",
-		ipa3_qmi_ctx->num_ipa_offload_connection);
-	return rc;
-}
-
-/* sending rmv offload-connection-request to modem*/
-int ipa3_qmi_rmv_offload_request_send(
-	struct ipa_remove_offload_connection_req_msg_v01 *req)
-{
-	struct ipa_remove_offload_connection_resp_msg_v01 resp;
-	struct ipa_msg_desc req_desc, resp_desc;
-	int rc = 0;
-	int i, j;
-	uint32_t id;
-
-	/* check if modem up */
-	if (!ipa3_qmi_modem_init_fin ||
-		!ipa_q6_clnt) {
-		IPAWANDBG("modem QMI haven't up yet\n");
-		return -EINVAL;
-	}
-
-	/* check if the # of handles from IPACM is valid */
-	if (req->filter_handle_list_len == 0) {
-		IPAWANDBG("IPACM deleted zero rules !\n");
-		return -EINVAL;
-	}
-
-	IPAWANDBG("IPACM pass (%d) rules handles to Q6, cur (%d)\n",
-	req->filter_handle_list_len,
-	ipa3_qmi_ctx->num_ipa_offload_connection);
-
-	/*  max as num_ipa_offload_connection */
-	if (req->filter_handle_list_len >=
-		ipa3_qmi_ctx->num_ipa_offload_connection) {
-		IPAWANDBG(
-		"cur(%d), req_rmv(%d)\n",
-			ipa3_qmi_ctx->num_ipa_offload_connection,
-			req->filter_handle_list_len);
-		return -EINVAL;
-	}
-
-	mutex_lock(&ipa3_qmi_lock);
-	for (i = 0; i < req->filter_handle_list_len; i++) {
-		/* check if rule-id match */
-		id =
-			req->filter_handle_list[i].filter_spec_identifier;
-		for (j = 0; j < QMI_IPA_MAX_FILTERS_V01; j++) {
-			if ((ipa3_qmi_ctx->ipa_offload_cache[j].valid) &&
-				(ipa3_qmi_ctx->ipa_offload_cache[j].rule_id ==
-				id))
-				break;
-		}
-		if (j == QMI_IPA_MAX_FILTERS_V01) {
-			IPAWANERR("can't find rule-id %d\n",
-				id);
-			mutex_unlock(&ipa3_qmi_lock);
-			return -EINVAL;
-		}
-
-		/* fill up the filter_handle */
-		req->filter_handle_list[i].filter_handle =
-			ipa3_qmi_ctx->ipa_offload_cache[j].rule_hdl;
-		ipa3_qmi_ctx->ipa_offload_cache[j].valid = false;
-		ipa3_qmi_ctx->num_ipa_offload_connection--;
-	}
-	mutex_unlock(&ipa3_qmi_lock);
-
-	req_desc.max_msg_len =
-		IPA_REMOVE_OFFLOAD_CONNECTION_REQ_MSG_V01_MAX_MSG_LEN;
-	req_desc.msg_id = QMI_IPA_REMOVE_OFFLOAD_CONNECTION_REQ_V01;
-	req_desc.ei_array = ipa_remove_offload_connection_req_msg_v01_ei;
-
-	memset(&resp, 0, sizeof(struct
-		ipa_remove_offload_connection_resp_msg_v01));
-	resp_desc.max_msg_len =
-		IPA_REMOVE_OFFLOAD_CONNECTION_RESP_MSG_V01_MAX_MSG_LEN;
-	resp_desc.msg_id = QMI_IPA_REMOVE_OFFLOAD_CONNECTION_RESP_V01;
-	resp_desc.ei_array = ipa_remove_offload_connection_resp_msg_v01_ei;
-
-	rc = ipa3_qmi_send_req_wait(ipa_q6_clnt,
-		&req_desc, req,
-		&resp_desc, &resp,
-		QMI_SEND_REQ_TIMEOUT_MS);
-
-	if (rc < 0) {
-		IPAWANERR("QMI send Req %d failed, rc= %d\n",
-			QMI_IPA_REMOVE_OFFLOAD_CONNECTION_REQ_V01,
-			rc);
-		return rc;
-	}
-	IPAWANDBG("left cached conntrack entries (%d)\n",
-		ipa3_qmi_ctx->num_ipa_offload_connection);
-
-	return ipa3_check_qmi_response(rc,
-		QMI_IPA_REMOVE_OFFLOAD_CONNECTION_REQ_V01, resp.resp.result,
-		resp.resp.error, "ipa_rmv_offload_connection");
-}
-
-/* sending ul-filter-install-request to modem*/
-int ipa3_qmi_ul_filter_request_send(
-	struct ipa_configure_ul_firewall_rules_req_msg_v01 *req)
-{
-	struct ipa_configure_ul_firewall_rules_resp_msg_v01 resp;
-	struct ipa_msg_desc req_desc, resp_desc;
-	int rc, i;
-
-	IPAWANDBG("IPACM pass %u rules to Q6\n",
-		req->firewall_rules_list_len);
-
-	mutex_lock(&ipa3_qmi_lock);
-	if (ipa3_qmi_ctx != NULL) {
-		/* cache the qmi_filter_request */
-		memcpy(
-		&(ipa3_qmi_ctx->ipa_configure_ul_firewall_rules_req_msg_cache[
-		ipa3_qmi_ctx->num_ipa_configure_ul_firewall_rules_req_msg]),
-		req,
-		sizeof(struct
-		ipa_configure_ul_firewall_rules_req_msg_v01));
-		ipa3_qmi_ctx->num_ipa_configure_ul_firewall_rules_req_msg++;
-		ipa3_qmi_ctx->num_ipa_configure_ul_firewall_rules_req_msg %=
-			MAX_NUM_QMI_RULE_CACHE;
-	}
-	mutex_unlock(&ipa3_qmi_lock);
-
-	/* check if modem is up */
-	if (!ipa3_qmi_indication_fin ||
-		!ipa3_qmi_modem_init_fin ||
-		!ipa_q6_clnt) {
-		IPAWANDBG("modem QMI service is not up yet\n");
-		return -EINVAL;
-	}
-
-	/* Passing 0 rules means that firewall is disabled */
-	if (req->firewall_rules_list_len == 0)
-		IPAWANDBG("IPACM passed 0 rules to Q6\n");
-
-	if (req->firewall_rules_list_len >= QMI_IPA_MAX_UL_FIREWALL_RULES_V01) {
-		IPAWANERR(
-		"Number of rules passed by IPACM, %d, exceed limit %d\n",
-			req->firewall_rules_list_len,
-			QMI_IPA_MAX_UL_FIREWALL_RULES_V01);
-		return -EINVAL;
-	}
-
-	/* Check for valid IP type */
-	for (i = 0; i < req->firewall_rules_list_len; i++) {
-		if (req->firewall_rules_list[i].ip_type !=
-				QMI_IPA_IP_TYPE_V4_V01 &&
-			req->firewall_rules_list[i].ip_type !=
-				QMI_IPA_IP_TYPE_V6_V01) {
-			IPAWANERR("Invalid IP type %d\n",
-					req->firewall_rules_list[i].ip_type);
-			return -EINVAL;
-		}
-	}
-
-	req_desc.max_msg_len =
-		QMI_IPA_INSTALL_UL_FIREWALL_RULES_REQ_MAX_MSG_LEN_V01;
-	req_desc.msg_id = QMI_IPA_INSTALL_UL_FIREWALL_RULES_REQ_V01;
-	req_desc.ei_array =
-		ipa3_configure_ul_firewall_rules_req_msg_data_v01_ei;
-
-	memset(&resp, 0,
-		sizeof(struct ipa_configure_ul_firewall_rules_resp_msg_v01));
-	resp_desc.max_msg_len =
-		QMI_IPA_INSTALL_UL_FIREWALL_RULES_RESP_MAX_MSG_LEN_V01;
-	resp_desc.msg_id = QMI_IPA_INSTALL_UL_FIREWALL_RULES_RESP_V01;
-	resp_desc.ei_array =
-		ipa3_configure_ul_firewall_rules_resp_msg_data_v01_ei;
-	rc = ipa3_qmi_send_req_wait(ipa_q6_clnt,
-		&req_desc, req,
-		&resp_desc, &resp,
-		QMI_SEND_REQ_TIMEOUT_MS);
-	if (rc < 0) {
-		IPAWANERR("send Req %d failed, rc= %d\n",
-			QMI_IPA_INSTALL_UL_FIREWALL_RULES_REQ_V01,
-			rc);
-		return rc;
-	}
-
-	return ipa3_check_qmi_response(rc,
-		QMI_IPA_INSTALL_UL_FIREWALL_RULES_REQ_V01,
-		resp.resp.result,
-		resp.resp.error, "ipa_received_ul_firewall_filter");
-}
-
-int ipa3_qmi_enable_force_clear_datapath_send(
-	struct ipa_enable_force_clear_datapath_req_msg_v01 *req)
-{
-	struct ipa_enable_force_clear_datapath_resp_msg_v01 resp;
-	struct ipa_msg_desc req_desc, resp_desc;
-	int rc = 0;
-
-	if (!req || !req->source_pipe_bitmask) {
-		IPAWANERR("invalid params\n");
-		return -EINVAL;
-	}
-
-	if (ipa3_ctx->ipa3_hw_mode == IPA_HW_MODE_VIRTUAL ||
-		ipa3_ctx->ipa3_hw_mode == IPA_HW_MODE_EMULATION) {
-		IPAWANDBG("Simulating success on emu/virt mode\n");
-		return 0;
-	}
-
-	req_desc.max_msg_len =
-	QMI_IPA_ENABLE_FORCE_CLEAR_DATAPATH_REQ_MAX_MSG_LEN_V01;
-	req_desc.msg_id = QMI_IPA_ENABLE_FORCE_CLEAR_DATAPATH_REQ_V01;
-	req_desc.ei_array =
-		ipa3_enable_force_clear_datapath_req_msg_data_v01_ei;
-
-	memset(&resp, 0, sizeof(struct ipa_fltr_installed_notif_resp_msg_v01));
-	resp_desc.max_msg_len =
-		QMI_IPA_ENABLE_FORCE_CLEAR_DATAPATH_RESP_MAX_MSG_LEN_V01;
-	resp_desc.msg_id = QMI_IPA_ENABLE_FORCE_CLEAR_DATAPATH_RESP_V01;
-	resp_desc.ei_array =
-		ipa3_enable_force_clear_datapath_resp_msg_data_v01_ei;
-
-	if (unlikely(!ipa_q6_clnt))
-		return -ETIMEDOUT;
-	rc = ipa3_qmi_send_req_wait(ipa_q6_clnt,
-		&req_desc, req,
-		&resp_desc, &resp,
-		QMI_IPA_FORCE_CLEAR_DATAPATH_TIMEOUT_MS);
-
-	if (rc < 0) {
-		IPAWANERR("send Req %d failed, rc= %d\n",
-			QMI_IPA_ENABLE_FORCE_CLEAR_DATAPATH_REQ_V01,
-			rc);
-		return rc;
-	}
-
-	if (resp.resp.result != IPA_QMI_RESULT_SUCCESS_V01) {
-		IPAWANERR("filter_notify failed %d\n",
-			resp.resp.result);
-		return resp.resp.result;
-	}
-
-	return ipa3_check_qmi_response(rc,
-		QMI_IPA_ENABLE_FORCE_CLEAR_DATAPATH_REQ_V01,
-		resp.resp.result,
-		resp.resp.error, "ipa_enable_force_clear_datapath");
-}
-
-int ipa3_qmi_disable_force_clear_datapath_send(
-	struct ipa_disable_force_clear_datapath_req_msg_v01 *req)
-{
-	struct ipa_disable_force_clear_datapath_resp_msg_v01 resp;
-	struct ipa_msg_desc req_desc, resp_desc;
-	int rc = 0;
-
-
-	if (!req) {
-		IPAWANERR("invalid params\n");
-		return -EINVAL;
-	}
-
-	if (ipa3_ctx->ipa3_hw_mode == IPA_HW_MODE_VIRTUAL ||
-		ipa3_ctx->ipa3_hw_mode == IPA_HW_MODE_EMULATION) {
-		IPAWANDBG("Simulating success on emu/virt mode\n");
-		return 0;
-	}
-
-	req_desc.max_msg_len =
-		QMI_IPA_DISABLE_FORCE_CLEAR_DATAPATH_REQ_MAX_MSG_LEN_V01;
-	req_desc.msg_id = QMI_IPA_DISABLE_FORCE_CLEAR_DATAPATH_REQ_V01;
-	req_desc.ei_array =
-		ipa3_disable_force_clear_datapath_req_msg_data_v01_ei;
-
-	memset(&resp, 0, sizeof(struct ipa_fltr_installed_notif_resp_msg_v01));
-	resp_desc.max_msg_len =
-		QMI_IPA_DISABLE_FORCE_CLEAR_DATAPATH_RESP_MAX_MSG_LEN_V01;
-	resp_desc.msg_id = QMI_IPA_DISABLE_FORCE_CLEAR_DATAPATH_RESP_V01;
-	resp_desc.ei_array =
-		ipa3_disable_force_clear_datapath_resp_msg_data_v01_ei;
-	if (unlikely(!ipa_q6_clnt))
-		return -ETIMEDOUT;
-	rc = ipa3_qmi_send_req_wait(ipa_q6_clnt,
-		&req_desc, req,
-		&resp_desc, &resp,
-		QMI_IPA_FORCE_CLEAR_DATAPATH_TIMEOUT_MS);
-
-	if (rc < 0) {
-		IPAWANERR("send Req %d failed, rc= %d\n",
-			QMI_IPA_DISABLE_FORCE_CLEAR_DATAPATH_REQ_V01,
-			rc);
-		return rc;
-	}
-
-	if (resp.resp.result != IPA_QMI_RESULT_SUCCESS_V01) {
-		IPAWANERR("filter_notify failed %d\n",
-			resp.resp.result);
-		return resp.resp.result;
-	}
-
-	return ipa3_check_qmi_response(rc,
-		QMI_IPA_DISABLE_FORCE_CLEAR_DATAPATH_REQ_V01,
-		resp.resp.result,
-		resp.resp.error, "ipa_disable_force_clear_datapath");
-}
-
-/* sending filter-installed-notify-request to modem*/
-int ipa3_qmi_filter_notify_send(
-		struct ipa_fltr_installed_notif_req_msg_v01 *req)
-{
-	struct ipa_fltr_installed_notif_resp_msg_v01 resp;
-	struct ipa_msg_desc req_desc, resp_desc;
-	int rc = 0;
-
-	/* check if the filter rules from IPACM is valid */
-	if (req->rule_id_len == 0) {
-		IPAWANDBG(" delete UL filter rule for pipe %d\n",
-		req->source_pipe_index);
-	} else if (req->rule_id_len > QMI_IPA_MAX_FILTERS_V01) {
-		IPAWANERR(" UL filter rule for pipe %d exceed max (%u)\n",
-		req->source_pipe_index,
-		req->rule_id_len);
-		return -EINVAL;
-	}
-
-	if (req->rule_id_ex_len == 0) {
-		IPAWANDBG(" delete UL filter rule for pipe %d\n",
-		req->source_pipe_index);
-	} else if (req->rule_id_ex_len > QMI_IPA_MAX_FILTERS_EX2_V01) {
-		IPAWANERR(" UL filter rule for pipe %d exceed max (%u)\n",
-		req->source_pipe_index,
-		req->rule_id_ex_len);
-		return -EINVAL;
-	}
-
-	if (req->install_status != IPA_QMI_RESULT_SUCCESS_V01) {
-		IPAWANERR(" UL filter rule for pipe %d install_status = %d\n",
-			req->source_pipe_index, req->install_status);
-		return -EINVAL;
-	} else if ((req->rule_id_valid != 1) &&
-		(req->rule_id_ex_valid != 1)) {
-		IPAWANERR(" UL filter rule for pipe %d rule_id_valid = %d/%d\n",
-			req->source_pipe_index, req->rule_id_valid,
-			req->rule_id_ex_valid);
-		return -EINVAL;
-	} else if (req->source_pipe_index >= ipa3_ctx->ipa_num_pipes) {
-		IPAWANDBG(
-		"IPACM passes source pipe index not valid ID = %d\n",
-		req->source_pipe_index);
-		return -EINVAL;
-	} else if (((req->embedded_pipe_index_valid != true) ||
-			(req->embedded_call_mux_id_valid != true)) &&
-			((req->embedded_pipe_index_valid != false) ||
-			(req->embedded_call_mux_id_valid != false))) {
-		IPAWANERR(
-			"IPACM passes embedded pipe and mux valid not valid\n");
-		return -EINVAL;
-	} else if (req->embedded_pipe_index >= ipa3_ctx->ipa_num_pipes) {
-		IPAWANERR("IPACM passes source pipe index not valid ID = %d\n",
-		req->source_pipe_index);
-		return -EINVAL;
-	}
-
-	if (req->source_pipe_index == -1) {
-		IPAWANERR("Source pipe index invalid\n");
-		return -EINVAL;
-	}
-
-	mutex_lock(&ipa3_qmi_lock);
-	if (ipa3_qmi_ctx != NULL) {
-		/* cache the qmi_filter_request */
-		memcpy(&(ipa3_qmi_ctx->ipa_fltr_installed_notif_req_msg_cache[
-			ipa3_qmi_ctx->num_ipa_fltr_installed_notif_req_msg]),
-			req,
-			sizeof(struct ipa_fltr_installed_notif_req_msg_v01));
-		ipa3_qmi_ctx->num_ipa_fltr_installed_notif_req_msg++;
-		ipa3_qmi_ctx->num_ipa_fltr_installed_notif_req_msg %= 10;
-	}
-	mutex_unlock(&ipa3_qmi_lock);
-
-	req_desc.max_msg_len =
-	QMI_IPA_FILTER_INSTALLED_NOTIF_REQ_MAX_MSG_LEN_V01;
-	req_desc.msg_id = QMI_IPA_FILTER_INSTALLED_NOTIF_REQ_V01;
-	req_desc.ei_array = ipa3_fltr_installed_notif_req_msg_data_v01_ei;
-
-	memset(&resp, 0, sizeof(struct ipa_fltr_installed_notif_resp_msg_v01));
-	resp_desc.max_msg_len =
-		QMI_IPA_FILTER_INSTALLED_NOTIF_RESP_MAX_MSG_LEN_V01;
-	resp_desc.msg_id = QMI_IPA_FILTER_INSTALLED_NOTIF_RESP_V01;
-	resp_desc.ei_array = ipa3_fltr_installed_notif_resp_msg_data_v01_ei;
-
-	if (unlikely(!ipa_q6_clnt))
-		return -ETIMEDOUT;
-	rc = ipa3_qmi_send_req_wait(ipa_q6_clnt,
-		&req_desc, req,
-		&resp_desc, &resp,
-		QMI_SEND_REQ_TIMEOUT_MS);
-
-	if (rc < 0) {
-		IPAWANERR("send Req %d failed, rc= %d\n",
-			QMI_IPA_FILTER_INSTALLED_NOTIF_REQ_V01,
-			rc);
-		return rc;
-	}
-
-	return ipa3_check_qmi_response(rc,
-		QMI_IPA_FILTER_INSTALLED_NOTIF_REQ_V01, resp.resp.result,
-		resp.resp.error, "ipa_fltr_installed_notif_resp");
-}
-
-static void ipa3_q6_clnt_quota_reached_ind_cb(struct qmi_handle *handle,
-	struct sockaddr_qrtr *sq,
-	struct qmi_txn *txn,
-	const void *data)
-{
-	struct ipa_data_usage_quota_reached_ind_msg_v01 *qmi_ind;
-
-	if (handle != ipa_q6_clnt) {
-		IPAWANERR("Wrong client\n");
-		return;
-	}
-
-	qmi_ind = (struct ipa_data_usage_quota_reached_ind_msg_v01 *) data;
-
-	IPAWANDBG("Quota reached indication on qmux(%d) Mbytes(%lu)\n",
-		qmi_ind->apn.mux_id, (unsigned long) qmi_ind->apn.num_Mbytes);
-	ipa3_broadcast_quota_reach_ind(qmi_ind->apn.mux_id,
-		IPA_UPSTEAM_MODEM);
-}
-
-static void ipa3_q6_clnt_install_firewall_rules_ind_cb(
-	struct qmi_handle *handle,
-	struct sockaddr_qrtr *sq,
-	struct qmi_txn *txn,
-	const void *data)
-{
-	struct ipa_configure_ul_firewall_rules_ind_msg_v01 qmi_ul_firewall_ind;
-
-	memset(&qmi_ul_firewall_ind, 0, sizeof(
-		struct ipa_configure_ul_firewall_rules_ind_msg_v01));
-	memcpy(&qmi_ul_firewall_ind, data, sizeof(
-		struct ipa_configure_ul_firewall_rules_ind_msg_v01));
-
-	IPAWANDBG("UL firewall rules install indication on Q6");
-	if (qmi_ul_firewall_ind.result.is_success ==
-		QMI_IPA_UL_FIREWALL_STATUS_SUCCESS_V01) {
-		IPAWANDBG(" : Success\n");
-		IPAWANDBG
-		("Mux ID : %d\n", qmi_ul_firewall_ind.result.mux_id);
-	} else if (qmi_ul_firewall_ind.result.is_success ==
-		QMI_IPA_UL_FIREWALL_STATUS_FAILURE_V01) {
-		IPAWANERR(": Failure\n");
-	} else {
-		IPAWANERR(": Unexpected Result");
-	}
-}
-
-static void ipa3_q6_clnt_svc_arrive(struct work_struct *work)
-{
-	int rc;
-	struct ipa_master_driver_init_complt_ind_msg_v01 ind;
-
-	rc = kernel_connect(ipa_q6_clnt->sock,
-		(struct sockaddr *) &ipa3_qmi_ctx->server_sq,
-		sizeof(ipa3_qmi_ctx->server_sq),
-		0);
-
-	if (rc < 0) {
-		IPAWANERR("Couldnt connect Server\n");
-		return;
-	}
-
-	if (!send_qmi_init_q6)
-		return;
-
-	IPAWANDBG("Q6 QMI service available now\n");
-	if (ipa3_is_apq()) {
-		ipa3_qmi_modem_init_fin = true;
-		IPAWANDBG("QMI-client complete, ipa3_qmi_modem_init_fin : %d\n",
-			ipa3_qmi_modem_init_fin);
-		return;
-	}
-
-	/* Initialize modem IPA-driver */
-	IPAWANDBG("send ipa3_qmi_init_modem_send_sync_msg to modem\n");
-	rc = ipa3_qmi_init_modem_send_sync_msg();
-	if ((rc == -ENETRESET) || (rc == -ENODEV)) {
-		IPAWANERR(
-		"ipa3_qmi_init_modem_send_sync_msg failed due to SSR!\n");
-		/* Cleanup when ipa3_wwan_remove is called */
-		vfree(ipa_q6_clnt);
-		ipa_q6_clnt = NULL;
-		return;
-	}
-
-	if (rc != 0) {
-		IPAWANERR("ipa3_qmi_init_modem_send_sync_msg failed\n");
-		/*
-		 * Hardware not responding.
-		 * This is a very unexpected scenario
-		 * which requires a kernel panic in
-		 * order to force dumps for QMI/Q6 side analysis.
-		 */
-		BUG();
-	}
-	ipa3_qmi_modem_init_fin = true;
-
-	/* In cold-bootup, first_time_handshake = false */
-	ipa3_q6_handshake_complete(first_time_handshake);
-	first_time_handshake = true;
-	IPAWANDBG("complete, ipa3_qmi_modem_init_fin : %d\n",
-		ipa3_qmi_modem_init_fin);
-
-	if (ipa3_qmi_indication_fin) {
-		IPAWANDBG("send indication to modem (%d)\n",
-		ipa3_qmi_indication_fin);
-		memset(&ind, 0, sizeof(struct
-			ipa_master_driver_init_complt_ind_msg_v01));
-		ind.master_driver_init_status.result =
-			IPA_QMI_RESULT_SUCCESS_V01;
-
-		rc = qmi_send_indication(ipa3_svc_handle,
-			&ipa3_qmi_ctx->client_sq,
-			QMI_IPA_MASTER_DRIVER_INIT_COMPLETE_IND_V01,
-			QMI_IPA_MASTER_DRIVER_INIT_COMPLETE_IND_MAX_MSG_LEN_V01,
-			ipa3_master_driver_init_complt_ind_msg_data_v01_ei,
-			&ind);
-
-		IPAWANDBG("ipa_qmi_service_client good\n");
-	} else {
-		IPAWANERR("not send indication (%d)\n",
-		ipa3_qmi_indication_fin);
-	}
-
-	send_qmi_init_q6 = false;
-
-}
-
-static void ipa3_q6_clnt_svc_exit(struct work_struct *work)
-{
-	if (ipa3_qmi_ctx != NULL) {
-		ipa3_qmi_ctx->server_sq.sq_family = 0;
-		ipa3_qmi_ctx->server_sq.sq_node = 0;
-		ipa3_qmi_ctx->server_sq.sq_port = 0;
-	}
-}
-
-static int ipa3_q6_clnt_svc_event_notify_svc_new(struct qmi_handle *qmi,
-	struct qmi_service *service)
-{
-	IPAWANDBG("QMI svc:%d vers:%d ins:%d node:%d port:%d\n",
-		  service->service, service->version, service->instance,
-		  service->node, service->port);
-
-	if (ipa3_qmi_ctx != NULL) {
-		ipa3_qmi_ctx->server_sq.sq_family = AF_QIPCRTR;
-		ipa3_qmi_ctx->server_sq.sq_node = service->node;
-		ipa3_qmi_ctx->server_sq.sq_port = service->port;
-	}
-	if (!workqueues_stopped) {
-		queue_delayed_work(ipa_clnt_req_workqueue,
-			&ipa3_work_svc_arrive, 0);
-	}
-	return 0;
-}
-
-static void ipa3_q6_clnt_svc_event_notify_net_reset(struct qmi_handle *qmi)
-{
-	if (!workqueues_stopped)
-		queue_delayed_work(ipa_clnt_req_workqueue,
-			&ipa3_work_svc_exit, 0);
-}
-
-static void ipa3_q6_clnt_svc_event_notify_svc_exit(struct qmi_handle *qmi,
-						   struct qmi_service *svc)
-{
-	IPAWANDBG("QMI svc:%d vers:%d ins:%d node:%d port:%d\n", svc->service,
-		  svc->version, svc->instance, svc->node, svc->port);
-
-	if (!workqueues_stopped)
-		queue_delayed_work(ipa_clnt_req_workqueue,
-			&ipa3_work_svc_exit, 0);
-}
-
-static struct qmi_ops server_ops = {
-	.del_client = ipa3_a5_svc_disconnect_cb,
-};
-
-static struct qmi_ops client_ops = {
-	.new_server = ipa3_q6_clnt_svc_event_notify_svc_new,
-	.del_server = ipa3_q6_clnt_svc_event_notify_svc_exit,
-	.net_reset = ipa3_q6_clnt_svc_event_notify_net_reset,
-};
-
-static struct qmi_msg_handler server_handlers[] = {
-	{
-		.type = QMI_REQUEST,
-		.msg_id = QMI_IPA_INDICATION_REGISTER_REQ_V01,
-		.ei = ipa3_indication_reg_req_msg_data_v01_ei,
-		.decoded_size =
-			QMI_IPA_INDICATION_REGISTER_REQ_MAX_MSG_LEN_V01,
-		.fn = ipa3_handle_indication_req,
-	},
-	{
-		.type = QMI_REQUEST,
-		.msg_id = QMI_IPA_INSTALL_FILTER_RULE_REQ_V01,
-		.ei = ipa3_install_fltr_rule_req_msg_data_v01_ei,
-		.decoded_size =
-			QMI_IPA_INSTALL_FILTER_RULE_REQ_MAX_MSG_LEN_V01,
-		.fn = ipa3_handle_install_filter_rule_req,
-	},
-	{
-		.type = QMI_REQUEST,
-		.msg_id = QMI_IPA_FILTER_INSTALLED_NOTIF_REQ_V01,
-		.ei = ipa3_fltr_installed_notif_req_msg_data_v01_ei,
-		.decoded_size =
-			QMI_IPA_FILTER_INSTALLED_NOTIF_REQ_MAX_MSG_LEN_V01,
-		.fn = ipa3_handle_filter_installed_notify_req,
-	},
-	{
-		.type = QMI_REQUEST,
-		.msg_id = QMI_IPA_CONFIG_REQ_V01,
-		.ei = ipa3_config_req_msg_data_v01_ei,
-		.decoded_size = QMI_IPA_CONFIG_REQ_MAX_MSG_LEN_V01,
-		.fn = handle_ipa_config_req,
-	},
-	{
-		.type = QMI_REQUEST,
-		.msg_id = QMI_IPA_INIT_MODEM_DRIVER_CMPLT_REQ_V01,
-		.ei = ipa3_init_modem_driver_cmplt_req_msg_data_v01_ei,
-		.decoded_size =
-			QMI_IPA_INIT_MODEM_DRIVER_CMPLT_REQ_MAX_MSG_LEN_V01,
-		.fn = ipa3_handle_modem_init_cmplt_req,
-	},
-	{
-		.type = QMI_REQUEST,
-		.msg_id = QMI_IPA_INIT_MODEM_DRIVER_CMPLT_REQ_V01,
-		.ei = ipa3_init_modem_driver_cmplt_req_msg_data_v01_ei,
-		.decoded_size =
-			QMI_IPA_INIT_MODEM_DRIVER_CMPLT_REQ_MAX_MSG_LEN_V01,
-		.fn = ipa3_handle_modem_init_cmplt_req,
-	},
-	{
-		.type = QMI_REQUEST,
-		.msg_id = QMI_IPA_MHI_ALLOC_CHANNEL_REQ_V01,
-		.ei = ipa_mhi_alloc_channel_req_msg_v01_ei,
-		.decoded_size =
-			IPA_MHI_ALLOC_CHANNEL_REQ_MSG_V01_MAX_MSG_LEN,
-		.fn = ipa3_handle_mhi_alloc_channel_req,
-	},
-	{
-		.type = QMI_REQUEST,
-		.msg_id = QMI_IPA_MHI_CLK_VOTE_REQ_V01,
-		.ei = ipa_mhi_clk_vote_req_msg_v01_ei,
-		.decoded_size =
-			IPA_MHI_CLK_VOTE_REQ_MSG_V01_MAX_MSG_LEN,
-		.fn = ipa3_handle_mhi_vote_req,
-	},
-
-};
-
-/*  clinet_handlers are client callbacks that will be called from QMI context
- *  when an indication from Q6 server arrives.
- *  In our case, client_handlers needs handling only for QMI_INDICATION,
- *  since the QMI_REQUEST/ QMI_RESPONSE are handled in a blocking fashion
- *  at the time of sending QMI_REQUESTs.
- */
-static struct qmi_msg_handler client_handlers[] = {
-	{
-		.type = QMI_INDICATION,
-		.msg_id = QMI_IPA_DATA_USAGE_QUOTA_REACHED_IND_V01,
-		.ei = ipa3_data_usage_quota_reached_ind_msg_data_v01_ei,
-		.decoded_size =
-			QMI_IPA_DATA_USAGE_QUOTA_REACHED_IND_MAX_MSG_LEN_V01,
-		.fn = ipa3_q6_clnt_quota_reached_ind_cb,
-	},
-	{
-		.type = QMI_INDICATION,
-		.msg_id = QMI_IPA_INSTALL_UL_FIREWALL_RULES_IND_V01,
-		.ei = ipa3_install_fltr_rule_req_msg_data_v01_ei,
-		.decoded_size =
-			QMI_IPA_INSTALL_UL_FIREWALL_RULES_IND_MAX_MSG_LEN_V01,
-		.fn = ipa3_q6_clnt_install_firewall_rules_ind_cb,
-	},
-};
-
-
-static void ipa3_qmi_service_init_worker(struct work_struct *work)
-{
-	int rc;
-
-	/* start the QMI msg cache */
-	ipa3_qmi_ctx = vzalloc(sizeof(*ipa3_qmi_ctx));
-	if (!ipa3_qmi_ctx) {
-		IPAWANERR("Failed to allocate ipa3_qmi_ctx\n");
-		return;
-	}
-
-	if (ipa3_is_apq()) {
-		/* Only start QMI-client */
-		IPAWANDBG("Only start IPA A7 QMI client\n");
-		goto qmi_client_start;
-	}
-
-	/* Initialize QMI-service*/
-	IPAWANDBG("IPA A7 QMI init OK :>>>>\n");
-
-	ipa3_qmi_ctx->modem_cfg_emb_pipe_flt =
-		ipa3_get_modem_cfg_emb_pipe_flt();
-
-	ipa3_qmi_ctx->num_ipa_offload_connection = 0;
-	ipa3_svc_handle = vzalloc(sizeof(*ipa3_svc_handle));
-
-	if (!ipa3_svc_handle)
-		goto destroy_ipa_A7_svc_wq;
-
-	rc = qmi_handle_init(ipa3_svc_handle,
-		QMI_IPA_MAX_MSG_LEN,
-		&server_ops,
-		server_handlers);
-
-	if (rc < 0) {
-		IPAWANERR("Initializing ipa_a5 svc failed %d\n", rc);
-		goto destroy_qmi_handle;
-	}
-
-	rc = qmi_add_server(ipa3_svc_handle,
-		IPA_A5_SERVICE_SVC_ID,
-		IPA_A5_SVC_VERS,
-		IPA_A5_SERVICE_INS_ID);
-
-	if (rc < 0) {
-		IPAWANERR("Registering ipa_a5 svc failed %d\n",
-				rc);
-		goto deregister_qmi_srv;
-	}
-
-qmi_client_start:
-	/* Initialize QMI-client */
-	ipa_clnt_req_workqueue = create_singlethread_workqueue("clnt_req");
-	if (!ipa_clnt_req_workqueue) {
-		IPAWANERR("Creating clnt_req workqueue failed\n");
-		goto deregister_qmi_srv;
-	}
-
-	/* Create a Local client port for QMI communication */
-	ipa_q6_clnt = vzalloc(sizeof(*ipa_q6_clnt));
-
-	if (!ipa_q6_clnt)
-		goto destroy_clnt_req_wq;
-
-	rc = qmi_handle_init(ipa_q6_clnt,
-		QMI_IPA_MAX_MSG_LEN,
-		&client_ops,
-		client_handlers);
-
-	if (rc < 0) {
-		IPAWANERR("Creating clnt handle failed\n");
-		goto destroy_qmi_client_handle;
-	}
-
-	rc = qmi_add_lookup(ipa_q6_clnt,
-		IPA_Q6_SERVICE_SVC_ID,
-		IPA_Q6_SVC_VERS,
-		IPA_Q6_SERVICE_INS_ID);
-
-	if (rc < 0) {
-		IPAWANERR("Adding Q6 Svc failed\n");
-		goto deregister_qmi_client;
-	}
-
-	/* get Q6 service and start send modem-initial to Q6 */
-	IPAWANDBG("wait service available\n");
-	return;
-
-deregister_qmi_client:
-	qmi_handle_release(ipa_q6_clnt);
-destroy_qmi_client_handle:
-	vfree(ipa_q6_clnt);
-	ipa_q6_clnt = NULL;
-destroy_clnt_req_wq:
-	destroy_workqueue(ipa_clnt_req_workqueue);
-	ipa_clnt_req_workqueue = NULL;
-deregister_qmi_srv:
-	if (!ipa3_is_apq())
-		qmi_handle_release(ipa3_svc_handle);
-destroy_qmi_handle:
-	vfree(ipa3_qmi_ctx);
-destroy_ipa_A7_svc_wq:
-	if (!ipa3_is_apq()) {
-		vfree(ipa3_svc_handle);
-		ipa3_svc_handle = NULL;
-	}
-	ipa3_qmi_ctx = NULL;
-}
-
-int ipa3_qmi_service_init(uint32_t wan_platform_type)
-{
-	ipa_wan_platform = wan_platform_type;
-	ipa3_qmi_modem_init_fin = false;
-	ipa3_qmi_indication_fin = false;
-	ipa3_modem_init_cmplt = false;
-	send_qmi_init_q6 = true;
-	workqueues_stopped = false;
-
-	if (!ipa3_svc_handle) {
-		INIT_WORK(&ipa3_qmi_service_init_work,
-			ipa3_qmi_service_init_worker);
-		schedule_work(&ipa3_qmi_service_init_work);
-	}
-	return 0;
-}
-
-void ipa3_qmi_service_exit(void)
-{
-
-	workqueues_stopped = true;
-
-	/* qmi-service */
-	if (ipa3_svc_handle != NULL) {
-		qmi_handle_release(ipa3_svc_handle);
-		vfree(ipa3_svc_handle);
-		ipa3_svc_handle = NULL;
-	}
-
-	/* qmi-client */
-
-	/* Release client handle */
-	if (ipa_q6_clnt != NULL) {
-		qmi_handle_release(ipa_q6_clnt);
-		vfree(ipa_q6_clnt);
-		ipa_q6_clnt = NULL;
-		if (ipa_clnt_req_workqueue) {
-			destroy_workqueue(ipa_clnt_req_workqueue);
-			ipa_clnt_req_workqueue = NULL;
-		}
-	}
-
-	/* clean the QMI msg cache */
-	mutex_lock(&ipa3_qmi_lock);
-	if (ipa3_qmi_ctx != NULL) {
-		vfree(ipa3_qmi_ctx);
-		ipa3_qmi_ctx = NULL;
-	}
-	mutex_unlock(&ipa3_qmi_lock);
-
-	ipa3_qmi_modem_init_fin = false;
-	ipa3_qmi_indication_fin = false;
-	ipa3_modem_init_cmplt = false;
-	send_qmi_init_q6 = true;
-}
-
-void ipa3_qmi_stop_workqueues(void)
-{
-	IPAWANDBG("Stopping all QMI workqueues\n");
-
-	/* Stopping all workqueues so new work won't be scheduled */
-	workqueues_stopped = true;
-
-	/* Making sure that the current scheduled work won't be executed */
-	cancel_delayed_work(&ipa3_work_svc_arrive);
-	cancel_delayed_work(&ipa3_work_svc_exit);
-}
-
-/* voting for bus BW to ipa_rm*/
-int ipa3_vote_for_bus_bw(uint32_t *bw_mbps)
-{
-	int ret;
-
-	IPAWANDBG("Bus BW is %d\n", *bw_mbps);
-
-	if (bw_mbps == NULL) {
-		IPAWANERR("Bus BW is invalid\n");
-		return -EINVAL;
-	}
-
-	ret = ipa3_wwan_set_modem_perf_profile(*bw_mbps);
-	if (ret)
-		IPAWANERR("Failed to set perf profile to BW %u\n",
-			*bw_mbps);
-	else
-		IPAWANDBG("Succeeded to set perf profile to BW %u\n",
-			*bw_mbps);
-
-	return ret;
-}
-
-int ipa3_qmi_get_data_stats(struct ipa_get_data_stats_req_msg_v01 *req,
-			   struct ipa_get_data_stats_resp_msg_v01 *resp)
-{
-	struct ipa_msg_desc req_desc, resp_desc;
-	int rc;
-
-	req_desc.max_msg_len = QMI_IPA_GET_DATA_STATS_REQ_MAX_MSG_LEN_V01;
-	req_desc.msg_id = QMI_IPA_GET_DATA_STATS_REQ_V01;
-	req_desc.ei_array = ipa3_get_data_stats_req_msg_data_v01_ei;
-
-	resp_desc.max_msg_len = QMI_IPA_GET_DATA_STATS_RESP_MAX_MSG_LEN_V01;
-	resp_desc.msg_id = QMI_IPA_GET_DATA_STATS_RESP_V01;
-	resp_desc.ei_array = ipa3_get_data_stats_resp_msg_data_v01_ei;
-
-	IPAWANDBG_LOW("Sending QMI_IPA_GET_DATA_STATS_REQ_V01\n");
-
-	if (unlikely(!ipa_q6_clnt))
-		return -ETIMEDOUT;
-	rc = ipa3_qmi_send_req_wait(ipa_q6_clnt,
-		&req_desc, req,
-		&resp_desc, resp,
-		QMI_SEND_STATS_REQ_TIMEOUT_MS);
-
-	if (rc < 0) {
-		IPAWANERR("QMI send Req %d failed, rc= %d\n",
-			QMI_IPA_GET_DATA_STATS_REQ_V01,
-			rc);
-		return rc;
-	}
-
-	IPAWANDBG_LOW("QMI_IPA_GET_DATA_STATS_RESP_V01 received\n");
-
-	return ipa3_check_qmi_response(rc,
-		QMI_IPA_GET_DATA_STATS_REQ_V01, resp->resp.result,
-		resp->resp.error, "ipa_get_data_stats_resp_msg_v01");
-}
-
-int ipa3_qmi_get_network_stats(struct ipa_get_apn_data_stats_req_msg_v01 *req,
-			      struct ipa_get_apn_data_stats_resp_msg_v01 *resp)
-{
-	struct ipa_msg_desc req_desc, resp_desc;
-	int rc;
-
-	req_desc.max_msg_len = QMI_IPA_GET_APN_DATA_STATS_REQ_MAX_MSG_LEN_V01;
-	req_desc.msg_id = QMI_IPA_GET_APN_DATA_STATS_REQ_V01;
-	req_desc.ei_array = ipa3_get_apn_data_stats_req_msg_data_v01_ei;
-
-	resp_desc.max_msg_len = QMI_IPA_GET_APN_DATA_STATS_RESP_MAX_MSG_LEN_V01;
-	resp_desc.msg_id = QMI_IPA_GET_APN_DATA_STATS_RESP_V01;
-	resp_desc.ei_array = ipa3_get_apn_data_stats_resp_msg_data_v01_ei;
-
-	IPAWANDBG_LOW("Sending QMI_IPA_GET_APN_DATA_STATS_REQ_V01\n");
-
-	if (unlikely(!ipa_q6_clnt))
-		return -ETIMEDOUT;
-	rc = ipa3_qmi_send_req_wait(ipa_q6_clnt,
-		&req_desc, req,
-		&resp_desc, resp,
-		QMI_SEND_STATS_REQ_TIMEOUT_MS);
-
-	if (rc < 0) {
-		IPAWANERR("QMI send Req %d failed, rc= %d\n",
-			QMI_IPA_GET_APN_DATA_STATS_REQ_V01,
-			rc);
-		return rc;
-	}
-
-	IPAWANDBG_LOW("QMI_IPA_GET_APN_DATA_STATS_RESP_V01 received\n");
-
-	return ipa3_check_qmi_response(rc,
-		QMI_IPA_GET_APN_DATA_STATS_REQ_V01, resp->resp.result,
-		resp->resp.error, "ipa_get_apn_data_stats_req_msg_v01");
-}
-
-int ipa3_qmi_set_data_quota(struct ipa_set_data_usage_quota_req_msg_v01 *req)
-{
-	struct ipa_set_data_usage_quota_resp_msg_v01 resp;
-	struct ipa_msg_desc req_desc, resp_desc;
-	int rc;
-
-	memset(&resp, 0, sizeof(struct ipa_set_data_usage_quota_resp_msg_v01));
-
-	req_desc.max_msg_len = QMI_IPA_SET_DATA_USAGE_QUOTA_REQ_MAX_MSG_LEN_V01;
-	req_desc.msg_id = QMI_IPA_SET_DATA_USAGE_QUOTA_REQ_V01;
-	req_desc.ei_array = ipa3_set_data_usage_quota_req_msg_data_v01_ei;
-
-	resp_desc.max_msg_len =
-		QMI_IPA_SET_DATA_USAGE_QUOTA_RESP_MAX_MSG_LEN_V01;
-	resp_desc.msg_id = QMI_IPA_SET_DATA_USAGE_QUOTA_RESP_V01;
-	resp_desc.ei_array = ipa3_set_data_usage_quota_resp_msg_data_v01_ei;
-
-	IPAWANDBG_LOW("Sending QMI_IPA_SET_DATA_USAGE_QUOTA_REQ_V01\n");
-	if (unlikely(!ipa_q6_clnt))
-		return -ETIMEDOUT;
-	rc = ipa3_qmi_send_req_wait(ipa_q6_clnt,
-		&req_desc, req,
-		&resp_desc, &resp,
-		QMI_SEND_STATS_REQ_TIMEOUT_MS);
-
-	if (rc < 0) {
-		IPAWANERR("QMI send Req %d failed, rc= %d\n",
-			QMI_IPA_SET_DATA_USAGE_QUOTA_REQ_V01,
-			rc);
-		return rc;
-	}
-
-	IPAWANDBG_LOW("QMI_IPA_SET_DATA_USAGE_QUOTA_RESP_V01 received\n");
-
-	return ipa3_check_qmi_response(rc,
-		QMI_IPA_SET_DATA_USAGE_QUOTA_REQ_V01, resp.resp.result,
-		resp.resp.error, "ipa_set_data_usage_quota_req_msg_v01");
-}
-
-int ipa3_qmi_set_aggr_info(enum ipa_aggr_enum_type_v01 aggr_enum_type)
-{
-	struct ipa_mhi_prime_aggr_info_resp_msg_v01 resp;
-	struct ipa_msg_desc req_desc, resp_desc;
-	int rc;
-
-	IPAWANDBG("sending aggr_info_request\n");
-
-	/* replace to right qmap format */
-	aggr_req.aggr_info[1].aggr_type = aggr_enum_type;
-	aggr_req.aggr_info[2].aggr_type = aggr_enum_type;
-	aggr_req.aggr_info[3].aggr_type = aggr_enum_type;
-	aggr_req.aggr_info[4].aggr_type = aggr_enum_type;
-
-	memset(&resp, 0, sizeof(struct ipa_mhi_prime_aggr_info_resp_msg_v01));
-
-	req_desc.max_msg_len = IPA_MHI_PRIME_AGGR_INFO_REQ_MSG_V01_MAX_MSG_LEN;
-	req_desc.msg_id = QMI_IPA_MHI_PRIME_AGGR_INFO_REQ_V01;
-	req_desc.ei_array = ipa_mhi_prime_aggr_info_req_msg_v01_ei;
-
-	resp_desc.max_msg_len =
-		IPA_MHI_PRIME_AGGR_INFO_RESP_MSG_V01_MAX_MSG_LEN;
-	resp_desc.msg_id = QMI_IPA_MHI_PRIME_AGGR_INFO_RESP_V01;
-	resp_desc.ei_array = ipa_mhi_prime_aggr_info_resp_msg_v01_ei;
-
-	IPAWANDBG("Sending QMI_IPA_MHI_PRIME_AGGR_INFO_REQ_V01(%d)\n",
-		aggr_enum_type);
-	if (unlikely(!ipa_q6_clnt)) {
-		IPAWANERR(" ipa_q6_clnt not initialized\n");
-		return -ETIMEDOUT;
-	}
-	rc = ipa3_qmi_send_req_wait(ipa_q6_clnt,
-		&req_desc, &aggr_req,
-		&resp_desc, &resp,
-		QMI_SEND_STATS_REQ_TIMEOUT_MS);
-
-	if (rc < 0) {
-		IPAWANERR("QMI send Req %d failed, rc= %d\n",
-			QMI_IPA_SET_DATA_USAGE_QUOTA_REQ_V01,
-			rc);
-		return rc;
-	}
-
-	IPAWANDBG_LOW("QMI_IPA_MHI_PRIME_AGGR_INFO_RESP_V01 received\n");
-
-	return ipa3_check_qmi_response(rc,
-		QMI_IPA_SET_DATA_USAGE_QUOTA_REQ_V01, resp.resp.result,
-		resp.resp.error, "ipa_mhi_prime_aggr_info_req_msg_v01");
-}
-
-int ipa3_qmi_stop_data_qouta(void)
-{
-	struct ipa_stop_data_usage_quota_req_msg_v01 req;
-	struct ipa_stop_data_usage_quota_resp_msg_v01 resp;
-	struct ipa_msg_desc req_desc, resp_desc;
-	int rc;
-
-	memset(&req, 0, sizeof(struct ipa_stop_data_usage_quota_req_msg_v01));
-	memset(&resp, 0, sizeof(struct ipa_stop_data_usage_quota_resp_msg_v01));
-
-	req_desc.max_msg_len =
-		QMI_IPA_STOP_DATA_USAGE_QUOTA_REQ_MAX_MSG_LEN_V01;
-	req_desc.msg_id = QMI_IPA_STOP_DATA_USAGE_QUOTA_REQ_V01;
-	req_desc.ei_array = ipa3_stop_data_usage_quota_req_msg_data_v01_ei;
-
-	resp_desc.max_msg_len =
-		QMI_IPA_STOP_DATA_USAGE_QUOTA_RESP_MAX_MSG_LEN_V01;
-	resp_desc.msg_id = QMI_IPA_STOP_DATA_USAGE_QUOTA_RESP_V01;
-	resp_desc.ei_array = ipa3_stop_data_usage_quota_resp_msg_data_v01_ei;
-
-	IPAWANDBG_LOW("Sending QMI_IPA_STOP_DATA_USAGE_QUOTA_REQ_V01\n");
-	if (unlikely(!ipa_q6_clnt))
-		return -ETIMEDOUT;
-	rc = ipa3_qmi_send_req_wait(ipa_q6_clnt,
-		&req_desc, &req,
-		&resp_desc, &resp,
-		QMI_SEND_STATS_REQ_TIMEOUT_MS);
-
-	if (rc < 0) {
-		IPAWANERR("QMI send Req %d failed, rc= %d\n",
-			QMI_IPA_STOP_DATA_USAGE_QUOTA_REQ_V01,
-			rc);
-		return rc;
-	}
-
-	IPAWANDBG_LOW("QMI_IPA_STOP_DATA_USAGE_QUOTA_RESP_V01 received\n");
-
-	return ipa3_check_qmi_response(rc,
-		QMI_IPA_STOP_DATA_USAGE_QUOTA_REQ_V01, resp.resp.result,
-		resp.resp.error, "ipa_stop_data_usage_quota_req_msg_v01");
-}
-
-int ipa3_qmi_enable_per_client_stats(
-	struct ipa_enable_per_client_stats_req_msg_v01 *req,
-	struct ipa_enable_per_client_stats_resp_msg_v01 *resp)
-{
-	struct ipa_msg_desc req_desc, resp_desc;
-	int rc = 0;
-
-	req_desc.max_msg_len =
-		QMI_IPA_ENABLE_PER_CLIENT_STATS_REQ_MAX_MSG_LEN_V01;
-	req_desc.msg_id =
-		QMI_IPA_ENABLE_PER_CLIENT_STATS_REQ_V01;
-	req_desc.ei_array =
-		ipa3_enable_per_client_stats_req_msg_data_v01_ei;
-
-	resp_desc.max_msg_len =
-		QMI_IPA_ENABLE_PER_CLIENT_STATS_RESP_MAX_MSG_LEN_V01;
-	resp_desc.msg_id =
-		QMI_IPA_ENABLE_PER_CLIENT_STATS_RESP_V01;
-	resp_desc.ei_array =
-		ipa3_enable_per_client_stats_resp_msg_data_v01_ei;
-
-	IPAWANDBG("Sending QMI_IPA_ENABLE_PER_CLIENT_STATS_REQ_V01\n");
-
-	rc = ipa3_qmi_send_req_wait(ipa_q6_clnt,
-		&req_desc, req,
-		&resp_desc, resp,
-		QMI_SEND_STATS_REQ_TIMEOUT_MS);
-
-	if (rc < 0) {
-		IPAWANERR("send Req %d failed, rc= %d\n",
-			QMI_IPA_ENABLE_PER_CLIENT_STATS_REQ_V01,
-			rc);
-		return rc;
-	}
-
-	IPAWANDBG("QMI_IPA_ENABLE_PER_CLIENT_STATS_RESP_V01 received\n");
-
-	return ipa3_check_qmi_response(rc,
-		QMI_IPA_ENABLE_PER_CLIENT_STATS_REQ_V01, resp->resp.result,
-		resp->resp.error, "ipa3_qmi_enable_per_client_stats");
-}
-
-int ipa3_qmi_get_per_client_packet_stats(
-	struct ipa_get_stats_per_client_req_msg_v01 *req,
-	struct ipa_get_stats_per_client_resp_msg_v01 *resp)
-{
-	struct ipa_msg_desc req_desc, resp_desc;
-	int rc;
-
-	req_desc.max_msg_len = QMI_IPA_GET_STATS_PER_CLIENT_REQ_MAX_MSG_LEN_V01;
-	req_desc.msg_id = QMI_IPA_GET_STATS_PER_CLIENT_REQ_V01;
-	req_desc.ei_array = ipa3_get_stats_per_client_req_msg_data_v01_ei;
-
-	resp_desc.max_msg_len =
-		QMI_IPA_GET_STATS_PER_CLIENT_RESP_MAX_MSG_LEN_V01;
-	resp_desc.msg_id = QMI_IPA_GET_STATS_PER_CLIENT_RESP_V01;
-	resp_desc.ei_array = ipa3_get_stats_per_client_resp_msg_data_v01_ei;
-
-	IPAWANDBG("Sending QMI_IPA_GET_STATS_PER_CLIENT_REQ_V01\n");
-
-	rc = ipa3_qmi_send_req_wait(ipa_q6_clnt,
-		&req_desc, req,
-		&resp_desc, resp,
-		QMI_SEND_STATS_REQ_TIMEOUT_MS);
-
-	if (rc < 0) {
-		IPAWANERR("send Req %d failed, rc= %d\n",
-			QMI_IPA_GET_STATS_PER_CLIENT_REQ_V01,
-			rc);
-		return rc;
-	}
-
-	IPAWANDBG("QMI_IPA_GET_STATS_PER_CLIENT_RESP_V01 received\n");
-
-	return ipa3_check_qmi_response(rc,
-		QMI_IPA_GET_STATS_PER_CLIENT_REQ_V01, resp->resp.result,
-		resp->resp.error,
-		"struct ipa_get_stats_per_client_req_msg_v01");
-}
-
-int ipa3_qmi_send_mhi_ready_indication(
-	struct ipa_mhi_ready_indication_msg_v01 *req)
-{
-	IPAWANDBG("Sending QMI_IPA_MHI_READY_IND_V01\n");
-
-	if (unlikely(!ipa3_svc_handle))
-		return -ETIMEDOUT;
-
-	return qmi_send_indication(ipa3_svc_handle,
-		&ipa3_qmi_ctx->client_sq,
-		QMI_IPA_MHI_READY_IND_V01,
-		IPA_MHI_READY_INDICATION_MSG_V01_MAX_MSG_LEN,
-		ipa_mhi_ready_indication_msg_v01_ei,
-		req);
-}
-
-int ipa3_qmi_send_mhi_cleanup_request(struct ipa_mhi_cleanup_req_msg_v01 *req)
-{
-
-	struct ipa_msg_desc req_desc, resp_desc;
-	struct ipa_mhi_cleanup_resp_msg_v01 resp;
-	int rc;
-
-	memset(&resp, 0, sizeof(resp));
-
-	IPAWANDBG("Sending QMI_IPA_MHI_CLEANUP_REQ_V01\n");
-	if (unlikely(!ipa_q6_clnt))
-		return -ETIMEDOUT;
-
-	req_desc.max_msg_len = IPA_MHI_CLK_VOTE_REQ_MSG_V01_MAX_MSG_LEN;
-	req_desc.msg_id = QMI_IPA_MHI_CLEANUP_REQ_V01;
-	req_desc.ei_array = ipa_mhi_cleanup_req_msg_v01_ei;
-
-	resp_desc.max_msg_len = IPA_MHI_CLK_VOTE_RESP_MSG_V01_MAX_MSG_LEN;
-	resp_desc.msg_id = QMI_IPA_MHI_CLEANUP_RESP_V01;
-	resp_desc.ei_array = ipa_mhi_cleanup_resp_msg_v01_ei;
-
-	rc = ipa3_qmi_send_req_wait(ipa_q6_clnt,
-		&req_desc, req,
-		&resp_desc, &resp,
-		QMI_MHI_SEND_REQ_TIMEOUT_MS);
-
-	IPAWANDBG("QMI_IPA_MHI_CLEANUP_RESP_V01 received\n");
-
-	return ipa3_check_qmi_response(rc,
-		QMI_IPA_MHI_CLEANUP_REQ_V01, resp.resp.result,
-		resp.resp.error, "ipa_mhi_cleanup_req_msg");
-}
-
-int ipa3_qmi_send_rsc_pipe_indication(
-	struct ipa_endp_desc_indication_msg_v01 *req)
-{
-	IPAWANDBG("Sending QMI_IPA_ENDP_DESC_INDICATION_V01\n");
-
-	if (unlikely(!ipa3_svc_handle))
-		return -ETIMEDOUT;
-
-	return qmi_send_indication(ipa3_svc_handle,
-		&ipa3_qmi_ctx->client_sq,
-		QMI_IPA_ENDP_DESC_INDICATION_V01,
-		IPA_ENDP_DESC_INDICATION_MSG_V01_MAX_MSG_LEN,
-		ipa_endp_desc_indication_msg_v01_ei,
-		req);
-}
-
-void ipa3_qmi_init(void)
-{
-	mutex_init(&ipa3_qmi_lock);
-}
-
-void ipa3_qmi_cleanup(void)
-{
-	mutex_destroy(&ipa3_qmi_lock);
-}
-

ipa/ipa_v3/ipa_qmi_service.h

@@ -1,533 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2013-2019, The Linux Foundation. All rights reserved.
- */
-
-#ifndef IPA_QMI_SERVICE_H
-#define IPA_QMI_SERVICE_H
-
-#include <linux/ipa.h>
-#include <linux/ipa_qmi_service_v01.h>
-#include <uapi/linux/msm_rmnet.h>
-#include <linux/soc/qcom/qmi.h>
-#include "ipa_i.h"
-#include <linux/rmnet_ipa_fd_ioctl.h>
-
-/**
- * name of the DL wwan default routing tables for v4 and v6
- */
-#define IPA_A7_QMAP_HDR_NAME "ipa_qmap_hdr"
-#define IPA_DFLT_WAN_RT_TBL_NAME "ipa_dflt_wan_rt"
-#define MAX_NUM_Q6_RULE 35
-#define MAX_NUM_QMI_RULE_CACHE 10
-#define MAX_NUM_QMI_MPM_AGGR_CACHE 3
-#define DEV_NAME "ipa-wan"
-#define SUBSYS_LOCAL_MODEM "modem"
-#define SUBSYS_REMOTE_MODEM "esoc0"
-
-
-#define IPAWANDBG(fmt, args...) \
-	do { \
-		pr_debug(DEV_NAME " %s:%d " fmt, __func__,\
-				__LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-				DEV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-				DEV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-
-#define IPAWANDBG_LOW(fmt, args...) \
-	do { \
-		pr_debug(DEV_NAME " %s:%d " fmt, __func__,\
-				__LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-				DEV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-#define IPAWANERR(fmt, args...) \
-	do { \
-		pr_err(DEV_NAME " %s:%d " fmt, __func__,\
-				__LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-				DEV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-				DEV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-#define IPAWANERR_RL(fmt, args...) \
-	do { \
-		pr_err_ratelimited_ipa(DEV_NAME " %s:%d " fmt, __func__,\
-				__LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-				DEV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-				DEV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-#define IPAWANINFO(fmt, args...) \
-	do { \
-		pr_info(DEV_NAME " %s:%d " fmt, __func__,\
-				__LINE__, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-				DEV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-				DEV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-extern struct ipa3_qmi_context *ipa3_qmi_ctx;
-
-struct ipa_offload_connection_val {
-	enum ipa_ip_type_enum_v01 ip_type;
-	bool valid;
-	uint32_t rule_id;
-	uint32_t  rule_hdl;
-};
-
-struct ipa3_qmi_context {
-	struct ipa_ioc_ext_intf_prop q6_ul_filter_rule[MAX_NUM_Q6_RULE];
-	u32 q6_ul_filter_rule_hdl[MAX_NUM_Q6_RULE];
-	int num_ipa_install_fltr_rule_req_msg;
-	struct ipa_install_fltr_rule_req_msg_v01
-		ipa_install_fltr_rule_req_msg_cache[MAX_NUM_QMI_RULE_CACHE];
-	int num_ipa_install_fltr_rule_req_ex_msg;
-	struct ipa_install_fltr_rule_req_ex_msg_v01
-		ipa_install_fltr_rule_req_ex_msg_cache[MAX_NUM_QMI_RULE_CACHE];
-	int num_ipa_fltr_installed_notif_req_msg;
-	struct ipa_fltr_installed_notif_req_msg_v01
-		ipa_fltr_installed_notif_req_msg_cache[MAX_NUM_QMI_RULE_CACHE];
-	int num_ipa_configure_ul_firewall_rules_req_msg;
-	struct ipa_configure_ul_firewall_rules_req_msg_v01
-		ipa_configure_ul_firewall_rules_req_msg_cache
-			[MAX_NUM_QMI_RULE_CACHE];
-	struct ipa_mhi_prime_aggr_info_req_msg_v01
-		ipa_mhi_prime_aggr_info_req_msg_cache
-			[MAX_NUM_QMI_MPM_AGGR_CACHE];
-	bool modem_cfg_emb_pipe_flt;
-	struct sockaddr_qrtr client_sq;
-	struct sockaddr_qrtr server_sq;
-	int num_ipa_offload_connection;
-	struct ipa_offload_connection_val
-		ipa_offload_cache[QMI_IPA_MAX_FILTERS_V01];
-	uint8_t ul_firewall_indices_list_valid;
-	uint32_t ul_firewall_indices_list_len;
-	uint32_t ul_firewall_indices_list[QMI_IPA_MAX_FILTERS_V01];
-};
-
-struct ipa3_rmnet_mux_val {
-	uint32_t  mux_id;
-	int8_t    vchannel_name[IFNAMSIZ];
-	bool mux_channel_set;
-	bool ul_flt_reg;
-	bool mux_hdr_set;
-	uint32_t  hdr_hdl;
-};
-
-extern struct qmi_elem_info
-	ipa3_init_modem_driver_req_msg_data_v01_ei[];
-extern struct qmi_elem_info
-	ipa3_init_modem_driver_resp_msg_data_v01_ei[];
-extern struct qmi_elem_info
-	ipa3_indication_reg_req_msg_data_v01_ei[];
-extern struct qmi_elem_info ipa3_indication_reg_resp_msg_data_v01_ei[];
-
-extern struct qmi_elem_info
-	ipa3_master_driver_init_complt_ind_msg_data_v01_ei[];
-extern struct qmi_elem_info ipa3_install_fltr_rule_req_msg_data_v01_ei[];
-extern struct qmi_elem_info ipa3_install_fltr_rule_resp_msg_data_v01_ei[];
-extern struct qmi_elem_info ipa3_fltr_installed_notif_req_msg_data_v01_ei[];
-
-extern struct qmi_elem_info
-	ipa3_fltr_installed_notif_resp_msg_data_v01_ei[];
-extern struct qmi_elem_info
-	ipa3_enable_force_clear_datapath_req_msg_data_v01_ei[];
-extern struct qmi_elem_info
-	ipa3_enable_force_clear_datapath_resp_msg_data_v01_ei[];
-extern struct qmi_elem_info
-	ipa3_disable_force_clear_datapath_req_msg_data_v01_ei[];
-extern struct qmi_elem_info
-	ipa3_disable_force_clear_datapath_resp_msg_data_v01_ei[];
-
-extern struct qmi_elem_info ipa3_config_req_msg_data_v01_ei[];
-extern struct qmi_elem_info ipa3_config_resp_msg_data_v01_ei[];
-extern struct qmi_elem_info ipa3_get_data_stats_req_msg_data_v01_ei[];
-extern struct qmi_elem_info ipa3_get_data_stats_resp_msg_data_v01_ei[];
-extern struct qmi_elem_info ipa3_get_apn_data_stats_req_msg_data_v01_ei[];
-extern struct qmi_elem_info ipa3_get_apn_data_stats_resp_msg_data_v01_ei[];
-extern struct qmi_elem_info ipa3_set_data_usage_quota_req_msg_data_v01_ei[];
-
-extern struct qmi_elem_info
-	ipa3_set_data_usage_quota_resp_msg_data_v01_ei[];
-extern struct qmi_elem_info
-	ipa3_data_usage_quota_reached_ind_msg_data_v01_ei[];
-extern struct qmi_elem_info
-	ipa3_stop_data_usage_quota_req_msg_data_v01_ei[];
-extern struct qmi_elem_info
-	ipa3_stop_data_usage_quota_resp_msg_data_v01_ei[];
-extern struct qmi_elem_info
-	ipa3_init_modem_driver_cmplt_req_msg_data_v01_ei[];
-extern struct qmi_elem_info
-	ipa3_init_modem_driver_cmplt_resp_msg_data_v01_ei[];
-extern struct qmi_elem_info
-	ipa3_install_fltr_rule_req_ex_msg_data_v01_ei[];
-extern struct qmi_elem_info
-	ipa3_install_fltr_rule_resp_ex_msg_data_v01_ei[];
-extern struct qmi_elem_info
-	ipa3_ul_firewall_rule_type_data_v01_ei[];
-extern struct qmi_elem_info
-	ipa3_ul_firewall_config_result_type_data_v01_ei[];
-extern struct
-	qmi_elem_info ipa3_per_client_stats_info_type_data_v01_ei[];
-extern struct qmi_elem_info
-	ipa3_enable_per_client_stats_req_msg_data_v01_ei[];
-extern struct qmi_elem_info
-	ipa3_enable_per_client_stats_resp_msg_data_v01_ei[];
-extern struct qmi_elem_info
-	ipa3_get_stats_per_client_req_msg_data_v01_ei[];
-
-extern struct qmi_elem_info
-	ipa3_get_stats_per_client_resp_msg_data_v01_ei[];
-extern struct qmi_elem_info
-	ipa3_configure_ul_firewall_rules_req_msg_data_v01_ei[];
-extern struct qmi_elem_info
-	ipa3_configure_ul_firewall_rules_resp_msg_data_v01_ei[];
-extern struct qmi_elem_info
-	ipa3_configure_ul_firewall_rules_ind_msg_data_v01_ei[];
-
-extern struct qmi_elem_info ipa_mhi_ready_indication_msg_v01_ei[];
-extern struct qmi_elem_info ipa_mhi_mem_addr_info_type_v01_ei[];
-extern struct qmi_elem_info ipa_mhi_tr_info_type_v01_ei[];
-extern struct qmi_elem_info ipa_mhi_er_info_type_v01_ei[];
-extern struct qmi_elem_info ipa_mhi_alloc_channel_req_msg_v01_ei[];
-extern struct qmi_elem_info ipa_mhi_ch_alloc_resp_type_v01_ei[];
-extern struct qmi_elem_info ipa_mhi_alloc_channel_resp_msg_v01_ei[];
-extern struct qmi_elem_info ipa_mhi_clk_vote_req_msg_v01_ei[];
-extern struct qmi_elem_info ipa_mhi_clk_vote_resp_msg_v01_ei[];
-extern struct qmi_elem_info ipa_mhi_cleanup_req_msg_v01_ei[];
-extern struct qmi_elem_info ipa_mhi_cleanup_resp_msg_v01_ei[];
-
-extern struct qmi_elem_info ipa_endp_desc_indication_msg_v01_ei[];
-extern struct qmi_elem_info ipa_mhi_prime_aggr_info_req_msg_v01_ei[];
-extern struct qmi_elem_info ipa_mhi_prime_aggr_info_resp_msg_v01_ei[];
-extern struct qmi_elem_info ipa_add_offload_connection_req_msg_v01_ei[];
-extern struct qmi_elem_info ipa_add_offload_connection_resp_msg_v01_ei[];
-extern struct qmi_elem_info ipa_remove_offload_connection_req_msg_v01_ei[];
-extern struct qmi_elem_info ipa_remove_offload_connection_resp_msg_v01_ei[];
-
-/**
- * struct ipa3_rmnet_context - IPA rmnet context
- * @ipa_rmnet_ssr: support modem SSR
- * @polling_interval: Requested interval for polling tethered statistics
- * @metered_mux_id: The mux ID on which quota has been set
- */
-struct ipa3_rmnet_context {
-	bool ipa_rmnet_ssr;
-	u64 polling_interval;
-	u32 metered_mux_id;
-};
-
-extern struct ipa3_rmnet_context ipa3_rmnet_ctx;
-
-#ifdef CONFIG_RMNET_IPA3
-
-int ipa3_qmi_service_init(uint32_t wan_platform_type);
-
-void ipa3_qmi_service_exit(void);
-
-/* sending filter-install-request to modem*/
-int ipa3_qmi_filter_request_send(
-	struct ipa_install_fltr_rule_req_msg_v01 *req);
-
-int ipa3_qmi_filter_request_ex_send(
-	struct ipa_install_fltr_rule_req_ex_msg_v01 *req);
-
-int ipa3_qmi_add_offload_request_send(
-	struct ipa_add_offload_connection_req_msg_v01 *req);
-
-int ipa3_qmi_rmv_offload_request_send(
-	struct ipa_remove_offload_connection_req_msg_v01 *req);
-
-int ipa3_qmi_ul_filter_request_send(
-	struct ipa_configure_ul_firewall_rules_req_msg_v01 *req);
-
-/* sending filter-installed-notify-request to modem*/
-int ipa3_qmi_filter_notify_send(struct ipa_fltr_installed_notif_req_msg_v01
-		*req);
-
-/* voting for bus BW to ipa_rm*/
-int ipa3_vote_for_bus_bw(uint32_t *bw_mbps);
-
-int ipa3_qmi_enable_force_clear_datapath_send(
-	struct ipa_enable_force_clear_datapath_req_msg_v01 *req);
-
-int ipa3_qmi_disable_force_clear_datapath_send(
-	struct ipa_disable_force_clear_datapath_req_msg_v01 *req);
-
-int ipa3_copy_ul_filter_rule_to_ipa(struct ipa_install_fltr_rule_req_msg_v01
-	*rule_req);
-
-int ipa3_wwan_update_mux_channel_prop(void);
-
-int ipa3_wan_ioctl_init(void);
-
-void ipa3_wan_ioctl_stop_qmi_messages(void);
-
-void ipa3_wan_ioctl_enable_qmi_messages(void);
-
-void ipa3_wan_ioctl_deinit(void);
-
-void ipa3_qmi_stop_workqueues(void);
-
-int rmnet_ipa3_poll_tethering_stats(struct wan_ioctl_poll_tethering_stats
-		*data);
-
-int rmnet_ipa3_set_data_quota(struct wan_ioctl_set_data_quota *data);
-
-void ipa3_broadcast_quota_reach_ind(uint32_t mux_id,
-	enum ipa_upstream_type upstream_type);
-
-int rmnet_ipa3_set_tether_client_pipe(struct wan_ioctl_set_tether_client_pipe
-	*data);
-
-int rmnet_ipa3_query_tethering_stats(struct wan_ioctl_query_tether_stats *data,
-	bool reset);
-
-int rmnet_ipa3_query_tethering_stats_all(
-	struct wan_ioctl_query_tether_stats_all *data);
-
-int rmnet_ipa3_reset_tethering_stats(struct wan_ioctl_reset_tether_stats *data);
-int rmnet_ipa3_set_lan_client_info(struct wan_ioctl_lan_client_info *data);
-
-int rmnet_ipa3_clear_lan_client_info(struct wan_ioctl_lan_client_info *data);
-
-int rmnet_ipa3_send_lan_client_msg(struct wan_ioctl_send_lan_client_msg *data);
-
-int rmnet_ipa3_enable_per_client_stats(bool *data);
-
-int rmnet_ipa3_query_per_client_stats(
-	struct wan_ioctl_query_per_client_stats *data);
-
-int ipa3_qmi_get_data_stats(struct ipa_get_data_stats_req_msg_v01 *req,
-	struct ipa_get_data_stats_resp_msg_v01 *resp);
-
-int ipa3_qmi_get_network_stats(struct ipa_get_apn_data_stats_req_msg_v01 *req,
-	struct ipa_get_apn_data_stats_resp_msg_v01 *resp);
-
-int ipa3_qmi_set_data_quota(struct ipa_set_data_usage_quota_req_msg_v01 *req);
-
-int ipa3_qmi_set_aggr_info(
-	enum ipa_aggr_enum_type_v01 aggr_enum_type);
-
-int ipa3_qmi_stop_data_qouta(void);
-
-void ipa3_q6_handshake_complete(bool ssr_bootup);
-
-int ipa3_wwan_set_modem_perf_profile(int throughput);
-
-int ipa3_wwan_set_modem_state(struct wan_ioctl_notify_wan_state *state);
-int ipa3_qmi_enable_per_client_stats(
-	struct ipa_enable_per_client_stats_req_msg_v01 *req,
-	struct ipa_enable_per_client_stats_resp_msg_v01 *resp);
-
-int ipa3_qmi_get_per_client_packet_stats(
-	struct ipa_get_stats_per_client_req_msg_v01 *req,
-	struct ipa_get_stats_per_client_resp_msg_v01 *resp);
-
-int ipa3_qmi_send_mhi_ready_indication(
-	struct ipa_mhi_ready_indication_msg_v01 *req);
-
-int ipa3_qmi_send_rsc_pipe_indication(
-	struct ipa_endp_desc_indication_msg_v01 *req);
-
-int ipa3_qmi_send_mhi_cleanup_request(struct ipa_mhi_cleanup_req_msg_v01 *req);
-
-void ipa3_qmi_init(void);
-
-void ipa3_qmi_cleanup(void);
-
-#else /* CONFIG_RMNET_IPA3 */
-
-static inline int ipa3_qmi_service_init(uint32_t wan_platform_type)
-{
-	return -EPERM;
-}
-
-static inline void ipa3_qmi_service_exit(void) { }
-
-/* sending filter-install-request to modem*/
-static inline int ipa3_qmi_filter_request_send(
-	struct ipa_install_fltr_rule_req_msg_v01 *req)
-{
-	return -EPERM;
-}
-
-static inline int ipa3_qmi_add_offload_request_send(
-	struct ipa_add_offload_connection_req_msg_v01 *req)
-{
-	return -EPERM;
-}
-
-static inline int ipa3_qmi_rmv_offload_request_send(
-	struct ipa_rmv_offload_connection_req_msg_v01 *req)
-{
-	return -EPERM;
-}
-
-static inline int ipa3_qmi_ul_filter_request_send(
-	struct ipa_configure_ul_firewall_rules_req_msg_v01 *req)
-{
-	return -EPERM;
-}
-
-static inline int ipa3_qmi_filter_request_ex_send(
-	struct ipa_install_fltr_rule_req_ex_msg_v01 *req)
-{
-	return -EPERM;
-}
-
-/* sending filter-installed-notify-request to modem*/
-static inline int ipa3_qmi_filter_notify_send(
-	struct ipa_fltr_installed_notif_req_msg_v01 *req)
-{
-	return -EPERM;
-}
-
-static inline int ipa3_qmi_enable_force_clear_datapath_send(
-	struct ipa_enable_force_clear_datapath_req_msg_v01 *req)
-{
-	return -EPERM;
-}
-
-static inline int ipa3_qmi_disable_force_clear_datapath_send(
-	struct ipa_disable_force_clear_datapath_req_msg_v01 *req)
-{
-	return -EPERM;
-}
-
-static inline int ipa3_copy_ul_filter_rule_to_ipa(
-	struct ipa_install_fltr_rule_req_msg_v01 *rule_req)
-{
-	return -EPERM;
-}
-
-static inline int ipa3_wwan_update_mux_channel_prop(void)
-{
-	return -EPERM;
-}
-
-static inline int ipa3_wan_ioctl_init(void)
-{
-	return -EPERM;
-}
-
-static inline void ipa3_wan_ioctl_stop_qmi_messages(void) { }
-
-static inline void ipa3_wan_ioctl_enable_qmi_messages(void) { }
-
-static inline void ipa3_wan_ioctl_deinit(void) { }
-
-static inline void ipa3_qmi_stop_workqueues(void) { }
-
-static inline int ipa3_vote_for_bus_bw(uint32_t *bw_mbps)
-{
-	return -EPERM;
-}
-
-static inline int rmnet_ipa3_poll_tethering_stats(
-	struct wan_ioctl_poll_tethering_stats *data)
-{
-	return -EPERM;
-}
-
-static inline int rmnet_ipa3_set_data_quota(
-	struct wan_ioctl_set_data_quota *data)
-{
-	return -EPERM;
-}
-
-static inline void ipa3_broadcast_quota_reach_ind(uint32_t mux_id,
-	enum ipa_upstream_type upstream_type) { }
-
-static inline int ipa3_qmi_get_data_stats(
-	struct ipa_get_data_stats_req_msg_v01 *req,
-	struct ipa_get_data_stats_resp_msg_v01 *resp)
-{
-	return -EPERM;
-}
-
-static inline int ipa3_qmi_get_network_stats(
-	struct ipa_get_apn_data_stats_req_msg_v01 *req,
-	struct ipa_get_apn_data_stats_resp_msg_v01 *resp)
-{
-	return -EPERM;
-}
-
-static inline int ipa3_qmi_set_data_quota(
-	struct ipa_set_data_usage_quota_req_msg_v01 *req)
-{
-	return -EPERM;
-}
-
-static inline int ipa3_qmi_stop_data_qouta(void)
-{
-	return -EPERM;
-}
-
-static inline void ipa3_q6_handshake_complete(bool ssr_bootup) { }
-
-static inline int ipa3_qmi_send_mhi_ready_indication(
-	struct ipa_mhi_ready_indication_msg_v01 *req)
-{
-	return -EPERM;
-}
-
-static int ipa3_qmi_send_rsc_pipe_indication(
-	struct ipa_endp_desc_indication_msg_v01 *req)
-{
-	return -EPERM;
-}
-
-static inline int ipa3_qmi_send_mhi_cleanup_request(
-	struct ipa_mhi_cleanup_req_msg_v01 *req)
-{
-	return -EPERM;
-}
-
-static inline int ipa3_wwan_set_modem_perf_profile(
-	int throughput)
-{
-	return -EPERM;
-}
-static inline int ipa3_qmi_enable_per_client_stats(
-	struct ipa_enable_per_client_stats_req_msg_v01 *req,
-	struct ipa_enable_per_client_stats_resp_msg_v01 *resp)
-{
-	return -EPERM;
-}
-
-static inline int ipa3_qmi_get_per_client_packet_stats(
-	struct ipa_get_stats_per_client_req_msg_v01 *req,
-	struct ipa_get_stats_per_client_resp_msg_v01 *resp)
-{
-	return -EPERM;
-}
-
-static inline int ipa3_qmi_set_aggr_info(
-	enum ipa_aggr_enum_type_v01 aggr_enum_type)
-{
-	return -EPERM;
-}
-
-static inline void ipa3_qmi_init(void)
-{
-
-}
-
-static inline void ipa3_qmi_cleanup(void)
-{
-
-}
-
-#endif /* CONFIG_RMNET_IPA3 */
-
-#endif /* IPA_QMI_SERVICE_H */

ipa/ipa_v3/ipa_qmi_service_v01.c

@@ -1,5110 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2013-2019, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/ipa_qmi_service_v01.h>
-
-#include <linux/soc/qcom/qmi.h>
-
-#include "ipa_qmi_service.h"
-
-/* Type Definitions  */
-static struct qmi_elem_info ipa3_hdr_tbl_info_type_data_v01_ei[] = {
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_hdr_tbl_info_type_v01,
-					modem_offset_start),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_hdr_tbl_info_type_v01,
-					modem_offset_end),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-static struct qmi_elem_info ipa3_route_tbl_info_type_data_v01_ei[] = {
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_route_tbl_info_type_v01,
-					route_tbl_start_addr),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_route_tbl_info_type_v01,
-					num_indices),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-static struct qmi_elem_info ipa3_modem_mem_info_type_data_v01_ei[] = {
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_modem_mem_info_type_v01,
-					block_start_addr),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_modem_mem_info_type_v01,
-					size),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-static struct qmi_elem_info ipa3_hdr_proc_ctx_tbl_info_type_data_v01_ei[] = {
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(
-			struct ipa_hdr_proc_ctx_tbl_info_type_v01,
-			modem_offset_start),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(
-			struct ipa_hdr_proc_ctx_tbl_info_type_v01,
-			modem_offset_end),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-static struct qmi_elem_info ipa3_zip_tbl_info_type_data_v01_ei[] = {
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_zip_tbl_info_type_v01,
-					modem_offset_start),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_zip_tbl_info_type_v01,
-					modem_offset_end),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-static struct qmi_elem_info ipa3_ipfltr_range_eq_16_type_data_v01_ei[] = {
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(
-			struct ipa_ipfltr_range_eq_16_type_v01,
-			offset),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_2_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint16_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(
-			struct ipa_ipfltr_range_eq_16_type_v01,
-			range_low),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_2_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint16_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(
-			struct ipa_ipfltr_range_eq_16_type_v01,
-			range_high),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-static struct qmi_elem_info ipa3_ipfltr_mask_eq_32_type_data_v01_ei[] = {
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(
-				struct ipa_ipfltr_mask_eq_32_type_v01,
-				offset),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(
-				struct ipa_ipfltr_mask_eq_32_type_v01,
-				mask),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(
-			struct ipa_ipfltr_mask_eq_32_type_v01,
-			value),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-static struct qmi_elem_info ipa3_ipfltr_eq_16_type_data_v01_ei[] = {
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(
-			struct ipa_ipfltr_eq_16_type_v01,
-			offset),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_2_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint16_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_ipfltr_eq_16_type_v01,
-					value),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-static struct qmi_elem_info ipa3_ipfltr_eq_32_type_data_v01_ei[] = {
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_ipfltr_eq_32_type_v01,
-					offset),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_ipfltr_eq_32_type_v01,
-					value),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-static struct qmi_elem_info ipa3_ipfltr_mask_eq_128_type_data_v01_ei[] = {
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(
-			struct ipa_ipfltr_mask_eq_128_type_v01,
-			offset),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 16,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= STATIC_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(
-			struct ipa_ipfltr_mask_eq_128_type_v01,
-			mask),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 16,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= STATIC_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(
-			struct ipa_ipfltr_mask_eq_128_type_v01,
-			value),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-static struct qmi_elem_info ipa3_filter_rule_type_data_v01_ei[] = {
-	{
-		.data_type	= QMI_UNSIGNED_2_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint16_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(
-			struct ipa_filter_rule_type_v01,
-			rule_eq_bitmap),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(
-			struct ipa_filter_rule_type_v01,
-			tos_eq_present),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_rule_type_v01,
-					tos_eq),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_rule_type_v01,
-					protocol_eq_present),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_rule_type_v01,
-					protocol_eq),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_rule_type_v01,
-					num_ihl_offset_range_16),
-	},
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= QMI_IPA_IPFLTR_NUM_IHL_RANGE_16_EQNS_V01,
-		.elem_size	= sizeof(
-			struct ipa_ipfltr_range_eq_16_type_v01),
-		.array_type	= STATIC_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_rule_type_v01,
-					ihl_offset_range_16),
-		.ei_array	= ipa3_ipfltr_range_eq_16_type_data_v01_ei,
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_rule_type_v01,
-					num_offset_meq_32),
-	},
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= QMI_IPA_IPFLTR_NUM_MEQ_32_EQNS_V01,
-		.elem_size	= sizeof(struct ipa_ipfltr_mask_eq_32_type_v01),
-		.array_type	= STATIC_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_rule_type_v01,
-					offset_meq_32),
-		.ei_array	= ipa3_ipfltr_mask_eq_32_type_data_v01_ei,
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_rule_type_v01,
-					tc_eq_present),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_rule_type_v01,
-					tc_eq),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_rule_type_v01,
-					flow_eq_present),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_rule_type_v01,
-					flow_eq),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_rule_type_v01,
-					ihl_offset_eq_16_present),
-	},
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(struct ipa_ipfltr_eq_16_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_rule_type_v01,
-					ihl_offset_eq_16),
-		.ei_array	= ipa3_ipfltr_eq_16_type_data_v01_ei,
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_rule_type_v01,
-					ihl_offset_eq_32_present),
-	},
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(struct ipa_ipfltr_eq_32_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_rule_type_v01,
-					ihl_offset_eq_32),
-		.ei_array	= ipa3_ipfltr_eq_32_type_data_v01_ei,
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_rule_type_v01,
-					num_ihl_offset_meq_32),
-	},
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= QMI_IPA_IPFLTR_NUM_IHL_MEQ_32_EQNS_V01,
-		.elem_size	= sizeof(struct ipa_ipfltr_mask_eq_32_type_v01),
-		.array_type	= STATIC_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_rule_type_v01,
-					ihl_offset_meq_32),
-		.ei_array	= ipa3_ipfltr_mask_eq_32_type_data_v01_ei,
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_rule_type_v01,
-					num_offset_meq_128),
-	},
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	=
-			QMI_IPA_IPFLTR_NUM_MEQ_128_EQNS_V01,
-		.elem_size	= sizeof(
-			struct ipa_ipfltr_mask_eq_128_type_v01),
-		.array_type	= STATIC_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(
-			struct ipa_filter_rule_type_v01,
-			offset_meq_128),
-		.ei_array	= ipa3_ipfltr_mask_eq_128_type_data_v01_ei,
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_rule_type_v01,
-					metadata_meq32_present),
-	},
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(struct ipa_ipfltr_mask_eq_32_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_rule_type_v01,
-					metadata_meq32),
-		.ei_array	= ipa3_ipfltr_mask_eq_32_type_data_v01_ei,
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_rule_type_v01,
-					ipv4_frag_eq_present),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-static struct qmi_elem_info ipa_filter_spec_type_data_v01_ei[] = {
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_spec_type_v01,
-					filter_spec_identifier),
-	},
-	{
-		.data_type	= QMI_SIGNED_4_BYTE_ENUM,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_spec_type_v01,
-					ip_type),
-	},
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(struct ipa_filter_rule_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_spec_type_v01,
-					filter_rule),
-		.ei_array	= ipa3_filter_rule_type_data_v01_ei,
-	},
-	{
-		.data_type	= QMI_SIGNED_4_BYTE_ENUM,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_spec_type_v01,
-					filter_action),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_spec_type_v01,
-					is_routing_table_index_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_spec_type_v01,
-					route_table_index),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_spec_type_v01,
-					is_mux_id_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_spec_type_v01,
-					mux_id),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-static struct qmi_elem_info ipa_filter_spec_ex_type_data_v01_ei[] = {
-	{
-		.data_type	= QMI_SIGNED_4_BYTE_ENUM,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_spec_ex_type_v01,
-					ip_type),
-	},
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(struct ipa_filter_rule_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_spec_ex_type_v01,
-					filter_rule),
-		.ei_array	= ipa3_filter_rule_type_data_v01_ei,
-	},
-	{
-		.data_type	= QMI_SIGNED_4_BYTE_ENUM,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_spec_ex_type_v01,
-					filter_action),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_spec_ex_type_v01,
-					is_routing_table_index_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_spec_ex_type_v01,
-					route_table_index),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_spec_ex_type_v01,
-					is_mux_id_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_spec_ex_type_v01,
-					mux_id),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_spec_ex_type_v01,
-					rule_id),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_filter_spec_ex_type_v01,
-					is_rule_hashable),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-static struct
-qmi_elem_info ipa3_filter_rule_identifier_to_handle_map_data_v01_ei[] = {
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(
-			struct ipa_filter_rule_identifier_to_handle_map_v01,
-			filter_spec_identifier),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(
-			struct ipa_filter_rule_identifier_to_handle_map_v01,
-			filter_handle),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-static struct qmi_elem_info ipa3_filter_handle_to_index_map_data_v01_ei[] = {
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(
-			struct ipa_filter_handle_to_index_map_v01,
-			filter_handle),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(
-			struct ipa_filter_handle_to_index_map_v01,
-			filter_index),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_init_modem_driver_req_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			platform_type_valid),
-	},
-	{
-		.data_type	= QMI_SIGNED_4_BYTE_ENUM,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			platform_type),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x11,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			hdr_tbl_info_valid),
-	},
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(struct ipa_hdr_tbl_info_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x11,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			hdr_tbl_info),
-		.ei_array	= ipa3_hdr_tbl_info_type_data_v01_ei,
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x12,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			v4_route_tbl_info_valid),
-	},
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(struct ipa_route_tbl_info_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x12,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			v4_route_tbl_info),
-		.ei_array	= ipa3_route_tbl_info_type_data_v01_ei,
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x13,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			v6_route_tbl_info_valid),
-	},
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(struct ipa_route_tbl_info_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x13,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			v6_route_tbl_info),
-		.ei_array	= ipa3_route_tbl_info_type_data_v01_ei,
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x14,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			v4_filter_tbl_start_addr_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x14,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			v4_filter_tbl_start_addr),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x15,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			v6_filter_tbl_start_addr_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x15,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			v6_filter_tbl_start_addr),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x16,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			modem_mem_info_valid),
-	},
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(struct ipa_modem_mem_info_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x16,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			modem_mem_info),
-		.ei_array	= ipa3_modem_mem_info_type_data_v01_ei,
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x17,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			ctrl_comm_dest_end_pt_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x17,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			ctrl_comm_dest_end_pt),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x18,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			is_ssr_bootup_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x18,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			is_ssr_bootup),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x19,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			hdr_proc_ctx_tbl_info_valid),
-	},
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(
-			struct ipa_hdr_proc_ctx_tbl_info_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x19,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			hdr_proc_ctx_tbl_info),
-		.ei_array	= ipa3_hdr_proc_ctx_tbl_info_type_data_v01_ei,
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x1A,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			zip_tbl_info_valid),
-	},
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(struct ipa_zip_tbl_info_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x1A,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			zip_tbl_info),
-		.ei_array	= ipa3_zip_tbl_info_type_data_v01_ei,
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x1B,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			v4_hash_route_tbl_info_valid),
-	},
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(struct ipa_route_tbl_info_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x1B,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			v4_hash_route_tbl_info),
-		.ei_array	= ipa3_route_tbl_info_type_data_v01_ei,
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x1C,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			v6_hash_route_tbl_info_valid),
-	},
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(struct ipa_route_tbl_info_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x1C,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			v6_hash_route_tbl_info),
-		.ei_array	= ipa3_route_tbl_info_type_data_v01_ei,
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x1D,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			v4_hash_filter_tbl_start_addr_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x1D,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			v4_hash_filter_tbl_start_addr),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x1E,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			v6_hash_filter_tbl_start_addr_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x1E,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			v6_hash_filter_tbl_start_addr),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x1F,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			hw_stats_quota_base_addr_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x1F,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			hw_stats_quota_base_addr),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x20,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			hw_stats_quota_size_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x20,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			hw_stats_quota_size),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x21,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			hw_drop_stats_base_addr_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x21,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			hw_drop_stats_base_addr),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x22,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			hw_drop_stats_table_size_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x22,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_req_msg_v01,
-			hw_drop_stats_table_size),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-
-};
-
-struct qmi_elem_info ipa3_init_modem_driver_resp_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(struct qmi_response_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x02,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_resp_msg_v01,
-			resp),
-		.ei_array	= qmi_response_type_v01_ei,
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_resp_msg_v01,
-			ctrl_comm_dest_end_pt_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_resp_msg_v01,
-			ctrl_comm_dest_end_pt),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x11,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_resp_msg_v01,
-			default_end_pt_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x11,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_resp_msg_v01,
-			default_end_pt),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x12,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_resp_msg_v01,
-			modem_driver_init_pending_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x12,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_resp_msg_v01,
-			modem_driver_init_pending),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_init_modem_driver_cmplt_req_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x01,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_cmplt_req_msg_v01,
-			status),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_init_modem_driver_cmplt_resp_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(struct qmi_response_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x02,
-		.offset		= offsetof(
-			struct ipa_init_modem_driver_cmplt_resp_msg_v01,
-			resp),
-		.ei_array	= qmi_response_type_v01_ei,
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_indication_reg_req_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_indication_reg_req_msg_v01,
-			master_driver_init_complete_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_indication_reg_req_msg_v01,
-			master_driver_init_complete),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x11,
-		.offset		= offsetof(
-			struct ipa_indication_reg_req_msg_v01,
-			data_usage_quota_reached_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x11,
-		.offset		= offsetof(
-			struct ipa_indication_reg_req_msg_v01,
-			data_usage_quota_reached),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(u8),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x12,
-		.offset		= offsetof(
-			struct ipa_indication_reg_req_msg_v01,
-				ipa_mhi_ready_ind_valid),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(u8),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x13,
-		.offset		= offsetof(
-			struct ipa_indication_reg_req_msg_v01,
-				endpoint_desc_ind_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(u8),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x13,
-		.offset		= offsetof(
-			struct ipa_indication_reg_req_msg_v01,
-				endpoint_desc_ind),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(u8),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x12,
-		.offset		= offsetof(
-			struct ipa_indication_reg_req_msg_v01,
-				ipa_mhi_ready_ind),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_indication_reg_resp_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(struct qmi_response_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x02,
-		.offset		= offsetof(
-			struct ipa_indication_reg_resp_msg_v01,
-			resp),
-		.ei_array	= qmi_response_type_v01_ei,
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_master_driver_init_complt_ind_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(struct qmi_response_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x02,
-		.offset		= offsetof(struct
-			ipa_master_driver_init_complt_ind_msg_v01,
-			master_driver_init_status),
-		.ei_array	= qmi_response_type_v01_ei,
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-static struct qmi_elem_info ipa_filter_rule_req2_type_v01_ei[] = {
-	{
-		.data_type      = QMI_UNSIGNED_2_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u16),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_rule_req2_type_v01,
-					   rule_eq_bitmap),
-	},
-	{
-		.data_type      = QMI_UNSIGNED_1_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_rule_req2_type_v01,
-					   pure_ack_eq_present),
-	},
-	{
-		.data_type      = QMI_UNSIGNED_1_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_rule_req2_type_v01,
-					   pure_ack_eq),
-	},
-	{
-		.data_type      = QMI_UNSIGNED_1_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_rule_req2_type_v01,
-					   protocol_eq_present),
-	},
-	{
-		.data_type      = QMI_UNSIGNED_1_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_rule_req2_type_v01,
-					   protocol_eq),
-	},
-	{
-		.data_type      = QMI_UNSIGNED_1_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_rule_req2_type_v01,
-					   num_ihl_offset_range_16),
-	},
-	{
-		.data_type      = QMI_STRUCT,
-		.elem_len       = QMI_IPA_IPFLTR_NUM_IHL_RANGE_16_EQNS_V01,
-		.elem_size      = sizeof(
-			struct ipa_ipfltr_range_eq_16_type_v01),
-		.array_type       = STATIC_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_rule_req2_type_v01,
-					   ihl_offset_range_16),
-		.ei_array      = ipa3_ipfltr_range_eq_16_type_data_v01_ei,
-	},
-	{
-		.data_type      = QMI_UNSIGNED_1_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_rule_req2_type_v01,
-					   num_offset_meq_32),
-	},
-	{
-		.data_type      = QMI_STRUCT,
-		.elem_len       = QMI_IPA_IPFLTR_NUM_MEQ_32_EQNS_V01,
-		.elem_size      = sizeof(struct ipa_ipfltr_mask_eq_32_type_v01),
-		.array_type       = STATIC_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_rule_req2_type_v01,
-					   offset_meq_32),
-		.ei_array      = ipa3_ipfltr_mask_eq_32_type_data_v01_ei,
-	},
-	{
-		.data_type      = QMI_UNSIGNED_1_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_rule_req2_type_v01,
-					   tc_eq_present),
-	},
-	{
-		.data_type      = QMI_UNSIGNED_1_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_rule_req2_type_v01,
-					   tc_eq),
-	},
-	{
-		.data_type      = QMI_UNSIGNED_1_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_rule_req2_type_v01,
-					   flow_eq_present),
-	},
-	{
-		.data_type      = QMI_UNSIGNED_4_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u32),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_rule_req2_type_v01,
-					   flow_eq),
-	},
-	{
-		.data_type      = QMI_UNSIGNED_1_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_rule_req2_type_v01,
-					   ihl_offset_eq_16_present),
-	},
-	{
-		.data_type      = QMI_STRUCT,
-		.elem_len       = 1,
-		.elem_size      = sizeof(struct ipa_ipfltr_eq_16_type_v01),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_rule_req2_type_v01,
-					   ihl_offset_eq_16),
-		.ei_array      = ipa3_ipfltr_eq_16_type_data_v01_ei,
-	},
-	{
-		.data_type      = QMI_UNSIGNED_1_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_rule_req2_type_v01,
-					   ihl_offset_eq_32_present),
-	},
-	{
-		.data_type      = QMI_STRUCT,
-		.elem_len       = 1,
-		.elem_size      = sizeof(struct ipa_ipfltr_eq_32_type_v01),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_rule_req2_type_v01,
-					   ihl_offset_eq_32),
-		.ei_array      = ipa3_ipfltr_eq_32_type_data_v01_ei,
-	},
-	{
-		.data_type      = QMI_UNSIGNED_1_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_rule_req2_type_v01,
-					   num_ihl_offset_meq_32),
-	},
-	{
-		.data_type      = QMI_STRUCT,
-		.elem_len       = QMI_IPA_IPFLTR_NUM_IHL_MEQ_32_EQNS_V01,
-		.elem_size      = sizeof(struct ipa_ipfltr_mask_eq_32_type_v01),
-		.array_type       = STATIC_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_rule_req2_type_v01,
-					   ihl_offset_meq_32),
-		.ei_array      = ipa3_ipfltr_mask_eq_32_type_data_v01_ei,
-	},
-	{
-		.data_type      = QMI_UNSIGNED_1_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_rule_req2_type_v01,
-					   num_offset_meq_128),
-	},
-	{
-		.data_type      = QMI_STRUCT,
-		.elem_len       = QMI_IPA_IPFLTR_NUM_MEQ_128_EQNS_V01,
-		.elem_size      = sizeof(
-			struct ipa_ipfltr_mask_eq_128_type_v01),
-		.array_type       = STATIC_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_rule_req2_type_v01,
-					   offset_meq_128),
-		.ei_array      = ipa3_ipfltr_mask_eq_128_type_data_v01_ei,
-	},
-	{
-		.data_type      = QMI_UNSIGNED_1_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_rule_req2_type_v01,
-					   metadata_meq32_present),
-	},
-	{
-		.data_type      = QMI_STRUCT,
-		.elem_len       = 1,
-		.elem_size      = sizeof(struct ipa_ipfltr_mask_eq_32_type_v01),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_rule_req2_type_v01,
-					   metadata_meq32),
-		.ei_array      = ipa3_ipfltr_mask_eq_32_type_data_v01_ei,
-	},
-	{
-		.data_type      = QMI_UNSIGNED_1_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_rule_req2_type_v01,
-					   ipv4_frag_eq_present),
-	},
-	{
-		.data_type      = QMI_EOTI,
-		.array_type       = NO_ARRAY,
-		.tlv_type       = QMI_COMMON_TLV_TYPE,
-	},
-};
-
-static struct qmi_elem_info ipa_filter_spec_ex2_type_v01_ei[] = {
-	{
-		.data_type      = QMI_SIGNED_4_BYTE_ENUM,
-		.elem_len       = 1,
-		.elem_size      = sizeof(enum ipa_ip_type_enum_v01),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_spec_ex2_type_v01,
-					   ip_type),
-	},
-	{
-		.data_type      = QMI_STRUCT,
-		.elem_len       = 1,
-		.elem_size      = sizeof(struct ipa_filter_rule_req2_type_v01),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_spec_ex2_type_v01,
-					   filter_rule),
-		.ei_array      = ipa_filter_rule_req2_type_v01_ei,
-	},
-	{
-		.data_type      = QMI_SIGNED_4_BYTE_ENUM,
-		.elem_len       = 1,
-		.elem_size      = sizeof(enum ipa_filter_action_enum_v01),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_spec_ex2_type_v01,
-					   filter_action),
-	},
-	{
-		.data_type      = QMI_UNSIGNED_1_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_spec_ex2_type_v01,
-					   is_routing_table_index_valid),
-	},
-	{
-		.data_type      = QMI_UNSIGNED_4_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u32),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_spec_ex2_type_v01,
-					   route_table_index),
-	},
-	{
-		.data_type      = QMI_UNSIGNED_1_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_spec_ex2_type_v01,
-					   is_mux_id_valid),
-	},
-	{
-		.data_type      = QMI_UNSIGNED_4_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u32),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_spec_ex2_type_v01,
-					   mux_id),
-	},
-	{
-		.data_type      = QMI_UNSIGNED_4_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u32),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_spec_ex2_type_v01,
-					   rule_id),
-	},
-	{
-		.data_type      = QMI_UNSIGNED_1_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_filter_spec_ex2_type_v01,
-					   is_rule_hashable),
-	},
-	{
-		.data_type      = QMI_EOTI,
-		.array_type       = NO_ARRAY,
-		.tlv_type       = QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_install_fltr_rule_req_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_req_msg_v01,
-			filter_spec_list_valid),
-	},
-	{
-		.data_type	= QMI_DATA_LEN,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_req_msg_v01,
-			filter_spec_list_len),
-	},
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= QMI_IPA_MAX_FILTERS_V01,
-		.elem_size	= sizeof(struct ipa_filter_spec_type_v01),
-		.array_type	= VAR_LEN_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_req_msg_v01,
-			filter_spec_list),
-		.ei_array	= ipa_filter_spec_type_data_v01_ei,
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x11,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_req_msg_v01,
-			source_pipe_index_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x11,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_req_msg_v01,
-			source_pipe_index),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x12,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_req_msg_v01,
-			num_ipv4_filters_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x12,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_req_msg_v01,
-			num_ipv4_filters),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x13,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_req_msg_v01,
-			num_ipv6_filters_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x13,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_req_msg_v01,
-			num_ipv6_filters),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x14,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_req_msg_v01,
-			xlat_filter_indices_list_valid),
-	},
-	{
-		.data_type	= QMI_DATA_LEN,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x14,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_req_msg_v01,
-			xlat_filter_indices_list_len),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= QMI_IPA_MAX_FILTERS_V01,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= VAR_LEN_ARRAY,
-		.tlv_type	= 0x14,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_req_msg_v01,
-			xlat_filter_indices_list),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x15,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_req_msg_v01,
-			filter_spec_ex_list_valid),
-	},
-	{
-		.data_type	= QMI_DATA_LEN,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x15,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_req_msg_v01,
-			filter_spec_ex_list_len),
-	},
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= QMI_IPA_MAX_FILTERS_V01,
-		.elem_size	= sizeof(struct ipa_filter_spec_ex_type_v01),
-		.array_type	= VAR_LEN_ARRAY,
-		.tlv_type	= 0x15,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_req_msg_v01,
-			filter_spec_ex_list),
-		.ei_array	= ipa_filter_spec_ex_type_data_v01_ei,
-	},
-	{
-		.data_type      = QMI_OPT_FLAG,
-		.elem_len       = 1,
-		.elem_size      = sizeof(uint8_t),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x16,
-		.offset         = offsetof(
-			struct ipa_install_fltr_rule_req_msg_v01,
-			filter_spec_ex2_list_valid),
-	},
-	{
-		.data_type      = QMI_DATA_LEN,
-		.elem_len       = 1,
-		.elem_size      = sizeof(uint8_t),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x16,
-		.offset         = offsetof(
-			struct ipa_install_fltr_rule_req_msg_v01,
-			filter_spec_ex2_list_len),
-	},
-	{
-		.data_type      = QMI_STRUCT,
-		.elem_len       = QMI_IPA_MAX_FILTERS_V01,
-		.elem_size      = sizeof(struct ipa_filter_spec_ex2_type_v01),
-		.array_type       = VAR_LEN_ARRAY,
-		.tlv_type       = 0x16,
-		.offset         = offsetof(
-			struct ipa_install_fltr_rule_req_msg_v01,
-			filter_spec_ex2_list),
-		.ei_array      = ipa_filter_spec_ex2_type_v01_ei,
-	},
-	{
-		.data_type      = QMI_OPT_FLAG,
-		.elem_len       = 1,
-		.elem_size      = sizeof(uint8_t),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x17,
-		.offset         = offsetof(
-			struct ipa_install_fltr_rule_req_msg_v01,
-			ul_firewall_indices_list_valid),
-	},
-	{
-		.data_type      = QMI_DATA_LEN,
-		.elem_len       = 1,
-		.elem_size      = sizeof(uint8_t),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x17,
-		.offset         = offsetof(
-			struct ipa_install_fltr_rule_req_msg_v01,
-			ul_firewall_indices_list_len),
-	},
-	{
-		.data_type      = QMI_UNSIGNED_4_BYTE,
-		.elem_len       = QMI_IPA_MAX_FILTERS_V01,
-		.elem_size      = sizeof(uint32_t),
-		.array_type       = VAR_LEN_ARRAY,
-		.tlv_type       = 0x17,
-		.offset         = offsetof(
-			struct ipa_install_fltr_rule_req_msg_v01,
-			ul_firewall_indices_list),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_install_fltr_rule_resp_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(struct qmi_response_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x02,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_resp_msg_v01,
-			resp),
-		.ei_array       = qmi_response_type_v01_ei,
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_resp_msg_v01,
-			filter_handle_list_valid),
-	},
-	{
-		.data_type	= QMI_DATA_LEN,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_resp_msg_v01,
-			filter_handle_list_len),
-	},
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= QMI_IPA_MAX_FILTERS_V01,
-		.elem_size	= sizeof(
-			struct ipa_filter_rule_identifier_to_handle_map_v01),
-		.array_type	= VAR_LEN_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_resp_msg_v01,
-			filter_handle_list),
-		.ei_array	=
-			ipa3_filter_rule_identifier_to_handle_map_data_v01_ei,
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x11,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_resp_msg_v01,
-			rule_id_valid),
-	},
-	{
-		.data_type	= QMI_DATA_LEN,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x11,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_resp_msg_v01,
-			rule_id_len),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= QMI_IPA_MAX_FILTERS_V01,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= VAR_LEN_ARRAY,
-		.tlv_type	= 0x11,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_resp_msg_v01,
-			rule_id),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_fltr_installed_notif_req_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x01,
-		.offset		= offsetof(
-			struct ipa_fltr_installed_notif_req_msg_v01,
-			source_pipe_index),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_2_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint16_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x02,
-		.offset		= offsetof(
-			struct ipa_fltr_installed_notif_req_msg_v01,
-			install_status),
-	},
-	{
-		.data_type	= QMI_DATA_LEN,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x03,
-		.offset		= offsetof(
-			struct ipa_fltr_installed_notif_req_msg_v01,
-			filter_index_list_len),
-	},
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= QMI_IPA_MAX_FILTERS_V01,
-		.elem_size	= sizeof(
-			struct ipa_filter_handle_to_index_map_v01),
-		.array_type	= VAR_LEN_ARRAY,
-		.tlv_type	= 0x03,
-		.offset		= offsetof(
-			struct ipa_fltr_installed_notif_req_msg_v01,
-			filter_index_list),
-		.ei_array	= ipa3_filter_handle_to_index_map_data_v01_ei,
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_fltr_installed_notif_req_msg_v01,
-			embedded_pipe_index_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_fltr_installed_notif_req_msg_v01,
-			embedded_pipe_index),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x11,
-		.offset		= offsetof(
-			struct ipa_fltr_installed_notif_req_msg_v01,
-			retain_header_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x11,
-		.offset		= offsetof(
-			struct ipa_fltr_installed_notif_req_msg_v01,
-			retain_header),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x12,
-		.offset		= offsetof(
-			struct ipa_fltr_installed_notif_req_msg_v01,
-			embedded_call_mux_id_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x12,
-		.offset		= offsetof(
-			struct ipa_fltr_installed_notif_req_msg_v01,
-			embedded_call_mux_id),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x13,
-		.offset		= offsetof(
-			struct ipa_fltr_installed_notif_req_msg_v01,
-			num_ipv4_filters_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x13,
-		.offset		= offsetof(
-			struct ipa_fltr_installed_notif_req_msg_v01,
-			num_ipv4_filters),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x14,
-		.offset		= offsetof(
-			struct ipa_fltr_installed_notif_req_msg_v01,
-			num_ipv6_filters_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x14,
-		.offset		= offsetof(
-			struct ipa_fltr_installed_notif_req_msg_v01,
-			num_ipv6_filters),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x15,
-		.offset		= offsetof(
-			struct ipa_fltr_installed_notif_req_msg_v01,
-			start_ipv4_filter_idx_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x15,
-		.offset		= offsetof(
-			struct ipa_fltr_installed_notif_req_msg_v01,
-			start_ipv4_filter_idx),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x16,
-		.offset		= offsetof(
-			struct ipa_fltr_installed_notif_req_msg_v01,
-			start_ipv6_filter_idx_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x16,
-		.offset		= offsetof(
-			struct ipa_fltr_installed_notif_req_msg_v01,
-			start_ipv6_filter_idx),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x17,
-		.offset		= offsetof(
-			struct ipa_fltr_installed_notif_req_msg_v01,
-			rule_id_valid),
-	},
-	{
-		.data_type	= QMI_DATA_LEN,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x17,
-		.offset		= offsetof(
-			struct ipa_fltr_installed_notif_req_msg_v01,
-			rule_id_len),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= QMI_IPA_MAX_FILTERS_V01,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= VAR_LEN_ARRAY,
-		.tlv_type	= 0x17,
-		.offset		= offsetof(
-			struct ipa_fltr_installed_notif_req_msg_v01,
-			rule_id),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x18,
-		.offset		= offsetof(
-			struct ipa_fltr_installed_notif_req_msg_v01,
-			dst_pipe_id_valid),
-	},
-	{
-		.data_type	= QMI_DATA_LEN,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x18,
-		.offset		= offsetof(
-			struct ipa_fltr_installed_notif_req_msg_v01,
-			dst_pipe_id_len),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= QMI_IPA_MAX_CLIENT_DST_PIPES_V01,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= VAR_LEN_ARRAY,
-		.tlv_type	= 0x18,
-		.offset		= offsetof(
-			struct ipa_fltr_installed_notif_req_msg_v01,
-			dst_pipe_id),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x19,
-		.offset		= offsetof(
-			struct ipa_fltr_installed_notif_req_msg_v01,
-			rule_id_ex_valid),
-	},
-	{
-		.data_type	= QMI_DATA_LEN,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x19,
-		.offset		= offsetof(
-			struct ipa_fltr_installed_notif_req_msg_v01,
-			rule_id_ex_len),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= QMI_IPA_MAX_FILTERS_EX2_V01,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= VAR_LEN_ARRAY,
-		.tlv_type	= 0x19,
-		.offset		= offsetof(
-			struct ipa_fltr_installed_notif_req_msg_v01,
-			rule_id_ex),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_fltr_installed_notif_resp_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(struct qmi_response_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x02,
-		.offset		= offsetof(
-			struct ipa_fltr_installed_notif_resp_msg_v01,
-			resp),
-		.ei_array	= qmi_response_type_v01_ei,
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_enable_force_clear_datapath_req_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x01,
-		.offset		= offsetof(
-			struct ipa_enable_force_clear_datapath_req_msg_v01,
-			source_pipe_bitmask),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x02,
-		.offset		= offsetof(
-			struct ipa_enable_force_clear_datapath_req_msg_v01,
-			request_id),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_enable_force_clear_datapath_req_msg_v01,
-			throttle_source_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_enable_force_clear_datapath_req_msg_v01,
-			throttle_source),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_enable_force_clear_datapath_resp_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(struct qmi_response_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x02,
-		.offset		= offsetof(
-			struct ipa_enable_force_clear_datapath_resp_msg_v01,
-			resp),
-		.ei_array	= qmi_response_type_v01_ei,
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_disable_force_clear_datapath_req_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x01,
-		.offset		= offsetof(
-			struct ipa_disable_force_clear_datapath_req_msg_v01,
-			request_id),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info
-	ipa3_disable_force_clear_datapath_resp_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(struct qmi_response_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x02,
-		.offset		= offsetof(
-			struct ipa_disable_force_clear_datapath_resp_msg_v01,
-			resp),
-		.ei_array	= qmi_response_type_v01_ei,
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_config_req_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_config_req_msg_v01,
-			peripheral_type_valid),
-	},
-	{
-		.data_type	= QMI_SIGNED_4_BYTE_ENUM,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_config_req_msg_v01,
-			peripheral_type),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x11,
-		.offset		= offsetof(
-			struct ipa_config_req_msg_v01,
-			hw_deaggr_supported_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x11,
-		.offset		= offsetof(
-			struct ipa_config_req_msg_v01,
-			hw_deaggr_supported),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x12,
-		.offset		= offsetof(
-			struct ipa_config_req_msg_v01,
-			max_aggr_frame_size_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x12,
-		.offset		= offsetof(
-			struct ipa_config_req_msg_v01,
-					max_aggr_frame_size),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x13,
-		.offset		= offsetof(
-			struct ipa_config_req_msg_v01,
-			ipa_ingress_pipe_mode_valid),
-	},
-	{
-		.data_type	= QMI_SIGNED_4_BYTE_ENUM,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x13,
-		.offset		= offsetof(
-			struct ipa_config_req_msg_v01,
-			ipa_ingress_pipe_mode),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x14,
-		.offset		= offsetof(
-			struct ipa_config_req_msg_v01,
-			peripheral_speed_info_valid),
-	},
-	{
-		.data_type	= QMI_SIGNED_4_BYTE_ENUM,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x14,
-		.offset		= offsetof(
-			struct ipa_config_req_msg_v01,
-			peripheral_speed_info),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x15,
-		.offset		= offsetof(
-			struct ipa_config_req_msg_v01,
-			dl_accumulation_time_limit_valid),
-	},
-	{
-		.data_type	= QMI_SIGNED_4_BYTE_ENUM,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x15,
-		.offset		= offsetof(
-			struct ipa_config_req_msg_v01,
-			dl_accumulation_time_limit),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x16,
-		.offset		= offsetof(
-			struct ipa_config_req_msg_v01,
-			dl_accumulation_pkt_limit_valid),
-	},
-	{
-		.data_type	= QMI_SIGNED_4_BYTE_ENUM,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x16,
-		.offset		= offsetof(
-			struct ipa_config_req_msg_v01,
-			dl_accumulation_pkt_limit),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x17,
-		.offset		= offsetof(
-			struct ipa_config_req_msg_v01,
-			dl_accumulation_byte_limit_valid),
-	},
-	{
-		.data_type	= QMI_SIGNED_4_BYTE_ENUM,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x17,
-		.offset		= offsetof(
-			struct ipa_config_req_msg_v01,
-			dl_accumulation_byte_limit),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x18,
-		.offset		= offsetof(
-			struct ipa_config_req_msg_v01,
-			ul_accumulation_time_limit_valid),
-	},
-	{
-		.data_type	= QMI_SIGNED_4_BYTE_ENUM,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x18,
-		.offset		= offsetof(
-			struct ipa_config_req_msg_v01,
-			ul_accumulation_time_limit),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x19,
-		.offset		= offsetof(
-			struct ipa_config_req_msg_v01,
-			hw_control_flags_valid),
-	},
-	{
-		.data_type	= QMI_SIGNED_4_BYTE_ENUM,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x19,
-		.offset		= offsetof(
-			struct ipa_config_req_msg_v01,
-			hw_control_flags),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x1A,
-		.offset		= offsetof(
-			struct ipa_config_req_msg_v01,
-			ul_msi_event_threshold_valid),
-	},
-	{
-		.data_type	= QMI_SIGNED_4_BYTE_ENUM,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x1A,
-		.offset		= offsetof(
-			struct ipa_config_req_msg_v01,
-			ul_msi_event_threshold),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x1B,
-		.offset		= offsetof(
-			struct ipa_config_req_msg_v01,
-			dl_msi_event_threshold_valid),
-	},
-	{
-		.data_type	= QMI_SIGNED_4_BYTE_ENUM,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x1B,
-		.offset		= offsetof(
-			struct ipa_config_req_msg_v01,
-			dl_msi_event_threshold),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x1C,
-		.offset		= offsetof(
-			struct ipa_config_req_msg_v01,
-			ul_fifo_size_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x1C,
-		.offset		= offsetof(
-			struct ipa_config_req_msg_v01,
-			ul_fifo_size),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x1D,
-		.offset		= offsetof(
-			struct ipa_config_req_msg_v01,
-			dl_fifo_size_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x1D,
-		.offset		= offsetof(
-			struct ipa_config_req_msg_v01,
-			dl_fifo_size),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x1E,
-		.offset		= offsetof(
-			struct ipa_config_req_msg_v01,
-			dl_buf_size_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x1E,
-		.offset		= offsetof(
-			struct ipa_config_req_msg_v01,
-			dl_buf_size),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_config_resp_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(struct qmi_response_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x02,
-		.offset		= offsetof(
-			struct ipa_config_resp_msg_v01,
-			resp),
-		.ei_array	= qmi_response_type_v01_ei,
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_get_data_stats_req_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_SIGNED_4_BYTE_ENUM,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x01,
-		.offset		= offsetof(
-			struct ipa_get_data_stats_req_msg_v01,
-			ipa_stats_type),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_get_data_stats_req_msg_v01,
-			reset_stats_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_get_data_stats_req_msg_v01,
-			reset_stats),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-static struct qmi_elem_info ipa3_pipe_stats_info_type_data_v01_ei[] = {
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_pipe_stats_info_type_v01,
-					pipe_index),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_8_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint64_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_pipe_stats_info_type_v01,
-					num_ipv4_packets),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_8_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint64_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_pipe_stats_info_type_v01,
-					num_ipv4_bytes),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_8_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint64_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_pipe_stats_info_type_v01,
-					num_ipv6_packets),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_8_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint64_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_pipe_stats_info_type_v01,
-					num_ipv6_bytes),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-static struct qmi_elem_info ipa3_stats_type_filter_rule_data_v01_ei[] = {
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct
-					ipa_stats_type_filter_rule_v01,
-					filter_rule_index),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_8_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint64_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct
-					ipa_stats_type_filter_rule_v01,
-					num_packets),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_get_data_stats_resp_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(struct qmi_response_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x02,
-		.offset		= offsetof(
-			struct ipa_get_data_stats_resp_msg_v01,
-			resp),
-		.ei_array	= qmi_response_type_v01_ei,
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_get_data_stats_resp_msg_v01,
-			ipa_stats_type_valid),
-	},
-	{
-		.data_type	= QMI_SIGNED_4_BYTE_ENUM,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_get_data_stats_resp_msg_v01,
-			ipa_stats_type),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x11,
-		.offset		= offsetof(
-			struct ipa_get_data_stats_resp_msg_v01,
-			ul_src_pipe_stats_list_valid),
-	},
-	{
-		.data_type	= QMI_DATA_LEN,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x11,
-		.offset		= offsetof(
-			struct ipa_get_data_stats_resp_msg_v01,
-			ul_src_pipe_stats_list_len),
-	},
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= QMI_IPA_MAX_PIPES_V01,
-		.elem_size	= sizeof(struct ipa_pipe_stats_info_type_v01),
-		.array_type	= VAR_LEN_ARRAY,
-		.tlv_type	= 0x11,
-		.offset		= offsetof(
-			struct ipa_get_data_stats_resp_msg_v01,
-			ul_src_pipe_stats_list),
-		.ei_array	= ipa3_pipe_stats_info_type_data_v01_ei,
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x12,
-		.offset		= offsetof(
-			struct ipa_get_data_stats_resp_msg_v01,
-			dl_dst_pipe_stats_list_valid),
-	},
-	{
-		.data_type	= QMI_DATA_LEN,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x12,
-		.offset		= offsetof(
-			struct ipa_get_data_stats_resp_msg_v01,
-			dl_dst_pipe_stats_list_len),
-	},
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= QMI_IPA_MAX_PIPES_V01,
-		.elem_size	= sizeof(struct ipa_pipe_stats_info_type_v01),
-		.array_type	= VAR_LEN_ARRAY,
-		.tlv_type	= 0x12,
-		.offset		= offsetof(
-			struct ipa_get_data_stats_resp_msg_v01,
-			dl_dst_pipe_stats_list),
-		.ei_array	= ipa3_pipe_stats_info_type_data_v01_ei,
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x13,
-		.offset		= offsetof(
-			struct ipa_get_data_stats_resp_msg_v01,
-			dl_filter_rule_stats_list_valid),
-	},
-	{
-		.data_type	= QMI_DATA_LEN,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x13,
-		.offset		= offsetof(
-			struct ipa_get_data_stats_resp_msg_v01,
-			dl_filter_rule_stats_list_len),
-	},
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= QMI_IPA_MAX_FILTERS_V01,
-		.elem_size	= sizeof(struct ipa_pipe_stats_info_type_v01),
-		.array_type	= VAR_LEN_ARRAY,
-		.tlv_type	= 0x13,
-		.offset		= offsetof(
-			struct ipa_get_data_stats_resp_msg_v01,
-			dl_filter_rule_stats_list),
-		.ei_array	= ipa3_stats_type_filter_rule_data_v01_ei,
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-static struct qmi_elem_info ipa3_apn_data_stats_info_type_data_v01_ei[] = {
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct
-					ipa_apn_data_stats_info_type_v01,
-					mux_id),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_8_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint64_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct
-					ipa_apn_data_stats_info_type_v01,
-					num_ul_packets),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_8_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint64_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct
-					ipa_apn_data_stats_info_type_v01,
-					num_ul_bytes),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_8_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint64_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct
-					ipa_apn_data_stats_info_type_v01,
-					num_dl_packets),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_8_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint64_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct
-					ipa_apn_data_stats_info_type_v01,
-					num_dl_bytes),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_get_apn_data_stats_req_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_get_apn_data_stats_req_msg_v01,
-			mux_id_list_valid),
-	},
-	{
-		.data_type	= QMI_DATA_LEN,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_get_apn_data_stats_req_msg_v01,
-			mux_id_list_len),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= QMI_IPA_MAX_APN_V01,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= VAR_LEN_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_get_apn_data_stats_req_msg_v01,
-			mux_id_list),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_get_apn_data_stats_resp_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(struct qmi_response_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x02,
-		.offset		= offsetof(
-			struct ipa_get_apn_data_stats_resp_msg_v01,
-			resp),
-		.ei_array	= qmi_response_type_v01_ei,
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_get_apn_data_stats_resp_msg_v01,
-			apn_data_stats_list_valid),
-	},
-	{
-		.data_type	= QMI_DATA_LEN,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_get_apn_data_stats_resp_msg_v01,
-			apn_data_stats_list_len),
-	},
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= QMI_IPA_MAX_APN_V01,
-		.elem_size	= sizeof(struct
-					ipa_apn_data_stats_info_type_v01),
-		.array_type	= VAR_LEN_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_get_apn_data_stats_resp_msg_v01,
-			apn_data_stats_list),
-		.ei_array	= ipa3_apn_data_stats_info_type_data_v01_ei,
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-static struct qmi_elem_info ipa3_data_usage_quota_info_type_data_v01_ei[] = {
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct
-					ipa_data_usage_quota_info_type_v01,
-					mux_id),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_8_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint64_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct
-					ipa_data_usage_quota_info_type_v01,
-					num_Mbytes),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_set_data_usage_quota_req_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_set_data_usage_quota_req_msg_v01,
-			apn_quota_list_valid),
-	},
-	{
-		.data_type	= QMI_DATA_LEN,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_set_data_usage_quota_req_msg_v01,
-			apn_quota_list_len),
-	},
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= QMI_IPA_MAX_APN_V01,
-		.elem_size	= sizeof(struct
-					ipa_data_usage_quota_info_type_v01),
-		.array_type	= VAR_LEN_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_set_data_usage_quota_req_msg_v01,
-			apn_quota_list),
-		.ei_array	= ipa3_data_usage_quota_info_type_data_v01_ei,
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_set_data_usage_quota_resp_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(struct qmi_response_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x02,
-		.offset		= offsetof(
-			struct ipa_set_data_usage_quota_resp_msg_v01,
-			resp),
-		.ei_array	= qmi_response_type_v01_ei,
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_data_usage_quota_reached_ind_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(struct
-					ipa_data_usage_quota_info_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x01,
-		.offset		= offsetof(
-			struct ipa_data_usage_quota_reached_ind_msg_v01,
-			apn),
-		.ei_array	= ipa3_data_usage_quota_info_type_data_v01_ei,
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_stop_data_usage_quota_req_msg_data_v01_ei[] = {
-	/* ipa_stop_data_usage_quota_req_msg is empty */
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_stop_data_usage_quota_resp_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(struct qmi_response_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x02,
-		.offset		= offsetof(
-			struct ipa_stop_data_usage_quota_resp_msg_v01,
-			resp),
-		.ei_array	= qmi_response_type_v01_ei,
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_install_fltr_rule_req_ex_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_req_ex_msg_v01,
-			filter_spec_ex_list_valid),
-	},
-	{
-		.data_type	= QMI_DATA_LEN,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_req_ex_msg_v01,
-			filter_spec_ex_list_len),
-	},
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= QMI_IPA_MAX_FILTERS_EX_V01,
-		.elem_size	= sizeof(struct
-					ipa_filter_spec_ex_type_v01),
-		.array_type	= VAR_LEN_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_req_ex_msg_v01,
-			filter_spec_ex_list),
-		.ei_array	= ipa_filter_spec_ex_type_data_v01_ei,
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x11,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_req_ex_msg_v01,
-			source_pipe_index_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x11,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_req_ex_msg_v01,
-			source_pipe_index),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x12,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_req_ex_msg_v01,
-			num_ipv4_filters_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x12,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_req_ex_msg_v01,
-			num_ipv4_filters),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x13,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_req_ex_msg_v01,
-			num_ipv6_filters_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x13,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_req_ex_msg_v01,
-			num_ipv6_filters),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x14,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_req_ex_msg_v01,
-			xlat_filter_indices_list_valid),
-	},
-	{
-		.data_type	= QMI_DATA_LEN,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x14,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_req_ex_msg_v01,
-			xlat_filter_indices_list_len),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= QMI_IPA_MAX_FILTERS_EX_V01,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= VAR_LEN_ARRAY,
-		.tlv_type	= 0x14,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_req_ex_msg_v01,
-			xlat_filter_indices_list),
-	},
-		{
-		.data_type      = QMI_OPT_FLAG,
-		.elem_len       = 1,
-		.elem_size      = sizeof(uint8_t),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x15,
-		.offset         = offsetof(
-			struct ipa_install_fltr_rule_req_ex_msg_v01,
-			filter_spec_ex2_list_valid),
-	},
-	{
-		.data_type      = QMI_DATA_LEN,
-		.elem_len       = 1,
-		.elem_size      = sizeof(uint8_t),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x15,
-		.offset         = offsetof(
-			struct ipa_install_fltr_rule_req_ex_msg_v01,
-			filter_spec_ex2_list_len),
-	},
-	{
-		.data_type      = QMI_STRUCT,
-		.elem_len       = QMI_IPA_MAX_FILTERS_V01,
-		.elem_size      = sizeof(struct ipa_filter_spec_ex2_type_v01),
-		.array_type       = VAR_LEN_ARRAY,
-		.tlv_type       = 0x15,
-		.offset         = offsetof(
-			struct ipa_install_fltr_rule_req_ex_msg_v01,
-			filter_spec_ex2_list),
-		.ei_array      = ipa_filter_spec_ex2_type_v01_ei,
-	},
-	{
-		.data_type      = QMI_OPT_FLAG,
-		.elem_len       = 1,
-		.elem_size      = sizeof(uint8_t),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x16,
-		.offset         = offsetof(
-			struct ipa_install_fltr_rule_req_ex_msg_v01,
-			ul_firewall_indices_list_valid),
-	},
-	{
-		.data_type      = QMI_DATA_LEN,
-		.elem_len       = 1,
-		.elem_size      = sizeof(uint8_t),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x16,
-		.offset         = offsetof(
-			struct ipa_install_fltr_rule_req_ex_msg_v01,
-			ul_firewall_indices_list_len),
-	},
-	{
-		.data_type      = QMI_UNSIGNED_4_BYTE,
-		.elem_len       = QMI_IPA_MAX_FILTERS_V01,
-		.elem_size      = sizeof(uint32_t),
-		.array_type       = VAR_LEN_ARRAY,
-		.tlv_type       = 0x16,
-		.offset         = offsetof(
-			struct ipa_install_fltr_rule_req_ex_msg_v01,
-			ul_firewall_indices_list),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_install_fltr_rule_resp_ex_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(struct qmi_response_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x02,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_resp_ex_msg_v01,
-			resp),
-		.ei_array	= qmi_response_type_v01_ei,
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_resp_ex_msg_v01,
-			rule_id_valid),
-	},
-	{
-		.data_type	= QMI_DATA_LEN,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_resp_ex_msg_v01,
-			rule_id_len),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= QMI_IPA_MAX_FILTERS_EX_V01,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= VAR_LEN_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_install_fltr_rule_resp_ex_msg_v01,
-			rule_id),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_per_client_stats_info_type_data_v01_ei[] = {
-	{
-			.data_type	= QMI_UNSIGNED_4_BYTE,
-			.elem_len	= 1,
-			.elem_size	= sizeof(uint32_t),
-			.array_type	= NO_ARRAY,
-			.tlv_type	= QMI_COMMON_TLV_TYPE,
-			.offset		= offsetof(
-				struct ipa_per_client_stats_info_type_v01,
-				client_id),
-	},
-	{
-			.data_type	= QMI_UNSIGNED_4_BYTE,
-			.elem_len	= 1,
-			.elem_size	= sizeof(uint32_t),
-			.array_type	= NO_ARRAY,
-			.tlv_type	= QMI_COMMON_TLV_TYPE,
-			.offset		= offsetof(
-				struct ipa_per_client_stats_info_type_v01,
-				src_pipe_id),
-	},
-	{
-			.data_type	= QMI_UNSIGNED_8_BYTE,
-			.elem_len	= 1,
-			.elem_size	= sizeof(uint64_t),
-			.array_type	= NO_ARRAY,
-			.tlv_type	= QMI_COMMON_TLV_TYPE,
-			.offset		= offsetof(
-				struct ipa_per_client_stats_info_type_v01,
-				num_ul_ipv4_bytes),
-
-	},
-	{
-			.data_type	= QMI_UNSIGNED_8_BYTE,
-			.elem_len	= 1,
-			.elem_size	= sizeof(uint64_t),
-			.array_type	= NO_ARRAY,
-			.tlv_type	= QMI_COMMON_TLV_TYPE,
-			.offset		= offsetof(
-				struct ipa_per_client_stats_info_type_v01,
-				num_ul_ipv6_bytes),
-
-	},
-	{
-			.data_type	= QMI_UNSIGNED_8_BYTE,
-			.elem_len	= 1,
-			.elem_size	= sizeof(uint64_t),
-			.array_type	= NO_ARRAY,
-			.tlv_type	= QMI_COMMON_TLV_TYPE,
-			.offset		= offsetof(
-				struct ipa_per_client_stats_info_type_v01,
-				num_dl_ipv4_bytes),
-
-	},
-	{
-			.data_type	= QMI_UNSIGNED_8_BYTE,
-			.elem_len	= 1,
-			.elem_size	= sizeof(uint64_t),
-			.array_type	= NO_ARRAY,
-			.tlv_type	= QMI_COMMON_TLV_TYPE,
-			.offset		= offsetof(
-				struct ipa_per_client_stats_info_type_v01,
-				num_dl_ipv6_bytes),
-
-	},
-	{
-			.data_type	= QMI_UNSIGNED_4_BYTE,
-			.elem_len	= 1,
-			.elem_size	= sizeof(uint32_t),
-			.array_type	= NO_ARRAY,
-			.tlv_type	= QMI_COMMON_TLV_TYPE,
-			.offset		= offsetof(
-				struct ipa_per_client_stats_info_type_v01,
-				num_ul_ipv4_pkts),
-
-	},
-	{
-			.data_type	= QMI_UNSIGNED_4_BYTE,
-			.elem_len	= 1,
-			.elem_size	= sizeof(uint32_t),
-			.array_type	= NO_ARRAY,
-			.tlv_type	= QMI_COMMON_TLV_TYPE,
-			.offset		= offsetof(
-				struct ipa_per_client_stats_info_type_v01,
-				num_ul_ipv6_pkts),
-
-	},
-	{
-			.data_type	= QMI_UNSIGNED_4_BYTE,
-			.elem_len	= 1,
-			.elem_size	= sizeof(uint32_t),
-			.array_type	= NO_ARRAY,
-			.tlv_type	= QMI_COMMON_TLV_TYPE,
-			.offset		= offsetof(
-				struct ipa_per_client_stats_info_type_v01,
-				num_dl_ipv4_pkts),
-
-	},
-	{
-			.data_type	= QMI_UNSIGNED_4_BYTE,
-			.elem_len	= 1,
-			.elem_size	= sizeof(uint32_t),
-			.array_type	= NO_ARRAY,
-			.tlv_type	= QMI_COMMON_TLV_TYPE,
-			.offset		= offsetof(
-				struct ipa_per_client_stats_info_type_v01,
-				num_dl_ipv6_pkts),
-
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_ul_firewall_rule_type_data_v01_ei[] = {
-	{
-			.data_type	= QMI_UNSIGNED_4_BYTE,
-			.elem_len	= 1,
-			.elem_size	= sizeof(uint32_t),
-			.array_type	= NO_ARRAY,
-			.tlv_type	= QMI_COMMON_TLV_TYPE,
-			.offset		= offsetof(
-				struct ipa_ul_firewall_rule_type_v01,
-				ip_type),
-	},
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(struct ipa_filter_rule_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(struct ipa_ul_firewall_rule_type_v01,
-					filter_rule),
-		.ei_array	= ipa3_filter_rule_type_data_v01_ei,
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_ul_firewall_config_result_type_data_v01_ei[] = {
-	{
-			.data_type	= QMI_UNSIGNED_4_BYTE,
-			.elem_len	= 1,
-			.elem_size	= sizeof(uint32_t),
-			.array_type	= NO_ARRAY,
-			.tlv_type	= QMI_COMMON_TLV_TYPE,
-			.offset		= offsetof(
-				struct ipa_ul_firewall_config_result_type_v01,
-				is_success),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-		.offset		= offsetof(
-			struct ipa_ul_firewall_config_result_type_v01,
-			mux_id),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_enable_per_client_stats_req_msg_data_v01_ei[] = {
-	{
-				.data_type	= QMI_UNSIGNED_1_BYTE,
-				.elem_len	= 1,
-				.elem_size	= sizeof(uint8_t),
-				.array_type	= NO_ARRAY,
-				.tlv_type	= 0x01,
-				.offset		= offsetof(struct
-				ipa_enable_per_client_stats_req_msg_v01,
-				enable_per_client_stats),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_enable_per_client_stats_resp_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(struct qmi_response_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x02,
-		.offset		= offsetof(
-			struct ipa_enable_per_client_stats_resp_msg_v01,
-			resp),
-		.ei_array	= qmi_response_type_v01_ei,
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_get_stats_per_client_req_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x01,
-		.offset		= offsetof(
-			struct ipa_get_stats_per_client_req_msg_v01,
-			client_id),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x02,
-		.offset		= offsetof(
-			struct ipa_get_stats_per_client_req_msg_v01,
-			src_pipe_id),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_get_stats_per_client_req_msg_v01,
-			reset_stats_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_get_stats_per_client_req_msg_v01,
-			reset_stats),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_get_stats_per_client_resp_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(struct qmi_response_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x02,
-		.offset		= offsetof(
-			struct ipa_get_stats_per_client_resp_msg_v01,
-			resp),
-		.ei_array	= qmi_response_type_v01_ei,
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_get_stats_per_client_resp_msg_v01,
-			per_client_stats_list_valid),
-	},
-	{
-		.data_type	= QMI_DATA_LEN,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_get_stats_per_client_resp_msg_v01,
-			per_client_stats_list_len),
-	},
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= QMI_IPA_MAX_PER_CLIENTS_V01,
-		.elem_size	=
-			sizeof(struct ipa_per_client_stats_info_type_v01),
-		.array_type	= VAR_LEN_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_get_stats_per_client_resp_msg_v01,
-			per_client_stats_list),
-		.ei_array	=
-			ipa3_per_client_stats_info_type_data_v01_ei,
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_configure_ul_firewall_rules_req_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_DATA_LEN,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x1,
-		.offset		= offsetof(
-			struct ipa_configure_ul_firewall_rules_req_msg_v01,
-			firewall_rules_list_len),
-	},
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= QMI_IPA_MAX_UL_FIREWALL_RULES_V01,
-		.elem_size	= sizeof(struct ipa_ul_firewall_rule_type_v01),
-		.array_type	= VAR_LEN_ARRAY,
-		.tlv_type	= 0x1,
-		.offset		= offsetof(
-			struct ipa_configure_ul_firewall_rules_req_msg_v01,
-			firewall_rules_list),
-		.ei_array	=
-			ipa3_ul_firewall_rule_type_data_v01_ei,
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x2,
-		.offset		= offsetof(
-			struct ipa_configure_ul_firewall_rules_req_msg_v01,
-			mux_id),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_configure_ul_firewall_rules_req_msg_v01,
-			disable_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_configure_ul_firewall_rules_req_msg_v01,
-			disable),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x11,
-		.offset		= offsetof(
-			struct ipa_configure_ul_firewall_rules_req_msg_v01,
-			are_blacklist_filters_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_1_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x11,
-		.offset		= offsetof(
-			struct ipa_configure_ul_firewall_rules_req_msg_v01,
-			are_blacklist_filters),
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_configure_ul_firewall_rules_resp_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(struct qmi_response_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x02,
-		.offset		= offsetof(
-			struct ipa_configure_ul_firewall_rules_resp_msg_v01,
-			resp),
-		.ei_array	= qmi_response_type_v01_ei,
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa3_configure_ul_firewall_rules_ind_msg_data_v01_ei[] = {
-	{
-		.data_type	= QMI_STRUCT,
-		.elem_len	= 1,
-		.elem_size	= sizeof(
-			struct ipa_ul_firewall_config_result_type_v01),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x01,
-		.offset		= offsetof(
-			struct ipa_configure_ul_firewall_rules_ind_msg_v01,
-			result),
-		.ei_array	=
-		ipa3_ul_firewall_config_result_type_data_v01_ei,
-	},
-	{
-		.data_type	= QMI_EOTI,
-		.array_type	= NO_ARRAY,
-		.tlv_type	= QMI_COMMON_TLV_TYPE,
-	},
-};
-
-static struct qmi_elem_info ipa_mhi_ch_init_info_type_v01_ei[] = {
-	{
-		.data_type = QMI_UNSIGNED_1_BYTE,
-		.elem_len = 1,
-		.elem_size = sizeof(u8),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0,
-		.offset = offsetof(struct ipa_mhi_ch_init_info_type_v01,
-		ch_id),
-	},
-	{
-		.data_type = QMI_UNSIGNED_1_BYTE,
-		.elem_len = 1,
-		.elem_size = sizeof(u8),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0,
-		.offset = offsetof(struct ipa_mhi_ch_init_info_type_v01,
-		er_id),
-	},
-	{
-		.data_type = QMI_UNSIGNED_4_BYTE,
-		.elem_len = 1,
-		.elem_size = sizeof(u32),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0,
-		.offset = offsetof(struct ipa_mhi_ch_init_info_type_v01,
-		ch_doorbell_addr),
-	},
-	{
-		.data_type = QMI_UNSIGNED_4_BYTE,
-		.elem_len = 1,
-		.elem_size = sizeof(u32),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0,
-		.offset = offsetof(struct ipa_mhi_ch_init_info_type_v01,
-		er_doorbell_addr),
-	},
-	{
-		.data_type = QMI_UNSIGNED_4_BYTE,
-		.elem_len = 1,
-		.elem_size = sizeof(u32),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0,
-		.offset = offsetof(struct ipa_mhi_ch_init_info_type_v01,
-		direction_type),
-	},
-	{
-		.data_type = QMI_EOTI,
-		.array_type = NO_ARRAY,
-		.tlv_type = QMI_COMMON_TLV_TYPE,
-	},
-};
-
-static struct qmi_elem_info ipa_mhi_smmu_info_type_v01_ei[] = {
-	{
-		.data_type = QMI_UNSIGNED_8_BYTE,
-		.elem_len = 1,
-		.elem_size = sizeof(u64),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0,
-		.offset = offsetof(struct ipa_mhi_smmu_info_type_v01,
-		iova_ctl_base_addr),
-	},
-	{
-		.data_type = QMI_UNSIGNED_8_BYTE,
-		.elem_len = 1,
-		.elem_size = sizeof(u64),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0,
-		.offset = offsetof(struct ipa_mhi_smmu_info_type_v01,
-		iova_ctl_size),
-	},
-	{
-		.data_type = QMI_UNSIGNED_8_BYTE,
-		.elem_len = 1,
-		.elem_size = sizeof(u64),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0,
-		.offset = offsetof(struct ipa_mhi_smmu_info_type_v01,
-		iova_data_base_addr),
-	},
-	{
-		.data_type = QMI_UNSIGNED_8_BYTE,
-		.elem_len = 1,
-		.elem_size = sizeof(u64),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0,
-		.offset = offsetof(struct ipa_mhi_smmu_info_type_v01,
-		iova_data_size),
-	},
-	{
-		.data_type = QMI_EOTI,
-		.array_type = NO_ARRAY,
-		.tlv_type = QMI_COMMON_TLV_TYPE,
-	},
-};
-
-
-struct qmi_elem_info ipa_mhi_ready_indication_msg_v01_ei[] = {
-	{
-		.data_type = QMI_DATA_LEN,
-		.elem_len = 1,
-		.elem_size = sizeof(u8),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0x01,
-		.offset = offsetof(struct ipa_mhi_ready_indication_msg_v01,
-		ch_info_arr_len),
-	},
-	{
-		.data_type = QMI_STRUCT,
-		.elem_len = QMI_IPA_REMOTE_MHI_CHANNELS_NUM_MAX_V01,
-		.elem_size = sizeof(struct ipa_mhi_ch_init_info_type_v01),
-		.array_type = VAR_LEN_ARRAY,
-		.tlv_type = 0x01,
-		.offset = offsetof(struct ipa_mhi_ready_indication_msg_v01,
-		ch_info_arr),
-		.ei_array = ipa_mhi_ch_init_info_type_v01_ei,
-	},
-	{
-		.data_type = QMI_OPT_FLAG,
-		.elem_len = 1,
-		.elem_size = sizeof(u8),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0x10,
-		.offset = offsetof(struct ipa_mhi_ready_indication_msg_v01,
-		smmu_info_valid),
-	},
-	{
-		.data_type = QMI_STRUCT,
-		.elem_len = 1,
-		.elem_size = sizeof(struct ipa_mhi_smmu_info_type_v01),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0x10,
-		.offset = offsetof(struct ipa_mhi_ready_indication_msg_v01,
-		smmu_info),
-		.ei_array = ipa_mhi_smmu_info_type_v01_ei,
-	},
-	{
-		.data_type = QMI_EOTI,
-		.array_type = NO_ARRAY,
-		.tlv_type = QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa_mhi_mem_addr_info_type_v01_ei[] = {
-	{
-		.data_type = QMI_UNSIGNED_8_BYTE,
-		.elem_len = 1,
-		.elem_size = sizeof(u64),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0,
-		.offset = offsetof(struct ipa_mhi_mem_addr_info_type_v01,
-		pa),
-	},
-	{
-		.data_type = QMI_UNSIGNED_8_BYTE,
-		.elem_len = 1,
-		.elem_size = sizeof(u64),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0,
-		.offset = offsetof(struct ipa_mhi_mem_addr_info_type_v01,
-		iova),
-	},
-	{
-		.data_type = QMI_UNSIGNED_8_BYTE,
-		.elem_len = 1,
-		.elem_size = sizeof(u64),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0,
-		.offset = offsetof(struct ipa_mhi_mem_addr_info_type_v01,
-		size),
-	},
-	{
-		.data_type = QMI_EOTI,
-		.array_type = NO_ARRAY,
-		.tlv_type = QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa_mhi_tr_info_type_v01_ei[] = {
-	{
-		.data_type = QMI_UNSIGNED_1_BYTE,
-		.elem_len = 1,
-		.elem_size = sizeof(u8),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0,
-		.offset = offsetof(struct ipa_mhi_tr_info_type_v01,
-		ch_id),
-	},
-	{
-		.data_type = QMI_UNSIGNED_2_BYTE,
-		.elem_len = 1,
-		.elem_size = sizeof(u16),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0,
-		.offset = offsetof(struct ipa_mhi_tr_info_type_v01,
-		poll_cfg),
-	},
-	{
-		.data_type = QMI_SIGNED_4_BYTE_ENUM,
-		.elem_len = 1,
-		.elem_size = sizeof(enum ipa_mhi_brst_mode_enum_v01),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0,
-		.offset = offsetof(struct ipa_mhi_tr_info_type_v01,
-		brst_mode_type),
-	},
-	{
-		.data_type = QMI_UNSIGNED_8_BYTE,
-		.elem_len = 1,
-		.elem_size = sizeof(u64),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0,
-		.offset = offsetof(struct ipa_mhi_tr_info_type_v01,
-		ring_iova),
-	},
-	{
-		.data_type = QMI_UNSIGNED_8_BYTE,
-		.elem_len = 1,
-		.elem_size = sizeof(u64),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0,
-		.offset = offsetof(struct ipa_mhi_tr_info_type_v01,
-		ring_len),
-	},
-	{
-		.data_type = QMI_UNSIGNED_8_BYTE,
-		.elem_len = 1,
-		.elem_size = sizeof(u64),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0,
-		.offset = offsetof(struct ipa_mhi_tr_info_type_v01,
-		rp),
-	},
-	{
-		.data_type = QMI_UNSIGNED_8_BYTE,
-		.elem_len = 1,
-		.elem_size = sizeof(u64),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0,
-		.offset = offsetof(struct ipa_mhi_tr_info_type_v01,
-		wp),
-	},
-	{
-		.data_type = QMI_EOTI,
-		.array_type = NO_ARRAY,
-		.tlv_type = QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa_mhi_er_info_type_v01_ei[] = {
-	{
-		.data_type = QMI_UNSIGNED_1_BYTE,
-		.elem_len = 1,
-		.elem_size = sizeof(u8),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0,
-		.offset = offsetof(struct ipa_mhi_er_info_type_v01,
-		er_id),
-	},
-	{
-		.data_type = QMI_UNSIGNED_4_BYTE,
-		.elem_len = 1,
-		.elem_size = sizeof(u32),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0,
-		.offset = offsetof(struct ipa_mhi_er_info_type_v01,
-		intmod_cycles),
-	},
-	{
-		.data_type = QMI_UNSIGNED_4_BYTE,
-		.elem_len = 1,
-		.elem_size = sizeof(u32),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0,
-		.offset = offsetof(struct ipa_mhi_er_info_type_v01,
-		intmod_count),
-	},
-	{
-		.data_type = QMI_UNSIGNED_4_BYTE,
-		.elem_len = 1,
-		.elem_size = sizeof(u32),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0,
-		.offset = offsetof(struct ipa_mhi_er_info_type_v01,
-		msi_addr),
-	},
-	{
-		.data_type = QMI_UNSIGNED_8_BYTE,
-		.elem_len = 1,
-		.elem_size = sizeof(u64),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0,
-		.offset = offsetof(struct ipa_mhi_er_info_type_v01,
-		ring_iova),
-	},
-	{
-		.data_type = QMI_UNSIGNED_8_BYTE,
-		.elem_len = 1,
-		.elem_size = sizeof(u64),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0,
-		.offset = offsetof(struct ipa_mhi_er_info_type_v01,
-		ring_len),
-	},
-	{
-		.data_type = QMI_UNSIGNED_8_BYTE,
-		.elem_len = 1,
-		.elem_size = sizeof(u64),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0,
-		.offset = offsetof(struct ipa_mhi_er_info_type_v01,
-		rp),
-	},
-	{
-		.data_type = QMI_UNSIGNED_8_BYTE,
-		.elem_len = 1,
-		.elem_size = sizeof(u64),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0,
-		.offset = offsetof(struct ipa_mhi_er_info_type_v01,
-		wp),
-	},
-	{
-		.data_type = QMI_EOTI,
-		.array_type = NO_ARRAY,
-		.tlv_type = QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa_mhi_alloc_channel_req_msg_v01_ei[] = {
-	{
-		.data_type = QMI_DATA_LEN,
-		.elem_len = 1,
-		.elem_size = sizeof(u8),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0x01,
-		.offset = offsetof(struct ipa_mhi_alloc_channel_req_msg_v01,
-		tr_info_arr_len),
-	},
-	{
-		.data_type = QMI_STRUCT,
-		.elem_len = QMI_IPA_REMOTE_MHI_CHANNELS_NUM_MAX_V01,
-		.elem_size = sizeof(struct ipa_mhi_tr_info_type_v01),
-		.array_type = VAR_LEN_ARRAY,
-		.tlv_type = 0x01,
-		.offset = offsetof(struct ipa_mhi_alloc_channel_req_msg_v01,
-		tr_info_arr),
-		.ei_array = ipa_mhi_tr_info_type_v01_ei,
-	},
-	{
-		.data_type = QMI_DATA_LEN,
-		.elem_len = 1,
-		.elem_size = sizeof(u8),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0x02,
-		.offset = offsetof(struct ipa_mhi_alloc_channel_req_msg_v01,
-		er_info_arr_len),
-	},
-	{
-		.data_type = QMI_STRUCT,
-		.elem_len = QMI_IPA_REMOTE_MHI_CHANNELS_NUM_MAX_V01,
-		.elem_size = sizeof(struct ipa_mhi_er_info_type_v01),
-		.array_type = VAR_LEN_ARRAY,
-		.tlv_type = 0x02,
-		.offset = offsetof(struct ipa_mhi_alloc_channel_req_msg_v01,
-		er_info_arr),
-		.ei_array = ipa_mhi_er_info_type_v01_ei,
-	},
-	{
-		.data_type = QMI_DATA_LEN,
-		.elem_len = 1,
-		.elem_size = sizeof(u8),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0x03,
-		.offset = offsetof(struct ipa_mhi_alloc_channel_req_msg_v01,
-		ctrl_addr_map_info_len),
-	},
-	{
-		.data_type = QMI_STRUCT,
-		.elem_len = QMI_IPA_REMOTE_MHI_MEMORY_MAPPING_NUM_MAX_V01,
-		.elem_size = sizeof(struct ipa_mhi_mem_addr_info_type_v01),
-		.array_type = VAR_LEN_ARRAY,
-		.tlv_type = 0x03,
-		.offset = offsetof(struct ipa_mhi_alloc_channel_req_msg_v01,
-		ctrl_addr_map_info),
-		.ei_array = ipa_mhi_mem_addr_info_type_v01_ei,
-	},
-	{
-		.data_type = QMI_DATA_LEN,
-		.elem_len = 1,
-		.elem_size = sizeof(u8),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0x04,
-		.offset = offsetof(struct ipa_mhi_alloc_channel_req_msg_v01,
-		data_addr_map_info_len),
-	},
-	{
-		.data_type = QMI_STRUCT,
-		.elem_len = QMI_IPA_REMOTE_MHI_MEMORY_MAPPING_NUM_MAX_V01,
-		.elem_size = sizeof(struct ipa_mhi_mem_addr_info_type_v01),
-		.array_type = VAR_LEN_ARRAY,
-		.tlv_type = 0x04,
-		.offset = offsetof(struct ipa_mhi_alloc_channel_req_msg_v01,
-		data_addr_map_info),
-		.ei_array = ipa_mhi_mem_addr_info_type_v01_ei,
-	},
-	{
-		.data_type = QMI_EOTI,
-		.array_type = NO_ARRAY,
-		.tlv_type = QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa_mhi_ch_alloc_resp_type_v01_ei[] = {
-	{
-		.data_type = QMI_UNSIGNED_1_BYTE,
-		.elem_len = 1,
-		.elem_size = sizeof(u8),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0,
-		.offset = offsetof(struct ipa_mhi_ch_alloc_resp_type_v01,
-		ch_id),
-	},
-	{
-		.data_type = QMI_UNSIGNED_1_BYTE,
-		.elem_len = 1,
-		.elem_size = sizeof(u8),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0,
-		.offset = offsetof(struct ipa_mhi_ch_alloc_resp_type_v01,
-		is_success),
-	},
-	{
-		.data_type = QMI_EOTI,
-		.array_type = NO_ARRAY,
-		.tlv_type = QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa_mhi_alloc_channel_resp_msg_v01_ei[] = {
-	{
-		.data_type = QMI_STRUCT,
-		.elem_len = 1,
-		.elem_size = sizeof(struct qmi_response_type_v01),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0x02,
-		.offset = offsetof(struct ipa_mhi_alloc_channel_resp_msg_v01,
-		resp),
-		.ei_array = qmi_response_type_v01_ei,
-	},
-	{
-		.data_type = QMI_OPT_FLAG,
-		.elem_len = 1,
-		.elem_size = sizeof(u8),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0x10,
-		.offset = offsetof(struct ipa_mhi_alloc_channel_resp_msg_v01,
-		alloc_resp_arr_valid),
-	},
-	{
-		.data_type = QMI_DATA_LEN,
-		.elem_len = 1,
-		.elem_size = sizeof(u8),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0x10,
-		.offset = offsetof(struct ipa_mhi_alloc_channel_resp_msg_v01,
-		alloc_resp_arr_len),
-	},
-	{
-		.data_type = QMI_STRUCT,
-		.elem_len = QMI_IPA_REMOTE_MHI_CHANNELS_NUM_MAX_V01,
-		.elem_size = sizeof(struct ipa_mhi_ch_alloc_resp_type_v01),
-		.array_type = VAR_LEN_ARRAY,
-		.tlv_type = 0x10,
-		.offset = offsetof(struct ipa_mhi_alloc_channel_resp_msg_v01,
-		alloc_resp_arr),
-		.ei_array = ipa_mhi_ch_alloc_resp_type_v01_ei,
-	},
-	{
-		.data_type = QMI_EOTI,
-		.array_type = NO_ARRAY,
-		.tlv_type = QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa_mhi_clk_vote_req_msg_v01_ei[] = {
-	{
-		.data_type = QMI_UNSIGNED_1_BYTE,
-		.elem_len = 1,
-		.elem_size = sizeof(u8),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0x01,
-		.offset = offsetof(struct ipa_mhi_clk_vote_req_msg_v01,
-		mhi_vote),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_mhi_clk_vote_req_msg_v01,
-			tput_value_valid),
-	},
-	{
-		.data_type	= QMI_UNSIGNED_4_BYTE,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x10,
-		.offset		= offsetof(
-			struct ipa_mhi_clk_vote_req_msg_v01,
-			tput_value),
-	},
-	{
-		.data_type	= QMI_OPT_FLAG,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint8_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x11,
-		.offset		= offsetof(
-			struct ipa_mhi_clk_vote_req_msg_v01,
-			clk_rate_valid),
-	},
-	{
-		.data_type	= QMI_SIGNED_4_BYTE_ENUM,
-		.elem_len	= 1,
-		.elem_size	= sizeof(uint32_t),
-		.array_type	= NO_ARRAY,
-		.tlv_type	= 0x11,
-		.offset		= offsetof(
-			struct ipa_mhi_clk_vote_req_msg_v01,
-			clk_rate),
-	},
-	{
-		.data_type = QMI_EOTI,
-		.array_type = NO_ARRAY,
-		.tlv_type = QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa_mhi_clk_vote_resp_msg_v01_ei[] = {
-	{
-		.data_type = QMI_STRUCT,
-		.elem_len = 1,
-		.elem_size = sizeof(struct qmi_response_type_v01),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0x02,
-		.offset = offsetof(struct ipa_mhi_clk_vote_resp_msg_v01,
-		resp),
-		.ei_array = qmi_response_type_v01_ei,
-	},
-	{
-		.data_type = QMI_EOTI,
-		.array_type = NO_ARRAY,
-		.tlv_type = QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa_mhi_cleanup_req_msg_v01_ei[] = {
-	{
-		.data_type = QMI_OPT_FLAG,
-		.elem_len = 1,
-		.elem_size = sizeof(u8),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0x10,
-		.offset = offsetof(struct ipa_mhi_cleanup_req_msg_v01,
-		cleanup_valid),
-	},
-	{
-		.data_type = QMI_UNSIGNED_1_BYTE,
-		.elem_len = 1,
-		.elem_size = sizeof(u8),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0x10,
-		.offset = offsetof(struct ipa_mhi_cleanup_req_msg_v01,
-		cleanup),
-	},
-	{
-		.data_type = QMI_EOTI,
-		.array_type = NO_ARRAY,
-		.tlv_type = QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa_mhi_cleanup_resp_msg_v01_ei[] = {
-	{
-		.data_type = QMI_STRUCT,
-		.elem_len = 1,
-		.elem_size = sizeof(struct qmi_response_type_v01),
-		.array_type = NO_ARRAY,
-		.tlv_type = 0x02,
-		.offset = offsetof(struct ipa_mhi_cleanup_resp_msg_v01,
-		resp),
-		.ei_array = qmi_response_type_v01_ei,
-	},
-	{
-		.data_type = QMI_EOTI,
-		.array_type = NO_ARRAY,
-		.tlv_type = QMI_COMMON_TLV_TYPE,
-	},
-};
-
-static struct qmi_elem_info ipa_ep_id_type_v01_ei[] = {
-	{
-		.data_type      = QMI_SIGNED_4_BYTE_ENUM,
-		.elem_len       = 1,
-		.elem_size      = sizeof(enum ipa_ic_type_enum_v01),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_ep_id_type_v01,
-					   ic_type),
-	},
-	{
-		.data_type      = QMI_SIGNED_4_BYTE_ENUM,
-		.elem_len       = 1,
-		.elem_size      = sizeof(enum ipa_ep_desc_type_enum_v01),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_ep_id_type_v01,
-					   ep_type),
-	},
-	{
-		.data_type      = QMI_UNSIGNED_4_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u32),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_ep_id_type_v01,
-					   ep_id),
-	},
-	{
-		.data_type      = QMI_SIGNED_4_BYTE_ENUM,
-		.elem_len       = 1,
-		.elem_size      = sizeof(enum ipa_ep_status_type_v01),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct ipa_ep_id_type_v01,
-					   ep_status),
-	},
-	{
-		.data_type      = QMI_EOTI,
-		.array_type       = NO_ARRAY,
-		.tlv_type       = QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa_endp_desc_indication_msg_v01_ei[] = {
-	{
-		.data_type      = QMI_OPT_FLAG,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x10,
-		.offset         = offsetof(
-			struct ipa_endp_desc_indication_msg_v01,
-			ep_info_valid),
-	},
-	{
-		.data_type      = QMI_DATA_LEN,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x10,
-		.offset         = offsetof(
-			struct ipa_endp_desc_indication_msg_v01,
-			ep_info_len),
-	},
-	{
-		.data_type      = QMI_STRUCT,
-		.elem_len       = QMI_IPA_ENDP_DESC_NUM_MAX_V01,
-		.elem_size      = sizeof(struct ipa_ep_id_type_v01),
-		.array_type       = VAR_LEN_ARRAY,
-		.tlv_type       = 0x10,
-		.offset         = offsetof(
-			struct ipa_endp_desc_indication_msg_v01,
-			ep_info),
-		.ei_array      = ipa_ep_id_type_v01_ei,
-	},
-	{
-		.data_type      = QMI_OPT_FLAG,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x11,
-		.offset         = offsetof(
-			struct ipa_endp_desc_indication_msg_v01,
-			num_eps_valid),
-	},
-	{
-		.data_type      = QMI_UNSIGNED_4_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u32),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x11,
-		.offset         = offsetof(
-			struct ipa_endp_desc_indication_msg_v01,
-			num_eps),
-	},
-	{
-		.data_type      = QMI_EOTI,
-		.array_type       = NO_ARRAY,
-		.tlv_type       = QMI_COMMON_TLV_TYPE,
-	},
-};
-
-static struct qmi_elem_info ipa_mhi_prime_aggr_info_type_v01_ei[] = {
-	{
-		.data_type      = QMI_SIGNED_4_BYTE_ENUM,
-		.elem_len       = 1,
-		.elem_size      = sizeof(enum ipa_ic_type_enum_v01),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(
-			struct ipa_mhi_prime_aggr_info_type_v01,
-			ic_type),
-	},
-	{
-		.data_type      = QMI_SIGNED_4_BYTE_ENUM,
-		.elem_len       = 1,
-		.elem_size      = sizeof(enum ipa_ep_desc_type_enum_v01),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(
-			struct ipa_mhi_prime_aggr_info_type_v01,
-			ep_type),
-	},
-	{
-		.data_type      = QMI_UNSIGNED_4_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u32),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(
-			struct ipa_mhi_prime_aggr_info_type_v01,
-			bytes_count),
-	},
-	{
-		.data_type      = QMI_UNSIGNED_4_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u32),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(
-			struct ipa_mhi_prime_aggr_info_type_v01,
-			pkt_count),
-	},
-	{
-		.data_type      = QMI_SIGNED_4_BYTE_ENUM,
-		.elem_len       = 1,
-		.elem_size      = sizeof(enum ipa_aggr_enum_type_v01),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(
-			struct ipa_mhi_prime_aggr_info_type_v01,
-			aggr_type),
-	},
-	{
-		.data_type      = QMI_EOTI,
-		.array_type       = NO_ARRAY,
-		.tlv_type       = QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa_mhi_prime_aggr_info_req_msg_v01_ei[] = {
-	{
-		.data_type      = QMI_OPT_FLAG,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x10,
-		.offset         = offsetof(
-			struct ipa_mhi_prime_aggr_info_req_msg_v01,
-			aggr_info_valid),
-	},
-	{
-		.data_type      = QMI_DATA_LEN,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x10,
-		.offset         = offsetof(
-			struct ipa_mhi_prime_aggr_info_req_msg_v01,
-			aggr_info_len),
-	},
-	{
-		.data_type      = QMI_STRUCT,
-		.elem_len       = QMI_IPA_ENDP_DESC_NUM_MAX_V01,
-		.elem_size      = sizeof(
-			struct ipa_mhi_prime_aggr_info_type_v01),
-		.array_type       = VAR_LEN_ARRAY,
-		.tlv_type       = 0x10,
-		.offset         = offsetof(
-			struct ipa_mhi_prime_aggr_info_req_msg_v01,
-			aggr_info),
-		.ei_array      = ipa_mhi_prime_aggr_info_type_v01_ei,
-	},
-	{
-		.data_type      = QMI_OPT_FLAG,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x11,
-		.offset         = offsetof(
-			struct ipa_mhi_prime_aggr_info_req_msg_v01,
-			num_eps_valid),
-	},
-	{
-		.data_type      = QMI_UNSIGNED_4_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u32),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x11,
-		.offset         = offsetof(
-			struct ipa_mhi_prime_aggr_info_req_msg_v01,
-			num_eps),
-	},
-	{
-		.data_type      = QMI_EOTI,
-		.array_type       = NO_ARRAY,
-		.tlv_type       = QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa_mhi_prime_aggr_info_resp_msg_v01_ei[] = {
-	{
-		.data_type      = QMI_STRUCT,
-		.elem_len       = 1,
-		.elem_size      = sizeof(struct qmi_response_type_v01),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x02,
-		.offset         = offsetof(
-			struct ipa_mhi_prime_aggr_info_resp_msg_v01,
-			resp),
-		.ei_array      = qmi_response_type_v01_ei,
-	},
-	{
-		.data_type      = QMI_EOTI,
-		.array_type       = NO_ARRAY,
-		.tlv_type       = QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa_add_offload_connection_req_msg_v01_ei[] = {
-	{
-		.data_type      = QMI_OPT_FLAG,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x10,
-		.offset         = offsetof(
-			struct ipa_add_offload_connection_req_msg_v01,
-			num_ipv4_filters_valid),
-	},
-	{
-		.data_type      = QMI_UNSIGNED_4_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u32),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x10,
-		.offset         = offsetof(
-			struct ipa_add_offload_connection_req_msg_v01,
-			num_ipv4_filters),
-	},
-	{
-		.data_type      = QMI_OPT_FLAG,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x11,
-		.offset         = offsetof(
-			struct ipa_add_offload_connection_req_msg_v01,
-			num_ipv6_filters_valid),
-	},
-	{
-		.data_type      = QMI_UNSIGNED_4_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u32),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x11,
-		.offset         = offsetof(
-			struct ipa_add_offload_connection_req_msg_v01,
-			num_ipv6_filters),
-	},
-	{
-		.data_type      = QMI_OPT_FLAG,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x12,
-		.offset         = offsetof(
-			struct ipa_add_offload_connection_req_msg_v01,
-			xlat_filter_indices_list_valid),
-	},
-	{
-		.data_type      = QMI_DATA_LEN,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x12,
-		.offset         = offsetof(
-			struct ipa_add_offload_connection_req_msg_v01,
-			xlat_filter_indices_list_len),
-	},
-	{
-		.data_type      = QMI_UNSIGNED_4_BYTE,
-		.elem_len       = QMI_IPA_MAX_FILTERS_V01,
-		.elem_size      = sizeof(u32),
-		.array_type       = VAR_LEN_ARRAY,
-		.tlv_type       = 0x12,
-		.offset         = offsetof(
-			struct ipa_add_offload_connection_req_msg_v01,
-			xlat_filter_indices_list),
-	},
-	{
-		.data_type      = QMI_OPT_FLAG,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x13,
-		.offset         = offsetof(
-			struct ipa_add_offload_connection_req_msg_v01,
-			filter_spec_ex2_list_valid),
-	},
-	{
-		.data_type      = QMI_DATA_LEN,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x13,
-		.offset         = offsetof(
-			struct ipa_add_offload_connection_req_msg_v01,
-			filter_spec_ex2_list_len),
-	},
-	{
-		.data_type      = QMI_STRUCT,
-		.elem_len       = QMI_IPA_MAX_FILTERS_V01,
-		.elem_size      = sizeof(struct ipa_filter_spec_ex2_type_v01),
-		.array_type       = VAR_LEN_ARRAY,
-		.tlv_type       = 0x13,
-		.offset         = offsetof(
-			struct ipa_add_offload_connection_req_msg_v01,
-			filter_spec_ex2_list),
-		.ei_array      = ipa_filter_spec_ex2_type_v01_ei,
-	},
-	{
-		.data_type      = QMI_OPT_FLAG,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x14,
-		.offset         = offsetof(
-			struct ipa_add_offload_connection_req_msg_v01,
-			embedded_call_mux_id_valid),
-	},
-	{
-		.data_type      = QMI_UNSIGNED_4_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(uint32_t),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x14,
-		.offset         = offsetof(
-			struct ipa_add_offload_connection_req_msg_v01,
-			embedded_call_mux_id),
-	},
-	{
-		.data_type      = QMI_OPT_FLAG,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x15,
-		.offset         = offsetof(
-			struct ipa_add_offload_connection_req_msg_v01,
-			default_mhi_path_valid),
-	},
-	{
-		.data_type      = QMI_UNSIGNED_1_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x15,
-		.offset         = offsetof(
-			struct ipa_add_offload_connection_req_msg_v01,
-			default_mhi_path),
-	},
-	{
-		.data_type      = QMI_EOTI,
-		.array_type       = NO_ARRAY,
-		.tlv_type       = QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa_add_offload_connection_resp_msg_v01_ei[] = {
-	{
-		.data_type      = QMI_STRUCT,
-		.elem_len       = 1,
-		.elem_size      = sizeof(struct qmi_response_type_v01),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x02,
-		.offset         = offsetof(
-			struct ipa_add_offload_connection_resp_msg_v01,
-			resp),
-		.ei_array      = qmi_response_type_v01_ei,
-	},
-	{
-		.data_type      = QMI_OPT_FLAG,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x10,
-		.offset         = offsetof(
-			struct ipa_add_offload_connection_resp_msg_v01,
-			filter_handle_list_valid),
-	},
-	{
-		.data_type      = QMI_DATA_LEN,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x10,
-		.offset         = offsetof(
-			struct ipa_add_offload_connection_resp_msg_v01,
-			filter_handle_list_len),
-	},
-	{
-		.data_type      = QMI_STRUCT,
-		.elem_len       = QMI_IPA_MAX_FILTERS_V01,
-		.elem_size      = sizeof(
-			struct ipa_filter_rule_identifier_to_handle_map_v01),
-		.array_type       = VAR_LEN_ARRAY,
-		.tlv_type       = 0x10,
-		.offset         = offsetof(
-			struct ipa_add_offload_connection_resp_msg_v01,
-			filter_handle_list),
-		.ei_array      =
-			ipa3_filter_rule_identifier_to_handle_map_data_v01_ei,
-	},
-	{
-		.data_type      = QMI_EOTI,
-		.array_type       = NO_ARRAY,
-		.tlv_type       = QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa_remove_offload_connection_req_msg_v01_ei[] = {
-	{
-		.data_type      = QMI_OPT_FLAG,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x10,
-		.offset         = offsetof(
-			struct ipa_remove_offload_connection_req_msg_v01,
-			filter_handle_list_valid),
-	},
-	{
-		.data_type      = QMI_DATA_LEN,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x10,
-		.offset         = offsetof(
-			struct ipa_remove_offload_connection_req_msg_v01,
-			filter_handle_list_len),
-	},
-	{
-		.data_type      = QMI_STRUCT,
-		.elem_len       = QMI_IPA_MAX_FILTERS_V01,
-		.elem_size      = sizeof(
-			struct ipa_filter_rule_identifier_to_handle_map_v01),
-		.array_type       = VAR_LEN_ARRAY,
-		.tlv_type       = 0x10,
-		.offset         = offsetof(
-			struct ipa_remove_offload_connection_req_msg_v01,
-			filter_handle_list),
-		.ei_array      =
-			ipa3_filter_rule_identifier_to_handle_map_data_v01_ei,
-	},
-	{
-		.data_type      = QMI_OPT_FLAG,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x11,
-		.offset         = offsetof(
-			struct ipa_remove_offload_connection_req_msg_v01,
-			clean_all_rules_valid),
-	},
-	{
-		.data_type      = QMI_UNSIGNED_1_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x11,
-		.offset         = offsetof(
-			struct ipa_remove_offload_connection_req_msg_v01,
-			clean_all_rules),
-	},
-	{
-		.data_type      = QMI_EOTI,
-		.array_type       = NO_ARRAY,
-		.tlv_type       = QMI_COMMON_TLV_TYPE,
-	},
-};
-
-struct qmi_elem_info ipa_remove_offload_connection_resp_msg_v01_ei[] = {
-	{
-		.data_type      = QMI_OPT_FLAG,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x02,
-		.offset         = offsetof(
-			struct ipa_remove_offload_connection_resp_msg_v01,
-			resp_valid),
-	},
-	{
-		.data_type      = QMI_STRUCT,
-		.elem_len       = 1,
-		.elem_size      = sizeof(struct qmi_response_type_v01),
-		.array_type       = NO_ARRAY,
-		.tlv_type       = 0x02,
-		.offset         = offsetof(
-			struct ipa_remove_offload_connection_resp_msg_v01,
-			resp),
-		.ei_array      = qmi_response_type_v01_ei,
-	},
-	{
-		.data_type      = QMI_EOTI,
-		.array_type       = NO_ARRAY,
-		.tlv_type       = QMI_COMMON_TLV_TYPE,
-	},
-};

ipa/ipa_v3/ipa_rt.c

@@ -1,2491 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2012-2019, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/bitops.h>
-#include <linux/idr.h>
-#include "ipa_i.h"
-#include "ipahal/ipahal.h"
-#include "ipahal/ipahal_fltrt.h"
-
-#define IPA_RT_INDEX_BITMAP_SIZE	(32)
-#define IPA_RT_STATUS_OF_ADD_FAILED	(-1)
-#define IPA_RT_STATUS_OF_DEL_FAILED	(-1)
-#define IPA_RT_STATUS_OF_MDFY_FAILED (-1)
-
-#define IPA_RT_MAX_NUM_OF_COMMIT_TABLES_CMD_DESC 5
-
-#define IPA_RT_GET_RULE_TYPE(__entry) \
-	( \
-	((__entry)->rule.hashable) ? \
-	(IPA_RULE_HASHABLE) : (IPA_RULE_NON_HASHABLE) \
-	)
-
-/**
- * ipa_generate_rt_hw_rule() - Generated the RT H/W single rule
- *  This func will do the preparation core driver work and then calls
- *  the HAL layer for the real work.
- * @ip: the ip address family type
- * @entry: routing entry
- * @buf: output buffer, buf == NULL means
- *	caller wants to know the size of the rule as seen
- *	by HW so they did not pass a valid buffer, we will use a
- *	scratch buffer instead.
- *	With this scheme we are going to
- *	generate the rule twice, once to know size using scratch
- *	buffer and second to write the rule to the actual caller
- *	supplied buffer which is of required size
- *
- * Returns: 0 on success, negative on failure
- *
- * caller needs to hold any needed locks to ensure integrity
- */
-static int ipa_generate_rt_hw_rule(enum ipa_ip_type ip,
-	struct ipa3_rt_entry *entry, u8 *buf)
-{
-	struct ipahal_rt_rule_gen_params gen_params;
-	struct ipa3_hdr_entry *hdr_entry;
-	struct ipa3_hdr_proc_ctx_entry *hdr_proc_entry;
-	int res = 0;
-
-	memset(&gen_params, 0, sizeof(gen_params));
-
-	if (entry->rule.hashable &&
-		entry->rule.attrib.attrib_mask & IPA_FLT_IS_PURE_ACK) {
-		IPAERR_RL("PURE_ACK rule atrb used with hash rule\n");
-		WARN_ON_RATELIMIT_IPA(1);
-		return -EPERM;
-	}
-
-	gen_params.ipt = ip;
-	gen_params.dst_pipe_idx = ipa3_get_ep_mapping(entry->rule.dst);
-	if (gen_params.dst_pipe_idx == -1) {
-		IPAERR_RL("Wrong destination pipe specified in RT rule\n");
-		WARN_ON_RATELIMIT_IPA(1);
-		return -EPERM;
-	}
-	if (!IPA_CLIENT_IS_CONS(entry->rule.dst)) {
-		IPAERR_RL("No RT rule on IPA_client_producer pipe.\n");
-		IPAERR_RL("pipe_idx: %d dst_pipe: %d\n",
-				gen_params.dst_pipe_idx, entry->rule.dst);
-		WARN_ON_RATELIMIT_IPA(1);
-		return -EPERM;
-	}
-
-	/* Adding check to confirm still
-	 * header entry present in header table or not
-	 */
-
-	if (entry->hdr) {
-		hdr_entry = ipa3_id_find(entry->rule.hdr_hdl);
-		if (!hdr_entry || hdr_entry->cookie != IPA_HDR_COOKIE) {
-			IPAERR_RL("Header entry already deleted\n");
-			return -EPERM;
-		}
-	} else if (entry->proc_ctx) {
-		hdr_proc_entry = ipa3_id_find(entry->rule.hdr_proc_ctx_hdl);
-		if (!hdr_proc_entry ||
-			hdr_proc_entry->cookie != IPA_PROC_HDR_COOKIE) {
-			IPAERR_RL("Proc header entry already deleted\n");
-			return -EPERM;
-		}
-	}
-
-	if (entry->proc_ctx || (entry->hdr && entry->hdr->is_hdr_proc_ctx)) {
-		struct ipa3_hdr_proc_ctx_entry *proc_ctx;
-
-		proc_ctx = (entry->proc_ctx) ? : entry->hdr->proc_ctx;
-		if ((proc_ctx == NULL) ||
-			(proc_ctx->cookie != IPA_PROC_HDR_COOKIE)) {
-			gen_params.hdr_type = IPAHAL_RT_RULE_HDR_NONE;
-			gen_params.hdr_ofst = 0;
-		} else {
-			gen_params.hdr_lcl = ipa3_ctx->hdr_proc_ctx_tbl_lcl;
-			gen_params.hdr_type = IPAHAL_RT_RULE_HDR_PROC_CTX;
-			gen_params.hdr_ofst = proc_ctx->offset_entry->offset +
-				ipa3_ctx->hdr_proc_ctx_tbl.start_offset;
-		}
-	} else if ((entry->hdr != NULL) &&
-		(entry->hdr->cookie == IPA_HDR_COOKIE)) {
-		gen_params.hdr_lcl = ipa3_ctx->hdr_tbl_lcl;
-		gen_params.hdr_type = IPAHAL_RT_RULE_HDR_RAW;
-		gen_params.hdr_ofst = entry->hdr->offset_entry->offset;
-	} else {
-		gen_params.hdr_type = IPAHAL_RT_RULE_HDR_NONE;
-		gen_params.hdr_ofst = 0;
-	}
-
-	gen_params.priority = entry->prio;
-	gen_params.id = entry->rule_id;
-	gen_params.rule = (const struct ipa_rt_rule_i *)&entry->rule;
-	gen_params.cnt_idx = entry->cnt_idx;
-
-	res = ipahal_rt_generate_hw_rule(&gen_params, &entry->hw_len, buf);
-	if (res)
-		IPAERR("failed to generate rt h/w rule\n");
-
-	return res;
-}
-
-/**
- * ipa_translate_rt_tbl_to_hw_fmt() - translate the routing driver structures
- *  (rules and tables) to HW format and fill it in the given buffers
- * @ip: the ip address family type
- * @rlt: the type of the rules to translate (hashable or non-hashable)
- * @base: the rules body buffer to be filled
- * @hdr: the rules header (addresses/offsets) buffer to be filled
- * @body_ofst: the offset of the rules body from the rules header at
- *  ipa sram (for local body usage)
- * @apps_start_idx: the first rt table index of apps tables
- *
- * Returns: 0 on success, negative on failure
- *
- * caller needs to hold any needed locks to ensure integrity
- *
- */
-static int ipa_translate_rt_tbl_to_hw_fmt(enum ipa_ip_type ip,
-	enum ipa_rule_type rlt, u8 *base, u8 *hdr,
-	u32 body_ofst, u32 apps_start_idx)
-{
-	struct ipa3_rt_tbl_set *set;
-	struct ipa3_rt_tbl *tbl;
-	struct ipa_mem_buffer tbl_mem;
-	u8 *tbl_mem_buf;
-	struct ipa3_rt_entry *entry;
-	int res;
-	u64 offset;
-	u8 *body_i;
-
-	set = &ipa3_ctx->rt_tbl_set[ip];
-	body_i = base;
-	list_for_each_entry(tbl, &set->head_rt_tbl_list, link) {
-		if (tbl->sz[rlt] == 0)
-			continue;
-		if (tbl->in_sys[rlt]) {
-			/* only body (no header) */
-			tbl_mem.size = tbl->sz[rlt] -
-				ipahal_get_hw_tbl_hdr_width();
-			if (ipahal_fltrt_allocate_hw_sys_tbl(&tbl_mem)) {
-				IPAERR_RL("fail to alloc sys tbl of size %d\n",
-					tbl_mem.size);
-				goto err;
-			}
-
-			if (ipahal_fltrt_write_addr_to_hdr(tbl_mem.phys_base,
-				hdr, tbl->idx - apps_start_idx, true)) {
-				IPAERR_RL("fail to wrt sys tbl addr to hdr\n");
-				goto hdr_update_fail;
-			}
-
-			tbl_mem_buf = tbl_mem.base;
-
-			/* generate the rule-set */
-			list_for_each_entry(entry, &tbl->head_rt_rule_list,
-					link) {
-				if (IPA_RT_GET_RULE_TYPE(entry) != rlt)
-					continue;
-				res = ipa_generate_rt_hw_rule(ip, entry,
-					tbl_mem_buf);
-				if (res) {
-					IPAERR_RL("failed to gen HW RT rule\n");
-					goto hdr_update_fail;
-				}
-				tbl_mem_buf += entry->hw_len;
-			}
-
-			if (tbl->curr_mem[rlt].phys_base) {
-				WARN_ON(tbl->prev_mem[rlt].phys_base);
-				tbl->prev_mem[rlt] = tbl->curr_mem[rlt];
-			}
-			tbl->curr_mem[rlt] = tbl_mem;
-		} else {
-			offset = body_i - base + body_ofst;
-
-			/* update the hdr at the right index */
-			if (ipahal_fltrt_write_addr_to_hdr(offset, hdr,
-				tbl->idx - apps_start_idx, true)) {
-				IPAERR_RL("fail to wrt lcl tbl ofst to hdr\n");
-				goto hdr_update_fail;
-			}
-
-			/* generate the rule-set */
-			list_for_each_entry(entry, &tbl->head_rt_rule_list,
-					link) {
-				if (IPA_RT_GET_RULE_TYPE(entry) != rlt)
-					continue;
-				res = ipa_generate_rt_hw_rule(ip, entry,
-					body_i);
-				if (res) {
-					IPAERR_RL("failed to gen HW RT rule\n");
-					goto err;
-				}
-				body_i += entry->hw_len;
-			}
-
-			/**
-			 * advance body_i to next table alignment as local
-			 * tables
-			 * are order back-to-back
-			 */
-			body_i += ipahal_get_lcl_tbl_addr_alignment();
-			body_i = (u8 *)((long)body_i &
-				~ipahal_get_lcl_tbl_addr_alignment());
-		}
-	}
-
-	return 0;
-
-hdr_update_fail:
-	ipahal_free_dma_mem(&tbl_mem);
-err:
-	return -EPERM;
-}
-
-static void __ipa_reap_sys_rt_tbls(enum ipa_ip_type ip)
-{
-	struct ipa3_rt_tbl *tbl;
-	struct ipa3_rt_tbl *next;
-	struct ipa3_rt_tbl_set *set;
-	int i;
-
-	set = &ipa3_ctx->rt_tbl_set[ip];
-	list_for_each_entry(tbl, &set->head_rt_tbl_list, link) {
-		for (i = 0; i < IPA_RULE_TYPE_MAX; i++) {
-			if (tbl->prev_mem[i].phys_base) {
-				IPADBG_LOW(
-				"reaping sys rt tbl name=%s ip=%d rlt=%d\n",
-				tbl->name, ip, i);
-				ipahal_free_dma_mem(&tbl->prev_mem[i]);
-				memset(&tbl->prev_mem[i], 0,
-					sizeof(tbl->prev_mem[i]));
-			}
-		}
-	}
-
-	set = &ipa3_ctx->reap_rt_tbl_set[ip];
-	list_for_each_entry_safe(tbl, next, &set->head_rt_tbl_list, link) {
-		for (i = 0; i < IPA_RULE_TYPE_MAX; i++) {
-			WARN_ON(tbl->prev_mem[i].phys_base != 0);
-			if (tbl->curr_mem[i].phys_base) {
-				IPADBG_LOW(
-				"reaping sys rt tbl name=%s ip=%d rlt=%d\n",
-				tbl->name, ip, i);
-				ipahal_free_dma_mem(&tbl->curr_mem[i]);
-			}
-		}
-		list_del(&tbl->link);
-		kmem_cache_free(ipa3_ctx->rt_tbl_cache, tbl);
-	}
-}
-
-/**
- * ipa_prep_rt_tbl_for_cmt() - preparing the rt table for commit
- *  assign priorities to the rules, calculate their sizes and calculate
- *  the overall table size
- * @ip: the ip address family type
- * @tbl: the rt tbl to be prepared
- *
- * Return: 0 on success, negative on failure
- */
-static int ipa_prep_rt_tbl_for_cmt(enum ipa_ip_type ip,
-	struct ipa3_rt_tbl *tbl)
-{
-	struct ipa3_rt_entry *entry;
-	int prio_i;
-	int res;
-	int max_prio;
-	u32 hdr_width;
-
-	tbl->sz[IPA_RULE_HASHABLE] = 0;
-	tbl->sz[IPA_RULE_NON_HASHABLE] = 0;
-
-	max_prio = ipahal_get_rule_max_priority();
-
-	prio_i = max_prio;
-	list_for_each_entry(entry, &tbl->head_rt_rule_list, link) {
-
-		if (entry->rule.max_prio) {
-			entry->prio = max_prio;
-		} else {
-			if (ipahal_rule_decrease_priority(&prio_i)) {
-				IPAERR("cannot rule decrease priority - %d\n",
-					prio_i);
-				return -EPERM;
-			}
-			entry->prio = prio_i;
-		}
-
-		res = ipa_generate_rt_hw_rule(ip, entry, NULL);
-		if (res) {
-			IPAERR_RL("failed to calculate HW RT rule size\n");
-			return -EPERM;
-		}
-
-		IPADBG_LOW("RT rule id (handle) %d hw_len %u priority %u\n",
-			entry->id, entry->hw_len, entry->prio);
-
-		if (entry->rule.hashable)
-			tbl->sz[IPA_RULE_HASHABLE] += entry->hw_len;
-		else
-			tbl->sz[IPA_RULE_NON_HASHABLE] += entry->hw_len;
-	}
-
-	if ((tbl->sz[IPA_RULE_HASHABLE] +
-		tbl->sz[IPA_RULE_NON_HASHABLE]) == 0) {
-		WARN_ON_RATELIMIT_IPA(1);
-		IPAERR_RL("rt tbl %s is with zero total size\n", tbl->name);
-	}
-
-	hdr_width = ipahal_get_hw_tbl_hdr_width();
-
-	if (tbl->sz[IPA_RULE_HASHABLE])
-		tbl->sz[IPA_RULE_HASHABLE] += hdr_width;
-	if (tbl->sz[IPA_RULE_NON_HASHABLE])
-		tbl->sz[IPA_RULE_NON_HASHABLE] += hdr_width;
-
-	IPADBG("RT tbl index %u hash_sz %u non-hash sz %u\n", tbl->idx,
-		tbl->sz[IPA_RULE_HASHABLE], tbl->sz[IPA_RULE_NON_HASHABLE]);
-
-	return 0;
-}
-
-/**
- * ipa_generate_rt_hw_tbl_img() - generates the rt hw tbls.
- *  headers and bodies (sys bodies) are being created into buffers that will
- *  be filled into the local memory (sram)
- * @ip: the ip address family type
- * @alloc_params: IN/OUT parameters to hold info regard the tables headers
- *  and bodies on DDR (DMA buffers), and needed info for the allocation
- *  that the HAL needs
- *
- * Return: 0 on success, negative on failure
- */
-static int ipa_generate_rt_hw_tbl_img(enum ipa_ip_type ip,
-	struct ipahal_fltrt_alloc_imgs_params *alloc_params)
-{
-	u32 hash_bdy_start_ofst, nhash_bdy_start_ofst;
-	u32 apps_start_idx;
-	int rc = 0;
-
-	if (ip == IPA_IP_v4) {
-		nhash_bdy_start_ofst = IPA_MEM_PART(apps_v4_rt_nhash_ofst) -
-			IPA_MEM_PART(v4_rt_nhash_ofst);
-		hash_bdy_start_ofst = IPA_MEM_PART(apps_v4_rt_hash_ofst) -
-			IPA_MEM_PART(v4_rt_hash_ofst);
-		apps_start_idx = IPA_MEM_PART(v4_apps_rt_index_lo);
-	} else {
-		nhash_bdy_start_ofst = IPA_MEM_PART(apps_v6_rt_nhash_ofst) -
-			IPA_MEM_PART(v6_rt_nhash_ofst);
-		hash_bdy_start_ofst = IPA_MEM_PART(apps_v6_rt_hash_ofst) -
-			IPA_MEM_PART(v6_rt_hash_ofst);
-		apps_start_idx = IPA_MEM_PART(v6_apps_rt_index_lo);
-	}
-
-	if (ipahal_fltrt_allocate_hw_tbl_imgs(alloc_params)) {
-		IPAERR("fail to allocate RT HW TBL images. IP %d\n", ip);
-		rc = -ENOMEM;
-		goto allocate_fail;
-	}
-
-	if (ipa_translate_rt_tbl_to_hw_fmt(ip, IPA_RULE_HASHABLE,
-		alloc_params->hash_bdy.base, alloc_params->hash_hdr.base,
-		hash_bdy_start_ofst, apps_start_idx)) {
-		IPAERR("fail to translate hashable rt tbls to hw format\n");
-		rc = -EPERM;
-		goto translate_fail;
-	}
-	if (ipa_translate_rt_tbl_to_hw_fmt(ip, IPA_RULE_NON_HASHABLE,
-		alloc_params->nhash_bdy.base, alloc_params->nhash_hdr.base,
-		nhash_bdy_start_ofst, apps_start_idx)) {
-		IPAERR("fail to translate non-hashable rt tbls to hw format\n");
-		rc = -EPERM;
-		goto translate_fail;
-	}
-
-	return rc;
-
-translate_fail:
-	if (alloc_params->hash_hdr.size)
-		ipahal_free_dma_mem(&alloc_params->hash_hdr);
-	ipahal_free_dma_mem(&alloc_params->nhash_hdr);
-	if (alloc_params->hash_bdy.size)
-		ipahal_free_dma_mem(&alloc_params->hash_bdy);
-	if (alloc_params->nhash_bdy.size)
-		ipahal_free_dma_mem(&alloc_params->nhash_bdy);
-allocate_fail:
-	return rc;
-}
-
-/**
- * ipa_rt_valid_lcl_tbl_size() - validate if the space allocated for rt tbl
- *  bodies at the sram is enough for the commit
- * @ipt: the ip address family type
- * @rlt: the rule type (hashable or non-hashable)
- *
- * Return: true if enough space available or false in other cases
- */
-static bool ipa_rt_valid_lcl_tbl_size(enum ipa_ip_type ipt,
-	enum ipa_rule_type rlt, struct ipa_mem_buffer *bdy)
-{
-	u16 avail;
-
-	if (ipt == IPA_IP_v4)
-		avail = (rlt == IPA_RULE_HASHABLE) ?
-			IPA_MEM_PART(apps_v4_rt_hash_size) :
-			IPA_MEM_PART(apps_v4_rt_nhash_size);
-	else
-		avail = (rlt == IPA_RULE_HASHABLE) ?
-			IPA_MEM_PART(apps_v6_rt_hash_size) :
-			IPA_MEM_PART(apps_v6_rt_nhash_size);
-
-	if (bdy->size <= avail)
-		return true;
-
-	IPAERR("tbl too big, needed %d avail %d ipt %d rlt %d\n",
-		bdy->size, avail, ipt, rlt);
-	return false;
-}
-
-/**
- * __ipa_commit_rt_v3() - commit rt tables to the hw
- * commit the headers and the bodies if are local with internal cache flushing
- * @ipt: the ip address family type
- *
- * Return: 0 on success, negative on failure
- */
-int __ipa_commit_rt_v3(enum ipa_ip_type ip)
-{
-	struct ipa3_desc desc[IPA_RT_MAX_NUM_OF_COMMIT_TABLES_CMD_DESC];
-	struct ipahal_imm_cmd_register_write reg_write_cmd = {0};
-	struct ipahal_imm_cmd_dma_shared_mem  mem_cmd = {0};
-	struct ipahal_imm_cmd_pyld
-		*cmd_pyld[IPA_RT_MAX_NUM_OF_COMMIT_TABLES_CMD_DESC];
-	int num_cmd = 0;
-	struct ipahal_fltrt_alloc_imgs_params alloc_params;
-	u32 num_modem_rt_index;
-	int rc = 0;
-	u32 lcl_hash_hdr, lcl_nhash_hdr;
-	u32 lcl_hash_bdy, lcl_nhash_bdy;
-	bool lcl_hash, lcl_nhash;
-	struct ipahal_reg_fltrt_hash_flush flush;
-	struct ipahal_reg_valmask valmask;
-	int i;
-	struct ipa3_rt_tbl_set *set;
-	struct ipa3_rt_tbl *tbl;
-	u32 tbl_hdr_width;
-
-	tbl_hdr_width = ipahal_get_hw_tbl_hdr_width();
-	memset(desc, 0, sizeof(desc));
-	memset(cmd_pyld, 0, sizeof(cmd_pyld));
-	memset(&alloc_params, 0, sizeof(alloc_params));
-	alloc_params.ipt = ip;
-
-	if (ip == IPA_IP_v4) {
-		num_modem_rt_index =
-			IPA_MEM_PART(v4_modem_rt_index_hi) -
-			IPA_MEM_PART(v4_modem_rt_index_lo) + 1;
-		lcl_hash_hdr = ipa3_ctx->smem_restricted_bytes +
-			IPA_MEM_PART(v4_rt_hash_ofst) +
-			num_modem_rt_index * tbl_hdr_width;
-		lcl_nhash_hdr = ipa3_ctx->smem_restricted_bytes +
-			IPA_MEM_PART(v4_rt_nhash_ofst) +
-			num_modem_rt_index * tbl_hdr_width;
-		lcl_hash_bdy = ipa3_ctx->smem_restricted_bytes +
-			IPA_MEM_PART(apps_v4_rt_hash_ofst);
-		lcl_nhash_bdy = ipa3_ctx->smem_restricted_bytes +
-			IPA_MEM_PART(apps_v4_rt_nhash_ofst);
-		lcl_hash = ipa3_ctx->ip4_rt_tbl_hash_lcl;
-		lcl_nhash = ipa3_ctx->ip4_rt_tbl_nhash_lcl;
-		alloc_params.tbls_num = IPA_MEM_PART(v4_apps_rt_index_hi) -
-			IPA_MEM_PART(v4_apps_rt_index_lo) + 1;
-	} else {
-		num_modem_rt_index =
-			IPA_MEM_PART(v6_modem_rt_index_hi) -
-			IPA_MEM_PART(v6_modem_rt_index_lo) + 1;
-		lcl_hash_hdr = ipa3_ctx->smem_restricted_bytes +
-			IPA_MEM_PART(v6_rt_hash_ofst) +
-			num_modem_rt_index * tbl_hdr_width;
-		lcl_nhash_hdr = ipa3_ctx->smem_restricted_bytes +
-			IPA_MEM_PART(v6_rt_nhash_ofst) +
-			num_modem_rt_index * tbl_hdr_width;
-		lcl_hash_bdy = ipa3_ctx->smem_restricted_bytes +
-			IPA_MEM_PART(apps_v6_rt_hash_ofst);
-		lcl_nhash_bdy = ipa3_ctx->smem_restricted_bytes +
-			IPA_MEM_PART(apps_v6_rt_nhash_ofst);
-		lcl_hash = ipa3_ctx->ip6_rt_tbl_hash_lcl;
-		lcl_nhash = ipa3_ctx->ip6_rt_tbl_nhash_lcl;
-		alloc_params.tbls_num = IPA_MEM_PART(v6_apps_rt_index_hi) -
-			IPA_MEM_PART(v6_apps_rt_index_lo) + 1;
-	}
-
-	if (!ipa3_ctx->rt_idx_bitmap[ip]) {
-		IPAERR("no rt tbls present\n");
-		rc = -EPERM;
-		goto no_rt_tbls;
-	}
-
-	set = &ipa3_ctx->rt_tbl_set[ip];
-	list_for_each_entry(tbl, &set->head_rt_tbl_list, link) {
-		if (ipa_prep_rt_tbl_for_cmt(ip, tbl)) {
-			rc = -EPERM;
-			goto no_rt_tbls;
-		}
-		if (!tbl->in_sys[IPA_RULE_HASHABLE] &&
-			tbl->sz[IPA_RULE_HASHABLE]) {
-			alloc_params.num_lcl_hash_tbls++;
-			alloc_params.total_sz_lcl_hash_tbls +=
-				tbl->sz[IPA_RULE_HASHABLE];
-			alloc_params.total_sz_lcl_hash_tbls -= tbl_hdr_width;
-		}
-		if (!tbl->in_sys[IPA_RULE_NON_HASHABLE] &&
-			tbl->sz[IPA_RULE_NON_HASHABLE]) {
-			alloc_params.num_lcl_nhash_tbls++;
-			alloc_params.total_sz_lcl_nhash_tbls +=
-				tbl->sz[IPA_RULE_NON_HASHABLE];
-			alloc_params.total_sz_lcl_nhash_tbls -= tbl_hdr_width;
-		}
-	}
-
-	if (ipa_generate_rt_hw_tbl_img(ip, &alloc_params)) {
-		IPAERR("fail to generate RT HW TBL images. IP %d\n", ip);
-		rc = -EFAULT;
-		goto no_rt_tbls;
-	}
-
-	if (!ipa_rt_valid_lcl_tbl_size(ip, IPA_RULE_HASHABLE,
-		&alloc_params.hash_bdy)) {
-		rc = -EFAULT;
-		goto fail_size_valid;
-	}
-	if (!ipa_rt_valid_lcl_tbl_size(ip, IPA_RULE_NON_HASHABLE,
-		&alloc_params.nhash_bdy)) {
-		rc = -EFAULT;
-		goto fail_size_valid;
-	}
-
-	/*
-	 * SRAM memory not allocated to hash tables. Sending
-	 * command to hash tables(filer/routing) operation not supported.
-	 */
-	if (!ipa3_ctx->ipa_fltrt_not_hashable) {
-		/* flushing ipa internal hashable rt rules cache */
-		memset(&flush, 0, sizeof(flush));
-		if (ip == IPA_IP_v4)
-			flush.v4_rt = true;
-		else
-			flush.v6_rt = true;
-		ipahal_get_fltrt_hash_flush_valmask(&flush, &valmask);
-		reg_write_cmd.skip_pipeline_clear = false;
-		reg_write_cmd.pipeline_clear_options = IPAHAL_HPS_CLEAR;
-		reg_write_cmd.offset = ipahal_get_reg_ofst(
-					IPA_FILT_ROUT_HASH_FLUSH);
-		reg_write_cmd.value = valmask.val;
-		reg_write_cmd.value_mask = valmask.mask;
-		cmd_pyld[num_cmd] = ipahal_construct_imm_cmd(
-				IPA_IMM_CMD_REGISTER_WRITE, &reg_write_cmd,
-							false);
-		if (!cmd_pyld[num_cmd]) {
-			IPAERR(
-			"fail construct register_write imm cmd. IP %d\n", ip);
-			goto fail_size_valid;
-		}
-		ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]);
-		num_cmd++;
-	}
-
-	mem_cmd.is_read = false;
-	mem_cmd.skip_pipeline_clear = false;
-	mem_cmd.pipeline_clear_options = IPAHAL_HPS_CLEAR;
-	mem_cmd.size = alloc_params.nhash_hdr.size;
-	mem_cmd.system_addr = alloc_params.nhash_hdr.phys_base;
-	mem_cmd.local_addr = lcl_nhash_hdr;
-	cmd_pyld[num_cmd] = ipahal_construct_imm_cmd(
-		IPA_IMM_CMD_DMA_SHARED_MEM, &mem_cmd, false);
-	if (!cmd_pyld[num_cmd]) {
-		IPAERR("fail construct dma_shared_mem imm cmd. IP %d\n", ip);
-		goto fail_imm_cmd_construct;
-	}
-	ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]);
-	num_cmd++;
-
-	/*
-	 * SRAM memory not allocated to hash tables. Sending
-	 * command to hash tables(filer/routing) operation not supported.
-	 */
-	if (!ipa3_ctx->ipa_fltrt_not_hashable) {
-		mem_cmd.is_read = false;
-		mem_cmd.skip_pipeline_clear = false;
-		mem_cmd.pipeline_clear_options = IPAHAL_HPS_CLEAR;
-		mem_cmd.size = alloc_params.hash_hdr.size;
-		mem_cmd.system_addr = alloc_params.hash_hdr.phys_base;
-		mem_cmd.local_addr = lcl_hash_hdr;
-		cmd_pyld[num_cmd] = ipahal_construct_imm_cmd(
-				IPA_IMM_CMD_DMA_SHARED_MEM, &mem_cmd, false);
-		if (!cmd_pyld[num_cmd]) {
-			IPAERR(
-			"fail construct dma_shared_mem imm cmd. IP %d\n", ip);
-			goto fail_imm_cmd_construct;
-		}
-		ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]);
-		num_cmd++;
-	}
-
-	if (lcl_nhash) {
-		if (num_cmd >= IPA_RT_MAX_NUM_OF_COMMIT_TABLES_CMD_DESC) {
-			IPAERR("number of commands is out of range: IP = %d\n",
-				ip);
-			rc = -ENOBUFS;
-			goto fail_imm_cmd_construct;
-		}
-
-		mem_cmd.is_read = false;
-		mem_cmd.skip_pipeline_clear = false;
-		mem_cmd.pipeline_clear_options = IPAHAL_HPS_CLEAR;
-		mem_cmd.size = alloc_params.nhash_bdy.size;
-		mem_cmd.system_addr = alloc_params.nhash_bdy.phys_base;
-		mem_cmd.local_addr = lcl_nhash_bdy;
-		cmd_pyld[num_cmd] = ipahal_construct_imm_cmd(
-			IPA_IMM_CMD_DMA_SHARED_MEM, &mem_cmd, false);
-		if (!cmd_pyld[num_cmd]) {
-			IPAERR("fail construct dma_shared_mem cmd. IP %d\n",
-				ip);
-			goto fail_imm_cmd_construct;
-		}
-		ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]);
-		num_cmd++;
-	}
-	if (lcl_hash) {
-		if (num_cmd >= IPA_RT_MAX_NUM_OF_COMMIT_TABLES_CMD_DESC) {
-			IPAERR("number of commands is out of range: IP = %d\n",
-				ip);
-			rc = -ENOBUFS;
-			goto fail_imm_cmd_construct;
-		}
-
-		mem_cmd.is_read = false;
-		mem_cmd.skip_pipeline_clear = false;
-		mem_cmd.pipeline_clear_options = IPAHAL_HPS_CLEAR;
-		mem_cmd.size = alloc_params.hash_bdy.size;
-		mem_cmd.system_addr = alloc_params.hash_bdy.phys_base;
-		mem_cmd.local_addr = lcl_hash_bdy;
-		cmd_pyld[num_cmd] = ipahal_construct_imm_cmd(
-			IPA_IMM_CMD_DMA_SHARED_MEM, &mem_cmd, false);
-		if (!cmd_pyld[num_cmd]) {
-			IPAERR("fail construct dma_shared_mem cmd. IP %d\n",
-				ip);
-			goto fail_imm_cmd_construct;
-		}
-		ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]);
-		num_cmd++;
-	}
-
-	if (ipa3_send_cmd(num_cmd, desc)) {
-		IPAERR_RL("fail to send immediate command\n");
-		rc = -EFAULT;
-		goto fail_imm_cmd_construct;
-	}
-
-	IPADBG_LOW("Hashable HEAD\n");
-	IPA_DUMP_BUFF(alloc_params.hash_hdr.base,
-		alloc_params.hash_hdr.phys_base, alloc_params.hash_hdr.size);
-
-	IPADBG_LOW("Non-Hashable HEAD\n");
-	IPA_DUMP_BUFF(alloc_params.nhash_hdr.base,
-		alloc_params.nhash_hdr.phys_base, alloc_params.nhash_hdr.size);
-
-	if (alloc_params.hash_bdy.size) {
-		IPADBG_LOW("Hashable BODY\n");
-		IPA_DUMP_BUFF(alloc_params.hash_bdy.base,
-			alloc_params.hash_bdy.phys_base,
-			alloc_params.hash_bdy.size);
-	}
-
-	if (alloc_params.nhash_bdy.size) {
-		IPADBG_LOW("Non-Hashable BODY\n");
-		IPA_DUMP_BUFF(alloc_params.nhash_bdy.base,
-			alloc_params.nhash_bdy.phys_base,
-			alloc_params.nhash_bdy.size);
-	}
-
-	__ipa_reap_sys_rt_tbls(ip);
-
-fail_imm_cmd_construct:
-	for (i = 0 ; i < num_cmd ; i++)
-		ipahal_destroy_imm_cmd(cmd_pyld[i]);
-fail_size_valid:
-	if (alloc_params.hash_hdr.size)
-		ipahal_free_dma_mem(&alloc_params.hash_hdr);
-	ipahal_free_dma_mem(&alloc_params.nhash_hdr);
-	if (alloc_params.hash_bdy.size)
-		ipahal_free_dma_mem(&alloc_params.hash_bdy);
-	if (alloc_params.nhash_bdy.size)
-		ipahal_free_dma_mem(&alloc_params.nhash_bdy);
-
-no_rt_tbls:
-	return rc;
-}
-
-/**
- * __ipa3_find_rt_tbl() - find the routing table
- *			which name is given as parameter
- * @ip:	[in] the ip address family type of the wanted routing table
- * @name:	[in] the name of the wanted routing table
- *
- * Returns: the routing table which name is given as parameter, or NULL if it
- * doesn't exist
- */
-struct ipa3_rt_tbl *__ipa3_find_rt_tbl(enum ipa_ip_type ip, const char *name)
-{
-	struct ipa3_rt_tbl *entry;
-	struct ipa3_rt_tbl_set *set;
-
-	if (strnlen(name, IPA_RESOURCE_NAME_MAX) == IPA_RESOURCE_NAME_MAX) {
-		IPAERR_RL("Name too long: %s\n", name);
-		return NULL;
-	}
-
-	set = &ipa3_ctx->rt_tbl_set[ip];
-	list_for_each_entry(entry, &set->head_rt_tbl_list, link) {
-		if (!strcmp(name, entry->name))
-			return entry;
-	}
-
-	return NULL;
-}
-
-/**
- * ipa3_query_rt_index() - find the routing table index
- *			which name and ip type are given as parameters
- * @in:	[out] the index of the wanted routing table
- *
- * Returns: the routing table which name is given as parameter, or NULL if it
- * doesn't exist
- */
-int ipa3_query_rt_index(struct ipa_ioc_get_rt_tbl_indx *in)
-{
-	struct ipa3_rt_tbl *entry;
-
-	if (in->ip >= IPA_IP_MAX) {
-		IPAERR_RL("bad param\n");
-		return -EINVAL;
-	}
-
-	mutex_lock(&ipa3_ctx->lock);
-	in->name[IPA_RESOURCE_NAME_MAX-1] = '\0';
-	/* check if this table exists */
-	entry = __ipa3_find_rt_tbl(in->ip, in->name);
-	if (!entry) {
-		mutex_unlock(&ipa3_ctx->lock);
-		return -EFAULT;
-	}
-	in->idx  = entry->idx;
-	mutex_unlock(&ipa3_ctx->lock);
-	return 0;
-}
-
-static struct ipa3_rt_tbl *__ipa_add_rt_tbl(enum ipa_ip_type ip,
-		const char *name)
-{
-	struct ipa3_rt_tbl *entry;
-	struct ipa3_rt_tbl_set *set;
-	int i;
-	int id;
-	int max_tbl_indx;
-
-	if (name == NULL) {
-		IPAERR_RL("no tbl name\n");
-		goto error;
-	}
-
-	if (ip == IPA_IP_v4) {
-		max_tbl_indx =
-			max(IPA_MEM_PART(v4_modem_rt_index_hi),
-			IPA_MEM_PART(v4_apps_rt_index_hi));
-	} else if (ip == IPA_IP_v6) {
-		max_tbl_indx =
-			max(IPA_MEM_PART(v6_modem_rt_index_hi),
-			IPA_MEM_PART(v6_apps_rt_index_hi));
-	} else {
-		IPAERR_RL("bad ip family type\n");
-		goto error;
-	}
-
-	set = &ipa3_ctx->rt_tbl_set[ip];
-	/* check if this table exists */
-	entry = __ipa3_find_rt_tbl(ip, name);
-	if (!entry) {
-		entry = kmem_cache_zalloc(ipa3_ctx->rt_tbl_cache, GFP_KERNEL);
-		if (!entry)
-			goto error;
-
-		/* find a routing tbl index */
-		for (i = 0; i < IPA_RT_INDEX_BITMAP_SIZE; i++) {
-			if (!test_bit(i, &ipa3_ctx->rt_idx_bitmap[ip])) {
-				entry->idx = i;
-				set_bit(i, &ipa3_ctx->rt_idx_bitmap[ip]);
-				break;
-			}
-		}
-		if (i == IPA_RT_INDEX_BITMAP_SIZE) {
-			IPAERR("not free RT tbl indices left\n");
-			goto fail_rt_idx_alloc;
-		}
-		if (i > max_tbl_indx) {
-			IPAERR("rt tbl index is above max\n");
-			goto fail_rt_idx_alloc;
-		}
-
-		INIT_LIST_HEAD(&entry->head_rt_rule_list);
-		INIT_LIST_HEAD(&entry->link);
-		strlcpy(entry->name, name, IPA_RESOURCE_NAME_MAX);
-		entry->set = set;
-		entry->cookie = IPA_RT_TBL_COOKIE;
-		entry->in_sys[IPA_RULE_HASHABLE] = (ip == IPA_IP_v4) ?
-			!ipa3_ctx->ip4_rt_tbl_hash_lcl :
-			!ipa3_ctx->ip6_rt_tbl_hash_lcl;
-		entry->in_sys[IPA_RULE_NON_HASHABLE] = (ip == IPA_IP_v4) ?
-			!ipa3_ctx->ip4_rt_tbl_nhash_lcl :
-			!ipa3_ctx->ip6_rt_tbl_nhash_lcl;
-		set->tbl_cnt++;
-		entry->rule_ids = &set->rule_ids;
-		list_add(&entry->link, &set->head_rt_tbl_list);
-
-		IPADBG("add rt tbl idx=%d tbl_cnt=%d ip=%d\n", entry->idx,
-				set->tbl_cnt, ip);
-
-		id = ipa3_id_alloc(entry);
-		if (id < 0) {
-			IPAERR_RL("failed to add to tree\n");
-			WARN_ON_RATELIMIT_IPA(1);
-			goto ipa_insert_failed;
-		}
-		entry->id = id;
-	}
-
-	return entry;
-ipa_insert_failed:
-	set->tbl_cnt--;
-	list_del(&entry->link);
-	idr_destroy(entry->rule_ids);
-fail_rt_idx_alloc:
-	entry->cookie = 0;
-	kmem_cache_free(ipa3_ctx->rt_tbl_cache, entry);
-error:
-	return NULL;
-}
-
-static int __ipa_del_rt_tbl(struct ipa3_rt_tbl *entry)
-{
-	enum ipa_ip_type ip = IPA_IP_MAX;
-	u32 id;
-	struct ipa3_rt_tbl_set *rset;
-
-	if (entry == NULL || (entry->cookie != IPA_RT_TBL_COOKIE)) {
-		IPAERR_RL("bad params\n");
-		return -EINVAL;
-	}
-	id = entry->id;
-	if (ipa3_id_find(id) == NULL) {
-		IPAERR_RL("lookup failed\n");
-		return -EPERM;
-	}
-
-	if (entry->set == &ipa3_ctx->rt_tbl_set[IPA_IP_v4])
-		ip = IPA_IP_v4;
-	else if (entry->set == &ipa3_ctx->rt_tbl_set[IPA_IP_v6])
-		ip = IPA_IP_v6;
-	else {
-		WARN_ON_RATELIMIT_IPA(1);
-		return -EPERM;
-	}
-
-	rset = &ipa3_ctx->reap_rt_tbl_set[ip];
-
-	entry->rule_ids = NULL;
-	if (entry->in_sys[IPA_RULE_HASHABLE] ||
-		entry->in_sys[IPA_RULE_NON_HASHABLE]) {
-		list_move(&entry->link, &rset->head_rt_tbl_list);
-		clear_bit(entry->idx, &ipa3_ctx->rt_idx_bitmap[ip]);
-		entry->set->tbl_cnt--;
-		IPADBG("del sys rt tbl_idx=%d tbl_cnt=%d ip=%d\n",
-			entry->idx, entry->set->tbl_cnt, ip);
-	} else {
-		list_del(&entry->link);
-		clear_bit(entry->idx, &ipa3_ctx->rt_idx_bitmap[ip]);
-		entry->set->tbl_cnt--;
-		IPADBG("del rt tbl_idx=%d tbl_cnt=%d ip=%d\n",
-			entry->idx, entry->set->tbl_cnt, ip);
-		kmem_cache_free(ipa3_ctx->rt_tbl_cache, entry);
-	}
-
-	/* remove the handle from the database */
-	ipa3_id_remove(id);
-	return 0;
-}
-
-static int __ipa_rt_validate_rule_id(u16 rule_id)
-{
-	if (!rule_id)
-		return 0;
-
-	if ((rule_id < ipahal_get_rule_id_hi_bit()) ||
-		(rule_id >= ((ipahal_get_rule_id_hi_bit()<<1)-1))) {
-		IPAERR_RL("Invalid rule_id provided 0x%x\n",
-			rule_id);
-		return -EPERM;
-	}
-
-	return 0;
-}
-static int __ipa_rt_validate_hndls(const struct ipa_rt_rule_i *rule,
-				struct ipa3_hdr_entry **hdr,
-				struct ipa3_hdr_proc_ctx_entry **proc_ctx)
-{
-	int index;
-
-	if (rule->hdr_hdl && rule->hdr_proc_ctx_hdl) {
-		IPAERR_RL("rule contains both hdr_hdl and hdr_proc_ctx_hdl\n");
-		return -EPERM;
-	}
-
-	if (rule->hdr_hdl) {
-		*hdr = ipa3_id_find(rule->hdr_hdl);
-		if ((*hdr == NULL) || ((*hdr)->cookie != IPA_HDR_COOKIE)) {
-			IPAERR_RL("rt rule does not point to valid hdr\n");
-			return -EPERM;
-		}
-	} else if (rule->hdr_proc_ctx_hdl) {
-		*proc_ctx = ipa3_id_find(rule->hdr_proc_ctx_hdl);
-		if ((*proc_ctx == NULL) ||
-			((*proc_ctx)->cookie != IPA_PROC_HDR_COOKIE)) {
-
-			IPAERR_RL("rt rule does not point to valid proc ctx\n");
-			return -EPERM;
-		}
-	}
-
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_5 && rule->coalesce) {
-		IPAERR_RL("rt rule should not allow coalescing\n");
-		return -EPERM;
-	}
-
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_5) {
-		if (rule->enable_stats && rule->cnt_idx) {
-			if (!ipahal_is_rule_cnt_id_valid(rule->cnt_idx)) {
-				IPAERR_RL(
-					"invalid cnt_idx %hhu out of range\n",
-					rule->cnt_idx);
-				return -EPERM;
-			}
-			index = rule->cnt_idx - 1;
-			if (!ipa3_ctx->flt_rt_counters.used_hw[index]) {
-				IPAERR_RL(
-					"invalid cnt_idx %hhu not alloc by driver\n",
-					rule->cnt_idx);
-				return -EPERM;
-			}
-		}
-	} else {
-		if (rule->enable_stats) {
-			IPAERR_RL(
-				"enable_stats won't support on ipa_hw_type %d\n",
-				ipa3_ctx->ipa_hw_type);
-			return -EPERM;
-		}
-	}
-	return 0;
-}
-
-static int __ipa_create_rt_entry(struct ipa3_rt_entry **entry,
-		const struct ipa_rt_rule_i *rule,
-		struct ipa3_rt_tbl *tbl, struct ipa3_hdr_entry *hdr,
-		struct ipa3_hdr_proc_ctx_entry *proc_ctx,
-		u16 rule_id, bool user)
-{
-	int id;
-
-	*entry = kmem_cache_zalloc(ipa3_ctx->rt_rule_cache, GFP_KERNEL);
-	if (!*entry)
-		goto error;
-
-	INIT_LIST_HEAD(&(*entry)->link);
-	(*(entry))->cookie = IPA_RT_RULE_COOKIE;
-	(*(entry))->rule = *rule;
-	(*(entry))->tbl = tbl;
-	(*(entry))->hdr = hdr;
-	(*(entry))->proc_ctx = proc_ctx;
-	if (rule_id) {
-		id = rule_id;
-		(*(entry))->rule_id_valid = 1;
-	} else {
-		id = ipa3_alloc_rule_id(tbl->rule_ids);
-		if (id < 0) {
-			IPAERR_RL("failed to allocate rule id\n");
-			WARN_ON_RATELIMIT_IPA(1);
-			goto alloc_rule_id_fail;
-		}
-	}
-	(*(entry))->rule_id = id;
-	(*(entry))->ipacm_installed = user;
-
-	if ((*(entry))->rule.coalesce &&
-		(*(entry))->rule.dst == IPA_CLIENT_APPS_WAN_CONS &&
-		ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS) != -1)
-		(*(entry))->rule.dst = IPA_CLIENT_APPS_WAN_COAL_CONS;
-
-	if (rule->enable_stats)
-		(*entry)->cnt_idx = rule->cnt_idx;
-	else
-		(*entry)->cnt_idx = 0;
-	return 0;
-
-alloc_rule_id_fail:
-	kmem_cache_free(ipa3_ctx->rt_rule_cache, *entry);
-error:
-	return -EPERM;
-}
-
-static int __ipa_finish_rt_rule_add(struct ipa3_rt_entry *entry, u32 *rule_hdl,
-		struct ipa3_rt_tbl *tbl)
-{
-	int id;
-
-	tbl->rule_cnt++;
-	if (entry->hdr)
-		entry->hdr->ref_cnt++;
-	else if (entry->proc_ctx)
-		entry->proc_ctx->ref_cnt++;
-	id = ipa3_id_alloc(entry);
-	if (id < 0) {
-		IPAERR_RL("failed to add to tree\n");
-		WARN_ON_RATELIMIT_IPA(1);
-		goto ipa_insert_failed;
-	}
-	IPADBG("add rt rule tbl_idx=%d rule_cnt=%d rule_id=%d\n",
-		tbl->idx, tbl->rule_cnt, entry->rule_id);
-	*rule_hdl = id;
-	entry->id = id;
-
-	return 0;
-
-ipa_insert_failed:
-	if (entry->hdr)
-		entry->hdr->ref_cnt--;
-	else if (entry->proc_ctx)
-		entry->proc_ctx->ref_cnt--;
-	idr_remove(tbl->rule_ids, entry->rule_id);
-	list_del(&entry->link);
-	kmem_cache_free(ipa3_ctx->rt_rule_cache, entry);
-	return -EPERM;
-}
-
-static int __ipa_add_rt_rule(enum ipa_ip_type ip, const char *name,
-		const struct ipa_rt_rule_i *rule, u8 at_rear, u32 *rule_hdl,
-		u16 rule_id, bool user)
-{
-	struct ipa3_rt_tbl *tbl;
-	struct ipa3_rt_entry *entry;
-	struct ipa3_hdr_entry *hdr = NULL;
-	struct ipa3_hdr_proc_ctx_entry *proc_ctx = NULL;
-
-	if (__ipa_rt_validate_hndls(rule, &hdr, &proc_ctx))
-		goto error;
-
-	if (__ipa_rt_validate_rule_id(rule_id))
-		goto error;
-
-	tbl = __ipa_add_rt_tbl(ip, name);
-	if (tbl == NULL || (tbl->cookie != IPA_RT_TBL_COOKIE)) {
-		IPAERR_RL("failed adding rt tbl name = %s\n",
-			name ? name : "");
-		goto error;
-	}
-	/*
-	 * do not allow any rule to be added at "default" routing
-	 * table
-	 */
-	if (!strcmp(tbl->name, IPA_DFLT_RT_TBL_NAME) &&
-	    (tbl->rule_cnt > 0)) {
-		IPAERR_RL("cannot add rules to default rt table\n");
-		goto error;
-	}
-
-	if (__ipa_create_rt_entry(&entry, rule, tbl, hdr, proc_ctx,
-		rule_id, user))
-		goto error;
-
-	if (at_rear)
-		list_add_tail(&entry->link, &tbl->head_rt_rule_list);
-	else
-		list_add(&entry->link, &tbl->head_rt_rule_list);
-
-	if (__ipa_finish_rt_rule_add(entry, rule_hdl, tbl))
-		goto error;
-
-	return 0;
-
-error:
-	return -EPERM;
-}
-
-static int __ipa_add_rt_rule_after(struct ipa3_rt_tbl *tbl,
-		const struct ipa_rt_rule_i *rule, u32 *rule_hdl,
-		struct ipa3_rt_entry **add_after_entry)
-{
-	struct ipa3_rt_entry *entry;
-	struct ipa3_hdr_entry *hdr = NULL;
-	struct ipa3_hdr_proc_ctx_entry *proc_ctx = NULL;
-
-	if (!*add_after_entry)
-		goto error;
-
-	if (__ipa_rt_validate_hndls(rule, &hdr, &proc_ctx))
-		goto error;
-
-	if (__ipa_create_rt_entry(&entry, rule, tbl, hdr, proc_ctx, 0, true))
-		goto error;
-
-	list_add(&entry->link, &((*add_after_entry)->link));
-
-	if (__ipa_finish_rt_rule_add(entry, rule_hdl, tbl))
-		goto error;
-
-	/*
-	 * prepare for next insertion
-	 */
-	*add_after_entry = entry;
-
-	return 0;
-
-error:
-	*add_after_entry = NULL;
-	return -EPERM;
-}
-
-static void __ipa_convert_rt_rule_in(struct ipa_rt_rule rule_in,
-	struct ipa_rt_rule_i *rule_out)
-{
-	if (unlikely(sizeof(struct ipa_rt_rule) >
-			sizeof(struct ipa_rt_rule_i))) {
-		IPAERR_RL("invalid size in: %d size out: %d\n",
-			sizeof(struct ipa_rt_rule),
-			sizeof(struct ipa_rt_rule_i));
-		return;
-	}
-	memset(rule_out, 0, sizeof(struct ipa_rt_rule_i));
-	memcpy(rule_out, &rule_in, sizeof(struct ipa_rt_rule));
-}
-
-static void __ipa_convert_rt_rule_out(struct ipa_rt_rule_i rule_in,
-	struct ipa_rt_rule *rule_out)
-{
-	if (unlikely(sizeof(struct ipa_rt_rule) >
-			sizeof(struct ipa_rt_rule_i))) {
-		IPAERR_RL("invalid size in:%d size out:%d\n",
-			sizeof(struct ipa_rt_rule),
-			sizeof(struct ipa_rt_rule_i));
-		return;
-	}
-	memset(rule_out, 0, sizeof(struct ipa_rt_rule));
-	memcpy(rule_out, &rule_in, sizeof(struct ipa_rt_rule));
-}
-
-static void __ipa_convert_rt_mdfy_in(struct ipa_rt_rule_mdfy rule_in,
-	struct ipa_rt_rule_mdfy_i *rule_out)
-{
-	if (unlikely(sizeof(struct ipa_rt_rule_mdfy) >
-			sizeof(struct ipa_rt_rule_mdfy_i))) {
-		IPAERR_RL("invalid size in:%d size out:%d\n",
-			sizeof(struct ipa_rt_rule_mdfy),
-			sizeof(struct ipa_rt_rule_mdfy_i));
-		return;
-	}
-	memset(rule_out, 0, sizeof(struct ipa_rt_rule_mdfy_i));
-	memcpy(&rule_out->rule, &rule_in.rule,
-		sizeof(struct ipa_rt_rule));
-	rule_out->rt_rule_hdl = rule_in.rt_rule_hdl;
-	rule_out->status = rule_in.status;
-}
-
-static void __ipa_convert_rt_mdfy_out(struct ipa_rt_rule_mdfy_i rule_in,
-	struct ipa_rt_rule_mdfy *rule_out)
-{
-	if (unlikely(sizeof(struct ipa_rt_rule_mdfy) >
-			sizeof(struct ipa_rt_rule_mdfy_i))) {
-		IPAERR_RL("invalid size in:%d size out:%d\n",
-			sizeof(struct ipa_rt_rule_mdfy),
-			sizeof(struct ipa_rt_rule_mdfy_i));
-		return;
-	}
-	memset(rule_out, 0, sizeof(struct ipa_rt_rule_mdfy));
-	memcpy(&rule_out->rule, &rule_in.rule,
-		sizeof(struct ipa_rt_rule));
-	rule_out->rt_rule_hdl = rule_in.rt_rule_hdl;
-	rule_out->status = rule_in.status;
-}
-
-/**
- * ipa3_add_rt_rule() - Add the specified routing rules to SW and optionally
- * commit to IPA HW
- * @rules:	[inout] set of routing rules to add
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-
-int ipa3_add_rt_rule(struct ipa_ioc_add_rt_rule *rules)
-{
-	return ipa3_add_rt_rule_usr(rules, false);
-}
-
-/**
- * ipa3_add_rt_rule_v2() - Add the specified routing rules to SW
- * and optionally commit to IPA HW
- * @rules:	[inout] set of routing rules to add
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-
-int ipa3_add_rt_rule_v2(struct ipa_ioc_add_rt_rule_v2 *rules)
-{
-	return ipa3_add_rt_rule_usr_v2(rules, false);
-}
-
-/**
- * ipa3_add_rt_rule_usr() - Add the specified routing rules to SW and optionally
- * commit to IPA HW
- * @rules:		[inout] set of routing rules to add
- * @user_only:	[in] indicate installed by userspace module
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-
-int ipa3_add_rt_rule_usr(struct ipa_ioc_add_rt_rule *rules, bool user_only)
-{
-	int i;
-	int ret;
-	struct ipa_rt_rule_i rule;
-
-	if (rules == NULL || rules->num_rules == 0 || rules->ip >= IPA_IP_MAX) {
-		IPAERR_RL("bad param\n");
-		return -EINVAL;
-	}
-
-	mutex_lock(&ipa3_ctx->lock);
-	for (i = 0; i < rules->num_rules; i++) {
-		rules->rt_tbl_name[IPA_RESOURCE_NAME_MAX-1] = '\0';
-		/* if hashing not supported, all tables are non-hash tables*/
-		if (ipa3_ctx->ipa_fltrt_not_hashable)
-			rules->rules[i].rule.hashable = false;
-		__ipa_convert_rt_rule_in(rules->rules[i].rule, &rule);
-		if (__ipa_add_rt_rule(rules->ip, rules->rt_tbl_name,
-					&rule,
-					rules->rules[i].at_rear,
-					&rules->rules[i].rt_rule_hdl,
-					0,
-					user_only)) {
-			IPAERR_RL("failed to add rt rule %d\n", i);
-			rules->rules[i].status = IPA_RT_STATUS_OF_ADD_FAILED;
-		} else {
-			__ipa_convert_rt_rule_out(rule, &rules->rules[i].rule);
-			rules->rules[i].status = 0;
-		}
-	}
-
-	if (rules->commit)
-		if (ipa3_ctx->ctrl->ipa3_commit_rt(rules->ip)) {
-			ret = -EPERM;
-			goto bail;
-		}
-
-	ret = 0;
-bail:
-	mutex_unlock(&ipa3_ctx->lock);
-	return ret;
-}
-
-/**
- * ipa3_add_rt_rule_usr_v2() - Add the specified routing rules
- * to SW and optionally commit to IPA HW
- * @rules:		[inout] set of routing rules to add
- * @user_only:	[in] indicate installed by userspace module
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-
-int ipa3_add_rt_rule_usr_v2(struct ipa_ioc_add_rt_rule_v2 *rules,
-	bool user_only)
-{
-	int i;
-	int ret;
-
-	if (rules == NULL || rules->num_rules == 0 || rules->ip >= IPA_IP_MAX) {
-		IPAERR_RL("bad param\n");
-		return -EINVAL;
-	}
-
-	mutex_lock(&ipa3_ctx->lock);
-	for (i = 0; i < rules->num_rules; i++) {
-		rules->rt_tbl_name[IPA_RESOURCE_NAME_MAX-1] = '\0';
-		/* if hashing not supported, all tables are non-hash tables*/
-		if (ipa3_ctx->ipa_fltrt_not_hashable)
-			((struct ipa_rt_rule_add_i *)
-			rules->rules)[i].rule.hashable = false;
-		if (__ipa_add_rt_rule(rules->ip, rules->rt_tbl_name,
-					&(((struct ipa_rt_rule_add_i *)
-					rules->rules)[i].rule),
-					((struct ipa_rt_rule_add_i *)
-					rules->rules)[i].at_rear,
-					&(((struct ipa_rt_rule_add_i *)
-					rules->rules)[i].rt_rule_hdl),
-					0,
-					user_only)) {
-			IPAERR_RL("failed to add rt rule %d\n", i);
-			((struct ipa_rt_rule_add_i *)rules->rules)[i].status
-				= IPA_RT_STATUS_OF_ADD_FAILED;
-		} else {
-			((struct ipa_rt_rule_add_i *)
-			rules->rules)[i].status = 0;
-		}
-	}
-
-	if (rules->commit)
-		if (ipa3_ctx->ctrl->ipa3_commit_rt(rules->ip)) {
-			ret = -EPERM;
-			goto bail;
-		}
-
-	ret = 0;
-bail:
-	mutex_unlock(&ipa3_ctx->lock);
-	return ret;
-}
-
-
-/**
- * ipa3_add_rt_rule_ext() - Add the specified routing rules to SW with rule id
- * and optionally commit to IPA HW
- * @rules:	[inout] set of routing rules to add
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_add_rt_rule_ext(struct ipa_ioc_add_rt_rule_ext *rules)
-{
-	int i;
-	int ret;
-	struct ipa_rt_rule_i rule;
-
-	if (rules == NULL || rules->num_rules == 0 || rules->ip >= IPA_IP_MAX) {
-		IPAERR_RL("bad param\n");
-		return -EINVAL;
-	}
-
-	mutex_lock(&ipa3_ctx->lock);
-	for (i = 0; i < rules->num_rules; i++) {
-		/* if hashing not supported, all tables are non-hash tables*/
-		if (ipa3_ctx->ipa_fltrt_not_hashable)
-			rules->rules[i].rule.hashable = false;
-		__ipa_convert_rt_rule_in(
-				rules->rules[i].rule, &rule);
-		if (__ipa_add_rt_rule(rules->ip, rules->rt_tbl_name,
-					&rule,
-					rules->rules[i].at_rear,
-					&rules->rules[i].rt_rule_hdl,
-					rules->rules[i].rule_id, true)) {
-			IPAERR_RL("failed to add rt rule %d\n", i);
-			rules->rules[i].status = IPA_RT_STATUS_OF_ADD_FAILED;
-		} else {
-			__ipa_convert_rt_rule_out(rule, &rules->rules[i].rule);
-			rules->rules[i].status = 0;
-		}
-	}
-
-	if (rules->commit)
-		if (ipa3_ctx->ctrl->ipa3_commit_rt(rules->ip)) {
-			ret = -EPERM;
-			goto bail;
-		}
-
-	ret = 0;
-bail:
-	mutex_unlock(&ipa3_ctx->lock);
-	return ret;
-}
-
-/**
- * ipa3_add_rt_rule_ext_v2() - Add the specified routing rules
- * to SW with rule id and optionally commit to IPA HW
- * @rules:	[inout] set of routing rules to add
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_add_rt_rule_ext_v2(struct ipa_ioc_add_rt_rule_ext_v2 *rules)
-{
-	int i;
-	int ret;
-
-	if (rules == NULL || rules->num_rules == 0 || rules->ip >= IPA_IP_MAX) {
-		IPAERR_RL("bad param\n");
-		return -EINVAL;
-	}
-
-	mutex_lock(&ipa3_ctx->lock);
-	for (i = 0; i < rules->num_rules; i++) {
-		/* if hashing not supported, all tables are non-hash tables*/
-		if (ipa3_ctx->ipa_fltrt_not_hashable)
-			((struct ipa_rt_rule_add_ext_i *)
-			rules->rules)[i].rule.hashable = false;
-		if (__ipa_add_rt_rule(rules->ip, rules->rt_tbl_name,
-					&(((struct ipa_rt_rule_add_ext_i *)
-					rules->rules)[i].rule),
-					((struct ipa_rt_rule_add_ext_i *)
-					rules->rules)[i].at_rear,
-					&(((struct ipa_rt_rule_add_ext_i *)
-					rules->rules)[i].rt_rule_hdl),
-					((struct ipa_rt_rule_add_ext_i *)
-					rules->rules)[i].rule_id, true)) {
-			IPAERR_RL("failed to add rt rule %d\n", i);
-			((struct ipa_rt_rule_add_ext_i *)
-			rules->rules)[i].status = IPA_RT_STATUS_OF_ADD_FAILED;
-		} else {
-			((struct ipa_rt_rule_add_ext_i *)
-			rules->rules)[i].status = 0;
-		}
-	}
-
-	if (rules->commit)
-		if (ipa3_ctx->ctrl->ipa3_commit_rt(rules->ip)) {
-			ret = -EPERM;
-			goto bail;
-		}
-
-	ret = 0;
-bail:
-	mutex_unlock(&ipa3_ctx->lock);
-	return ret;
-}
-
-/**
- * ipa3_add_rt_rule_after() - Add the given routing rules after the
- * specified rule to SW and optionally commit to IPA HW
- * @rules:	[inout] set of routing rules to add + handle where to add
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_add_rt_rule_after(struct ipa_ioc_add_rt_rule_after *rules)
-{
-	int i;
-	int ret = 0;
-	struct ipa3_rt_tbl *tbl = NULL;
-	struct ipa3_rt_entry *entry = NULL;
-	struct ipa_rt_rule_i rule;
-
-	if (rules == NULL || rules->num_rules == 0 || rules->ip >= IPA_IP_MAX) {
-		IPAERR_RL("bad param\n");
-		return -EINVAL;
-	}
-
-	mutex_lock(&ipa3_ctx->lock);
-	rules->rt_tbl_name[IPA_RESOURCE_NAME_MAX-1] = '\0';
-	tbl = __ipa3_find_rt_tbl(rules->ip, rules->rt_tbl_name);
-	if (tbl == NULL || (tbl->cookie != IPA_RT_TBL_COOKIE)) {
-		IPAERR_RL("failed finding rt tbl name = %s\n",
-			rules->rt_tbl_name);
-		ret = -EINVAL;
-		goto bail;
-	}
-
-	if (!tbl->rule_cnt) {
-		IPAERR_RL("tbl->rule_cnt == 0");
-		ret = -EINVAL;
-		goto bail;
-	}
-
-	entry = ipa3_id_find(rules->add_after_hdl);
-	if (!entry) {
-		IPAERR_RL("failed finding rule %d in rt tbls\n",
-			rules->add_after_hdl);
-		ret = -EINVAL;
-		goto bail;
-	}
-
-	if (entry->cookie != IPA_RT_RULE_COOKIE) {
-		IPAERR_RL("Invalid cookie value =  %u rule %d in rt tbls\n",
-			entry->cookie, rules->add_after_hdl);
-		ret = -EINVAL;
-		goto bail;
-	}
-
-	if (entry->tbl != tbl) {
-		IPAERR_RL("given rt rule does not match the table\n");
-		ret = -EINVAL;
-		goto bail;
-	}
-
-	/*
-	 * do not allow any rule to be added at "default" routing
-	 * table
-	 */
-	if (!strcmp(tbl->name, IPA_DFLT_RT_TBL_NAME) &&
-		(tbl->rule_cnt > 0)) {
-		IPAERR_RL("cannot add rules to default rt table\n");
-		ret = -EINVAL;
-		goto bail;
-	}
-
-	/*
-	 * we add all rules one after the other, if one insertion fails, it cuts
-	 * the chain (all following will receive fail status) following calls to
-	 * __ipa_add_rt_rule_after will fail (entry == NULL)
-	 */
-
-	for (i = 0; i < rules->num_rules; i++) {
-		/* if hashing not supported, all tables are non-hash tables*/
-		if (ipa3_ctx->ipa_fltrt_not_hashable)
-			rules->rules[i].rule.hashable = false;
-		__ipa_convert_rt_rule_in(
-				rules->rules[i].rule, &rule);
-		if (__ipa_add_rt_rule_after(tbl,
-					&rule,
-					&rules->rules[i].rt_rule_hdl,
-					&entry)) {
-			IPAERR_RL("failed to add rt rule %d\n", i);
-			rules->rules[i].status = IPA_RT_STATUS_OF_ADD_FAILED;
-		} else {
-			rules->rules[i].status = 0;
-			__ipa_convert_rt_rule_out(rule, &rules->rules[i].rule);
-		}
-	}
-
-	if (rules->commit)
-		if (ipa3_ctx->ctrl->ipa3_commit_rt(rules->ip)) {
-			IPAERR_RL("failed to commit\n");
-			ret = -EPERM;
-			goto bail;
-		}
-
-	ret = 0;
-	goto bail;
-
-bail:
-	mutex_unlock(&ipa3_ctx->lock);
-	return ret;
-}
-
-/**
- * ipa3_add_rt_rule_after_v2() - Add the given routing rules
- * after the specified rule to SW and optionally commit to IPA
- * HW
- * @rules:	[inout] set of routing rules to add + handle where to add
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_add_rt_rule_after_v2(struct ipa_ioc_add_rt_rule_after_v2
-	*rules)
-{
-	int i;
-	int ret = 0;
-	struct ipa3_rt_tbl *tbl = NULL;
-	struct ipa3_rt_entry *entry = NULL;
-
-	if (rules == NULL || rules->num_rules == 0 || rules->ip >= IPA_IP_MAX) {
-		IPAERR_RL("bad param\n");
-		return -EINVAL;
-	}
-
-	mutex_lock(&ipa3_ctx->lock);
-	rules->rt_tbl_name[IPA_RESOURCE_NAME_MAX-1] = '\0';
-	tbl = __ipa3_find_rt_tbl(rules->ip, rules->rt_tbl_name);
-	if (tbl == NULL || (tbl->cookie != IPA_RT_TBL_COOKIE)) {
-		IPAERR_RL("failed finding rt tbl name = %s\n",
-			rules->rt_tbl_name);
-		ret = -EINVAL;
-		goto bail;
-	}
-
-	if (!tbl->rule_cnt) {
-		IPAERR_RL("tbl->rule_cnt == 0");
-		ret = -EINVAL;
-		goto bail;
-	}
-
-	entry = ipa3_id_find(rules->add_after_hdl);
-	if (!entry) {
-		IPAERR_RL("failed finding rule %d in rt tbls\n",
-			rules->add_after_hdl);
-		ret = -EINVAL;
-		goto bail;
-	}
-
-	if (entry->cookie != IPA_RT_RULE_COOKIE) {
-		IPAERR_RL("Invalid cookie value =  %u rule %d in rt tbls\n",
-			entry->cookie, rules->add_after_hdl);
-		ret = -EINVAL;
-		goto bail;
-	}
-
-	if (entry->tbl != tbl) {
-		IPAERR_RL("given rt rule does not match the table\n");
-		ret = -EINVAL;
-		goto bail;
-	}
-
-	/*
-	 * do not allow any rule to be added at "default" routing
-	 * table
-	 */
-	if (!strcmp(tbl->name, IPA_DFLT_RT_TBL_NAME) &&
-		(tbl->rule_cnt > 0)) {
-		IPAERR_RL("cannot add rules to default rt table\n");
-		ret = -EINVAL;
-		goto bail;
-	}
-
-	/*
-	 * we add all rules one after the other, if one insertion fails, it cuts
-	 * the chain (all following will receive fail status) following calls to
-	 * __ipa_add_rt_rule_after will fail (entry == NULL)
-	 */
-
-	for (i = 0; i < rules->num_rules; i++) {
-		/* if hashing not supported, all tables are non-hash tables*/
-		if (ipa3_ctx->ipa_fltrt_not_hashable)
-			((struct ipa_rt_rule_add_i *)
-			rules->rules)[i].rule.hashable = false;
-		if (__ipa_add_rt_rule_after(tbl,
-					&(((struct ipa_rt_rule_add_i *)
-					rules->rules)[i].rule),
-					&(((struct ipa_rt_rule_add_i *)
-					rules->rules)[i].rt_rule_hdl),
-					&entry)) {
-			IPAERR_RL("failed to add rt rule %d\n", i);
-			((struct ipa_rt_rule_add_i *)
-			rules->rules)[i].status = IPA_RT_STATUS_OF_ADD_FAILED;
-		} else {
-			((struct ipa_rt_rule_add_i *)
-			rules->rules)[i].status = 0;
-		}
-	}
-
-	if (rules->commit)
-		if (ipa3_ctx->ctrl->ipa3_commit_rt(rules->ip)) {
-			IPAERR_RL("failed to commit\n");
-			ret = -EPERM;
-			goto bail;
-		}
-
-	ret = 0;
-	goto bail;
-
-bail:
-	mutex_unlock(&ipa3_ctx->lock);
-	return ret;
-}
-
-int __ipa3_del_rt_rule(u32 rule_hdl)
-{
-	struct ipa3_rt_entry *entry;
-	int id;
-	struct ipa3_hdr_entry *hdr_entry;
-	struct ipa3_hdr_proc_ctx_entry *hdr_proc_entry;
-
-	entry = ipa3_id_find(rule_hdl);
-
-	if (entry == NULL) {
-		IPAERR_RL("lookup failed\n");
-		return -EINVAL;
-	}
-
-	if (entry->cookie != IPA_RT_RULE_COOKIE) {
-		IPAERR_RL("bad params\n");
-		return -EINVAL;
-	}
-
-	if (!strcmp(entry->tbl->name, IPA_DFLT_RT_TBL_NAME)) {
-		IPADBG("Deleting rule from default rt table idx=%u\n",
-			entry->tbl->idx);
-		if (entry->tbl->rule_cnt == 1) {
-			IPAERR_RL("Default tbl last rule cannot be deleted\n");
-			return -EINVAL;
-		}
-	}
-
-	/* Adding check to confirm still
-	 * header entry present in header table or not
-	 */
-
-	if (entry->hdr) {
-		hdr_entry = ipa3_id_find(entry->rule.hdr_hdl);
-		if (!hdr_entry || hdr_entry->cookie != IPA_HDR_COOKIE) {
-			IPAERR_RL("Header entry already deleted\n");
-			return -EINVAL;
-		}
-	} else if (entry->proc_ctx) {
-		hdr_proc_entry = ipa3_id_find(entry->rule.hdr_proc_ctx_hdl);
-		if (!hdr_proc_entry ||
-			hdr_proc_entry->cookie != IPA_PROC_HDR_COOKIE) {
-			IPAERR_RL("Proc header entry already deleted\n");
-			return -EINVAL;
-		}
-	}
-
-	if (entry->hdr)
-		__ipa3_release_hdr(entry->hdr->id);
-	else if (entry->proc_ctx)
-		__ipa3_release_hdr_proc_ctx(entry->proc_ctx->id);
-	list_del(&entry->link);
-	entry->tbl->rule_cnt--;
-	IPADBG("del rt rule tbl_idx=%d rule_cnt=%d rule_id=%d\n ref_cnt=%u",
-		entry->tbl->idx, entry->tbl->rule_cnt,
-		entry->rule_id, entry->tbl->ref_cnt);
-		/* if rule id was allocated from idr, remove it */
-	if (!entry->rule_id_valid)
-		idr_remove(entry->tbl->rule_ids, entry->rule_id);
-	if (entry->tbl->rule_cnt == 0 && entry->tbl->ref_cnt == 0) {
-		if (__ipa_del_rt_tbl(entry->tbl))
-			IPAERR_RL("fail to del RT tbl\n");
-	}
-	entry->cookie = 0;
-	id = entry->id;
-	kmem_cache_free(ipa3_ctx->rt_rule_cache, entry);
-
-	/* remove the handle from the database */
-	ipa3_id_remove(id);
-
-	return 0;
-}
-
-/**
- * ipa3_del_rt_rule() - Remove the specified routing rules to SW and optionally
- * commit to IPA HW
- * @hdls:	[inout] set of routing rules to delete
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_del_rt_rule(struct ipa_ioc_del_rt_rule *hdls)
-{
-	int i;
-	int ret;
-
-	if (hdls == NULL || hdls->num_hdls == 0 || hdls->ip >= IPA_IP_MAX) {
-		IPAERR_RL("bad param\n");
-		return -EINVAL;
-	}
-
-	mutex_lock(&ipa3_ctx->lock);
-	for (i = 0; i < hdls->num_hdls; i++) {
-		if (__ipa3_del_rt_rule(hdls->hdl[i].hdl)) {
-			IPAERR_RL("failed to del rt rule %i\n", i);
-			hdls->hdl[i].status = IPA_RT_STATUS_OF_DEL_FAILED;
-		} else {
-			hdls->hdl[i].status = 0;
-		}
-	}
-
-	if (hdls->commit)
-		if (ipa3_ctx->ctrl->ipa3_commit_rt(hdls->ip)) {
-			ret = -EPERM;
-			goto bail;
-		}
-
-	ret = 0;
-bail:
-	mutex_unlock(&ipa3_ctx->lock);
-	return ret;
-}
-
-/**
- * ipa_commit_rt_rule() - Commit the current SW routing table of specified type
- * to IPA HW
- * @ip:	The family of routing tables
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_commit_rt(enum ipa_ip_type ip)
-{
-	int ret;
-
-	if (ip >= IPA_IP_MAX) {
-		IPAERR_RL("bad param\n");
-		return -EINVAL;
-	}
-
-	/*
-	 * issue a commit on the filtering module of same IP type since
-	 * filtering rules point to routing tables
-	 */
-	if (ipa3_commit_flt(ip))
-		return -EPERM;
-
-	mutex_lock(&ipa3_ctx->lock);
-	if (ipa3_ctx->ctrl->ipa3_commit_rt(ip)) {
-		ret = -EPERM;
-		goto bail;
-	}
-
-	ret = 0;
-bail:
-	mutex_unlock(&ipa3_ctx->lock);
-	return ret;
-}
-
-/**
- * ipa3_reset_rt() - reset the current SW routing table of specified type
- * (does not commit to HW)
- * @ip:			[in] The family of routing tables
- * @user_only:	[in] indicate delete rules installed by userspace
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_reset_rt(enum ipa_ip_type ip, bool user_only)
-{
-	struct ipa3_rt_tbl *tbl;
-	struct ipa3_rt_tbl *tbl_next;
-	struct ipa3_rt_tbl_set *set;
-	struct ipa3_rt_entry *rule;
-	struct ipa3_rt_entry *rule_next;
-	struct ipa3_rt_tbl_set *rset;
-	struct ipa3_hdr_entry *hdr_entry;
-	struct ipa3_hdr_proc_ctx_entry *hdr_proc_entry;
-	u32 apps_start_idx;
-	int id;
-	bool tbl_user = false;
-
-	if (ip >= IPA_IP_MAX) {
-		IPAERR_RL("bad param\n");
-		return -EINVAL;
-	}
-
-	if (ip == IPA_IP_v4)
-		apps_start_idx =
-			IPA_MEM_PART(v4_apps_rt_index_lo);
-	else
-		apps_start_idx =
-			IPA_MEM_PART(v6_apps_rt_index_lo);
-
-	/*
-	 * issue a reset on the filtering module of same IP type since
-	 * filtering rules point to routing tables
-	 */
-	if (ipa3_reset_flt(ip, user_only))
-		IPAERR_RL("fail to reset flt ip=%d\n", ip);
-
-	set = &ipa3_ctx->rt_tbl_set[ip];
-	rset = &ipa3_ctx->reap_rt_tbl_set[ip];
-	mutex_lock(&ipa3_ctx->lock);
-	IPADBG("reset rt ip=%d\n", ip);
-	list_for_each_entry_safe(tbl, tbl_next, &set->head_rt_tbl_list, link) {
-		tbl_user = false;
-		list_for_each_entry_safe(rule, rule_next,
-					 &tbl->head_rt_rule_list, link) {
-			if (ipa3_id_find(rule->id) == NULL) {
-				WARN_ON_RATELIMIT_IPA(1);
-				mutex_unlock(&ipa3_ctx->lock);
-				return -EFAULT;
-			}
-
-			/* indicate if tbl used for user-specified rules*/
-			if (rule->ipacm_installed) {
-				IPADBG("tbl_user %d, tbl-index %d\n",
-				tbl_user, tbl->id);
-				tbl_user = true;
-			}
-			/*
-			 * for the "default" routing tbl, remove all but the
-			 *  last rule
-			 */
-			if (tbl->idx == apps_start_idx && tbl->rule_cnt == 1)
-				continue;
-
-			if (!user_only ||
-				rule->ipacm_installed) {
-				list_del(&rule->link);
-				if (rule->hdr) {
-					hdr_entry = ipa3_id_find(
-							rule->rule.hdr_hdl);
-					if (!hdr_entry ||
-					hdr_entry->cookie != IPA_HDR_COOKIE) {
-						mutex_unlock(&ipa3_ctx->lock);
-						IPAERR_RL(
-						"Header already deleted\n");
-						return -EINVAL;
-					}
-				} else if (rule->proc_ctx) {
-					hdr_proc_entry =
-						ipa3_id_find(
-						rule->rule.hdr_proc_ctx_hdl);
-					if (!hdr_proc_entry ||
-						hdr_proc_entry->cookie !=
-							IPA_PROC_HDR_COOKIE) {
-						mutex_unlock(&ipa3_ctx->lock);
-						IPAERR_RL(
-						"Proc entry already deleted\n");
-						return -EINVAL;
-					}
-				}
-				tbl->rule_cnt--;
-				if (rule->hdr)
-					__ipa3_release_hdr(rule->hdr->id);
-				else if (rule->proc_ctx)
-					__ipa3_release_hdr_proc_ctx(
-						rule->proc_ctx->id);
-				rule->cookie = 0;
-				if (!rule->rule_id_valid)
-					idr_remove(tbl->rule_ids,
-						rule->rule_id);
-				id = rule->id;
-				kmem_cache_free(ipa3_ctx->rt_rule_cache, rule);
-
-				/* remove the handle from the database */
-				ipa3_id_remove(id);
-			}
-		}
-
-		if (ipa3_id_find(tbl->id) == NULL) {
-			WARN_ON_RATELIMIT_IPA(1);
-			mutex_unlock(&ipa3_ctx->lock);
-			return -EFAULT;
-		}
-		id = tbl->id;
-
-		/* do not remove the "default" routing tbl which has index 0 */
-		if (tbl->idx != apps_start_idx) {
-			if (!user_only || tbl_user) {
-				tbl->rule_ids = NULL;
-				if (tbl->in_sys[IPA_RULE_HASHABLE] ||
-					tbl->in_sys[IPA_RULE_NON_HASHABLE]) {
-					list_move(&tbl->link,
-						&rset->head_rt_tbl_list);
-					clear_bit(tbl->idx,
-					  &ipa3_ctx->rt_idx_bitmap[ip]);
-					set->tbl_cnt--;
-					IPADBG("rst tbl_idx=%d cnt=%d\n",
-						tbl->idx, set->tbl_cnt);
-				} else {
-					list_del(&tbl->link);
-					set->tbl_cnt--;
-					clear_bit(tbl->idx,
-					  &ipa3_ctx->rt_idx_bitmap[ip]);
-					IPADBG("rst rt tbl_idx=%d tbl_cnt=%d\n",
-						tbl->idx, set->tbl_cnt);
-					kmem_cache_free(ipa3_ctx->rt_tbl_cache,
-						tbl);
-				}
-				/* remove the handle from the database */
-				ipa3_id_remove(id);
-			}
-		}
-	}
-
-	/* commit the change to IPA-HW */
-	if (ipa3_ctx->ctrl->ipa3_commit_rt(IPA_IP_v4) ||
-		ipa3_ctx->ctrl->ipa3_commit_rt(IPA_IP_v6)) {
-		IPAERR("fail to commit rt-rule\n");
-		WARN_ON_RATELIMIT_IPA(1);
-		mutex_unlock(&ipa3_ctx->lock);
-		return -EPERM;
-	}
-	mutex_unlock(&ipa3_ctx->lock);
-
-	return 0;
-}
-
-/**
- * ipa3_get_rt_tbl() - lookup the specified routing table and return handle if
- * it exists, if lookup succeeds the routing table ref cnt is increased
- * @lookup:	[inout] routing table to lookup and its handle
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- *	Caller should call ipa3_put_rt_tbl later if this function succeeds
- */
-int ipa3_get_rt_tbl(struct ipa_ioc_get_rt_tbl *lookup)
-{
-	struct ipa3_rt_tbl *entry;
-	int result = -EFAULT;
-
-	if (lookup == NULL || lookup->ip >= IPA_IP_MAX) {
-		IPAERR_RL("bad param\n");
-		return -EINVAL;
-	}
-	mutex_lock(&ipa3_ctx->lock);
-	lookup->name[IPA_RESOURCE_NAME_MAX-1] = '\0';
-	entry = __ipa3_find_rt_tbl(lookup->ip, lookup->name);
-	if (entry && entry->cookie == IPA_RT_TBL_COOKIE) {
-		if (entry->ref_cnt == U32_MAX) {
-			IPAERR_RL("fail: ref count crossed limit\n");
-			goto ret;
-		}
-		entry->ref_cnt++;
-		lookup->hdl = entry->id;
-
-		/* commit for get */
-		if (ipa3_ctx->ctrl->ipa3_commit_rt(lookup->ip))
-			IPAERR_RL("fail to commit RT tbl\n");
-
-		result = 0;
-	}
-
-ret:
-	mutex_unlock(&ipa3_ctx->lock);
-
-	return result;
-}
-
-/**
- * ipa3_put_rt_tbl() - Release the specified routing table handle
- * @rt_tbl_hdl:	[in] the routing table handle to release
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_put_rt_tbl(u32 rt_tbl_hdl)
-{
-	struct ipa3_rt_tbl *entry;
-	enum ipa_ip_type ip = IPA_IP_MAX;
-	int result = 0;
-
-	mutex_lock(&ipa3_ctx->lock);
-	entry = ipa3_id_find(rt_tbl_hdl);
-	if (entry == NULL) {
-		IPAERR_RL("lookup failed\n");
-		result = -EINVAL;
-		goto ret;
-	}
-
-	if ((entry->cookie != IPA_RT_TBL_COOKIE) || entry->ref_cnt == 0) {
-		IPAERR_RL("bad params\n");
-		result = -EINVAL;
-		goto ret;
-	}
-
-	if (entry->set == &ipa3_ctx->rt_tbl_set[IPA_IP_v4])
-		ip = IPA_IP_v4;
-	else if (entry->set == &ipa3_ctx->rt_tbl_set[IPA_IP_v6])
-		ip = IPA_IP_v6;
-	else {
-		WARN_ON_RATELIMIT_IPA(1);
-		result = -EINVAL;
-		goto ret;
-	}
-
-	entry->ref_cnt--;
-	if (entry->ref_cnt == 0 && entry->rule_cnt == 0) {
-		IPADBG("zero ref_cnt, delete rt tbl (idx=%u)\n",
-			entry->idx);
-		if (__ipa_del_rt_tbl(entry))
-			IPAERR_RL("fail to del RT tbl\n");
-		/* commit for put */
-		if (ipa3_ctx->ctrl->ipa3_commit_rt(ip))
-			IPAERR_RL("fail to commit RT tbl\n");
-	}
-
-	result = 0;
-
-ret:
-	mutex_unlock(&ipa3_ctx->lock);
-
-	return result;
-}
-
-
-static int __ipa_mdfy_rt_rule(struct ipa_rt_rule_mdfy_i *rtrule)
-{
-	struct ipa3_rt_entry *entry;
-	struct ipa3_hdr_entry *hdr = NULL;
-	struct ipa3_hdr_proc_ctx_entry *proc_ctx = NULL;
-	struct ipa3_hdr_entry *hdr_entry;
-	struct ipa3_hdr_proc_ctx_entry *hdr_proc_entry;
-
-	if (rtrule->rule.hdr_hdl) {
-		hdr = ipa3_id_find(rtrule->rule.hdr_hdl);
-		if ((hdr == NULL) || (hdr->cookie != IPA_HDR_COOKIE)) {
-			IPAERR_RL("rt rule does not point to valid hdr\n");
-			goto error;
-		}
-	} else if (rtrule->rule.hdr_proc_ctx_hdl) {
-		proc_ctx = ipa3_id_find(rtrule->rule.hdr_proc_ctx_hdl);
-		if ((proc_ctx == NULL) ||
-			(proc_ctx->cookie != IPA_PROC_HDR_COOKIE)) {
-			IPAERR_RL("rt rule does not point to valid proc ctx\n");
-			goto error;
-		}
-	}
-
-	entry = ipa3_id_find(rtrule->rt_rule_hdl);
-	if (entry == NULL) {
-		IPAERR_RL("lookup failed\n");
-		goto error;
-	}
-
-	if (entry->cookie != IPA_RT_RULE_COOKIE) {
-		IPAERR_RL("bad params\n");
-		goto error;
-	}
-
-	if (!strcmp(entry->tbl->name, IPA_DFLT_RT_TBL_NAME)) {
-		IPAERR_RL("Default tbl rule cannot be modified\n");
-		return -EINVAL;
-	}
-	/* Adding check to confirm still
-	 * header entry present in header table or not
-	 */
-
-	if (entry->hdr) {
-		hdr_entry = ipa3_id_find(entry->rule.hdr_hdl);
-		if (!hdr_entry || hdr_entry->cookie != IPA_HDR_COOKIE) {
-			IPAERR_RL("Header entry already deleted\n");
-			return -EPERM;
-		}
-	} else if (entry->proc_ctx) {
-		hdr_proc_entry = ipa3_id_find(entry->rule.hdr_proc_ctx_hdl);
-		if (!hdr_proc_entry ||
-			hdr_proc_entry->cookie != IPA_PROC_HDR_COOKIE) {
-			IPAERR_RL("Proc header entry already deleted\n");
-			return -EPERM;
-		}
-	}
-
-	if (entry->hdr)
-		entry->hdr->ref_cnt--;
-	if (entry->proc_ctx)
-		entry->proc_ctx->ref_cnt--;
-
-	entry->rule = rtrule->rule;
-	entry->hdr = hdr;
-	entry->proc_ctx = proc_ctx;
-
-	if (entry->hdr)
-		entry->hdr->ref_cnt++;
-	if (entry->proc_ctx)
-		entry->proc_ctx->ref_cnt++;
-
-	entry->hw_len = 0;
-	entry->prio = 0;
-	if (rtrule->rule.enable_stats)
-		entry->cnt_idx = rtrule->rule.cnt_idx;
-	else
-		entry->cnt_idx = 0;
-	return 0;
-
-error:
-	return -EPERM;
-}
-
-/**
- * ipa3_mdfy_rt_rule() - Modify the specified routing rules in SW and optionally
- * commit to IPA HW
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_mdfy_rt_rule(struct ipa_ioc_mdfy_rt_rule *hdls)
-{
-	int i;
-	int result;
-	struct ipa_rt_rule_mdfy_i rule;
-
-	if (hdls == NULL || hdls->num_rules == 0 || hdls->ip >= IPA_IP_MAX) {
-		IPAERR_RL("bad param\n");
-		return -EINVAL;
-	}
-
-	mutex_lock(&ipa3_ctx->lock);
-	for (i = 0; i < hdls->num_rules; i++) {
-		/* if hashing not supported, all tables are non-hash tables*/
-		if (ipa3_ctx->ipa_fltrt_not_hashable)
-			hdls->rules[i].rule.hashable = false;
-		__ipa_convert_rt_mdfy_in(hdls->rules[i], &rule);
-		if (__ipa_mdfy_rt_rule(&rule)) {
-			IPAERR_RL("failed to mdfy rt rule %i\n", i);
-			hdls->rules[i].status = IPA_RT_STATUS_OF_MDFY_FAILED;
-		} else {
-			hdls->rules[i].status = 0;
-			__ipa_convert_rt_mdfy_out(rule, &hdls->rules[i]);
-		}
-	}
-
-	if (hdls->commit)
-		if (ipa3_ctx->ctrl->ipa3_commit_rt(hdls->ip)) {
-			result = -EPERM;
-			goto bail;
-		}
-	result = 0;
-bail:
-	mutex_unlock(&ipa3_ctx->lock);
-
-	return result;
-}
-
-/**
- * ipa3_mdfy_rt_rule_v2() - Modify the specified routing rules
- * in SW and optionally commit to IPA HW
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_mdfy_rt_rule_v2(struct ipa_ioc_mdfy_rt_rule_v2 *hdls)
-{
-	int i;
-	int result;
-
-	if (hdls == NULL || hdls->num_rules == 0 || hdls->ip >= IPA_IP_MAX) {
-		IPAERR_RL("bad param\n");
-		return -EINVAL;
-	}
-
-	mutex_lock(&ipa3_ctx->lock);
-	for (i = 0; i < hdls->num_rules; i++) {
-		/* if hashing not supported, all tables are non-hash tables*/
-		if (ipa3_ctx->ipa_fltrt_not_hashable)
-			((struct ipa_rt_rule_mdfy_i *)
-			hdls->rules)[i].rule.hashable = false;
-		if (__ipa_mdfy_rt_rule(&(((struct ipa_rt_rule_mdfy_i *)
-			hdls->rules)[i]))) {
-			IPAERR_RL("failed to mdfy rt rule %i\n", i);
-			((struct ipa_rt_rule_mdfy_i *)
-			hdls->rules)[i].status = IPA_RT_STATUS_OF_MDFY_FAILED;
-		} else {
-			((struct ipa_rt_rule_mdfy_i *)
-			hdls->rules)[i].status = 0;
-		}
-	}
-
-	if (hdls->commit)
-		if (ipa3_ctx->ctrl->ipa3_commit_rt(hdls->ip)) {
-			result = -EPERM;
-			goto bail;
-		}
-	result = 0;
-bail:
-	mutex_unlock(&ipa3_ctx->lock);
-
-	return result;
-}
-
-/**
- * ipa3_set_rt_tuple_mask() - Sets the rt tuple masking for the given tbl
- *  table index must be for AP EP (not modem)
- *  updates the the routing masking values without changing the flt ones.
- *
- * @tbl_idx: routing table index to configure the tuple masking
- * @tuple: the tuple members masking
- * Returns:	0 on success, negative on failure
- *
- */
-int ipa3_set_rt_tuple_mask(int tbl_idx, struct ipahal_reg_hash_tuple *tuple)
-{
-	struct ipahal_reg_fltrt_hash_tuple fltrt_tuple;
-
-	if (!tuple) {
-		IPAERR_RL("bad tuple\n");
-		return -EINVAL;
-	}
-
-	if (tbl_idx >=
-		max(IPA_MEM_PART(v6_rt_num_index),
-		IPA_MEM_PART(v4_rt_num_index)) ||
-		tbl_idx < 0) {
-		IPAERR_RL("bad table index\n");
-		return -EINVAL;
-	}
-
-	if (tbl_idx >= IPA_MEM_PART(v4_modem_rt_index_lo) &&
-		tbl_idx <= IPA_MEM_PART(v4_modem_rt_index_hi)) {
-		IPAERR_RL("cannot configure modem v4 rt tuple by AP\n");
-		return -EINVAL;
-	}
-
-	if (tbl_idx >= IPA_MEM_PART(v6_modem_rt_index_lo) &&
-		tbl_idx <= IPA_MEM_PART(v6_modem_rt_index_hi)) {
-		IPAERR_RL("cannot configure modem v6 rt tuple by AP\n");
-		return -EINVAL;
-	}
-
-	ipahal_read_reg_n_fields(IPA_ENDP_FILTER_ROUTER_HSH_CFG_n,
-		tbl_idx, &fltrt_tuple);
-	fltrt_tuple.rt = *tuple;
-	ipahal_write_reg_n_fields(IPA_ENDP_FILTER_ROUTER_HSH_CFG_n,
-		tbl_idx, &fltrt_tuple);
-
-	return 0;
-}
-
-/**
- * ipa3_rt_read_tbl_from_hw() -Read routing table from IPA HW
- * @tbl_idx: routing table index
- * @ip_type: IPv4 or IPv6 table
- * @hashable: hashable or non-hashable table
- * @entry: array to fill the table entries
- * @num_entry: number of entries in entry array. set by the caller to indicate
- *  entry array size. Then set by this function as an output parameter to
- *  indicate the number of entries in the array
- *
- * This function reads the routing table from IPA SRAM and prepares an array
- * of entries. This function is mainly used for debugging purposes.
- *
- * If empty table or Modem Apps table, zero entries will be returned.
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa3_rt_read_tbl_from_hw(u32 tbl_idx, enum ipa_ip_type ip_type,
-	bool hashable, struct ipahal_rt_rule_entry entry[], int *num_entry)
-{
-	void *ipa_sram_mmio;
-	u64 hdr_base_ofst;
-	int res = 0;
-	u64 tbl_addr;
-	bool is_sys;
-	struct ipa_mem_buffer *sys_tbl_mem;
-	u8 *rule_addr;
-	int rule_idx;
-
-	IPADBG_LOW("tbl_idx=%d ip_t=%d hash=%d entry=0x%pK num_entry=0x%pK\n",
-		tbl_idx, ip_type, hashable, entry, num_entry);
-
-	/*
-	 * SRAM memory not allocated to hash tables. Reading of hash table
-	 * rules operation not supported
-	 */
-	if (hashable && ipa3_ctx->ipa_fltrt_not_hashable) {
-		IPADBG("Reading hashable rules not supported\n");
-		*num_entry = 0;
-		return 0;
-	}
-
-	if (ip_type == IPA_IP_v4 && tbl_idx >= IPA_MEM_PART(v4_rt_num_index)) {
-		IPAERR_RL("Invalid params\n");
-		return -EFAULT;
-	}
-
-	if (ip_type == IPA_IP_v6 && tbl_idx >= IPA_MEM_PART(v6_rt_num_index)) {
-		IPAERR_RL("Invalid params\n");
-		return -EFAULT;
-	}
-
-	/* map IPA SRAM */
-	ipa_sram_mmio = ioremap(ipa3_ctx->ipa_wrapper_base +
-		ipa3_ctx->ctrl->ipa_reg_base_ofst +
-		ipahal_get_reg_n_ofst(IPA_SW_AREA_RAM_DIRECT_ACCESS_n,
-			ipa3_ctx->smem_restricted_bytes / 4),
-		ipa3_ctx->smem_sz);
-	if (!ipa_sram_mmio) {
-		IPAERR("fail to ioremap IPA SRAM\n");
-		return -ENOMEM;
-	}
-
-	memset(entry, 0, sizeof(*entry) * (*num_entry));
-	if (hashable) {
-		if (ip_type == IPA_IP_v4)
-			hdr_base_ofst =
-				IPA_MEM_PART(v4_rt_hash_ofst);
-		else
-			hdr_base_ofst =
-				IPA_MEM_PART(v6_rt_hash_ofst);
-	} else {
-		if (ip_type == IPA_IP_v4)
-			hdr_base_ofst =
-				IPA_MEM_PART(v4_rt_nhash_ofst);
-		else
-			hdr_base_ofst =
-				IPA_MEM_PART(v6_rt_nhash_ofst);
-	}
-
-	IPADBG_LOW("hdr_base_ofst=0x%llx\n", hdr_base_ofst);
-
-	res = ipahal_fltrt_read_addr_from_hdr(ipa_sram_mmio + hdr_base_ofst,
-		tbl_idx, &tbl_addr, &is_sys);
-	if (res) {
-		IPAERR("failed to read table address from header structure\n");
-		goto bail;
-	}
-	IPADBG_LOW("rt tbl %d: tbl_addr=0x%llx is_sys=%d\n",
-		tbl_idx, tbl_addr, is_sys);
-	if (!tbl_addr) {
-		IPAERR("invalid rt tbl addr\n");
-		res = -EFAULT;
-		goto bail;
-	}
-
-	/* for tables which reside in DDR access it from the virtual memory */
-	if (is_sys) {
-		struct ipa3_rt_tbl_set *set;
-		struct ipa3_rt_tbl *tbl;
-
-		set = &ipa3_ctx->rt_tbl_set[ip_type];
-		rule_addr = NULL;
-		list_for_each_entry(tbl, &set->head_rt_tbl_list, link) {
-			if (tbl->idx == tbl_idx) {
-				sys_tbl_mem = &(tbl->curr_mem[hashable ?
-					IPA_RULE_HASHABLE :
-					IPA_RULE_NON_HASHABLE]);
-				if (sys_tbl_mem->phys_base &&
-					sys_tbl_mem->phys_base != tbl_addr) {
-					IPAERR("mismatch:parsed=%llx sw=%pad\n"
-						, tbl_addr,
-						&sys_tbl_mem->phys_base);
-				}
-				if (sys_tbl_mem->phys_base)
-					rule_addr = sys_tbl_mem->base;
-				else
-					rule_addr = NULL;
-			}
-		}
-	} else {
-		rule_addr = ipa_sram_mmio + hdr_base_ofst + tbl_addr;
-	}
-
-	IPADBG_LOW("First rule addr 0x%pK\n", rule_addr);
-
-	if (!rule_addr) {
-		/* Modem table in system memory or empty table */
-		*num_entry = 0;
-		goto bail;
-	}
-
-	rule_idx = 0;
-	while (rule_idx < *num_entry) {
-		res = ipahal_rt_parse_hw_rule(rule_addr, &entry[rule_idx]);
-		if (res) {
-			IPAERR("failed parsing rt rule\n");
-			goto bail;
-		}
-
-		IPADBG_LOW("rule_size=%d\n", entry[rule_idx].rule_size);
-		if (!entry[rule_idx].rule_size)
-			break;
-
-		rule_addr += entry[rule_idx].rule_size;
-		rule_idx++;
-	}
-	*num_entry = rule_idx;
-bail:
-	iounmap(ipa_sram_mmio);
-	return res;
-}

ipa/ipa_v3/ipa_trace.h

@@ -1,183 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2012-2019, The Linux Foundation. All rights reserved.
- */
-
-#undef TRACE_SYSTEM
-#define TRACE_SYSTEM ipa
-#define TRACE_INCLUDE_FILE ipa_trace
-
-#if !defined(_IPA_TRACE_H) || defined(TRACE_HEADER_MULTI_READ)
-#define _IPA_TRACE_H
-
-#include <linux/tracepoint.h>
-
-TRACE_EVENT(
-	intr_to_poll3,
-
-	TP_PROTO(unsigned long client),
-
-	TP_ARGS(client),
-
-	TP_STRUCT__entry(
-		__field(unsigned long,	client)
-	),
-
-	TP_fast_assign(
-		__entry->client = client;
-	),
-
-	TP_printk("client=%lu", __entry->client)
-);
-
-TRACE_EVENT(
-	poll_to_intr3,
-
-	TP_PROTO(unsigned long client),
-
-	TP_ARGS(client),
-
-	TP_STRUCT__entry(
-		__field(unsigned long,	client)
-	),
-
-	TP_fast_assign(
-		__entry->client = client;
-	),
-
-	TP_printk("client=%lu", __entry->client)
-);
-
-TRACE_EVENT(
-	idle_sleep_enter3,
-
-	TP_PROTO(unsigned long client),
-
-	TP_ARGS(client),
-
-	TP_STRUCT__entry(
-		__field(unsigned long,	client)
-	),
-
-	TP_fast_assign(
-		__entry->client = client;
-	),
-
-	TP_printk("client=%lu", __entry->client)
-);
-
-TRACE_EVENT(
-	idle_sleep_exit3,
-
-	TP_PROTO(unsigned long client),
-
-	TP_ARGS(client),
-
-	TP_STRUCT__entry(
-		__field(unsigned long,	client)
-	),
-
-	TP_fast_assign(
-		__entry->client = client;
-	),
-
-	TP_printk("client=%lu", __entry->client)
-);
-
-TRACE_EVENT(
-	rmnet_ipa_netifni3,
-
-	TP_PROTO(unsigned long rx_pkt_cnt),
-
-	TP_ARGS(rx_pkt_cnt),
-
-	TP_STRUCT__entry(
-		__field(unsigned long,	rx_pkt_cnt)
-	),
-
-	TP_fast_assign(
-		__entry->rx_pkt_cnt = rx_pkt_cnt;
-	),
-
-	TP_printk("rx_pkt_cnt=%lu", __entry->rx_pkt_cnt)
-);
-
-TRACE_EVENT(
-	rmnet_ipa_netifrx3,
-
-	TP_PROTO(unsigned long rx_pkt_cnt),
-
-	TP_ARGS(rx_pkt_cnt),
-
-	TP_STRUCT__entry(
-		__field(unsigned long,	rx_pkt_cnt)
-	),
-
-	TP_fast_assign(
-		__entry->rx_pkt_cnt = rx_pkt_cnt;
-	),
-
-	TP_printk("rx_pkt_cnt=%lu", __entry->rx_pkt_cnt)
-);
-
-TRACE_EVENT(
-	rmnet_ipa_netif_rcv_skb3,
-
-	TP_PROTO(unsigned long rx_pkt_cnt),
-
-	TP_ARGS(rx_pkt_cnt),
-
-	TP_STRUCT__entry(
-		__field(unsigned long,	rx_pkt_cnt)
-	),
-
-	TP_fast_assign(
-		__entry->rx_pkt_cnt = rx_pkt_cnt;
-	),
-
-	TP_printk("rx_pkt_cnt=%lu", __entry->rx_pkt_cnt)
-);
-
-TRACE_EVENT(
-	ipa3_rx_poll_num,
-
-	TP_PROTO(int poll_num),
-
-	TP_ARGS(poll_num),
-
-	TP_STRUCT__entry(
-		__field(int,	poll_num)
-	),
-
-	TP_fast_assign(
-		__entry->poll_num = poll_num;
-	),
-
-	TP_printk("each_poll_aggr_pkt_num=%d", __entry->poll_num)
-);
-
-TRACE_EVENT(
-	ipa3_rx_poll_cnt,
-
-	TP_PROTO(int poll_num),
-
-	TP_ARGS(poll_num),
-
-	TP_STRUCT__entry(
-		__field(int,	poll_num)
-	),
-
-	TP_fast_assign(
-		__entry->poll_num = poll_num;
-	),
-
-	TP_printk("napi_overall_poll_pkt_cnt=%d", __entry->poll_num)
-);
-
-
-#endif /* _IPA_TRACE_H */
-
-/* This part must be outside protection */
-#undef TRACE_INCLUDE_PATH
-#define TRACE_INCLUDE_PATH ../../techpack/dataipa/ipa/ipa_v3
-#include <trace/define_trace.h>

ipa/ipa_v3/ipa_uc.c

@@ -1,1101 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2012-2019, The Linux Foundation. All rights reserved.
- */
-
-#include "ipa_i.h"
-#include <linux/delay.h>
-
-#define IPA_RAM_UC_SMEM_SIZE 128
-#define IPA_HW_INTERFACE_VERSION     0x2000
-#define IPA_PKT_FLUSH_TO_US 100
-#define IPA_UC_POLL_SLEEP_USEC 100
-#define IPA_UC_POLL_MAX_RETRY 10000
-
-#define IPA_UC_DBG_STATS_GET_PROT_ID(x) (0xff & ((x) >> 24))
-#define IPA_UC_DBG_STATS_GET_OFFSET(x) (0x00ffffff & (x))
-
-/**
- * Mailbox register to Interrupt HWP for CPU cmd
- * Usage of IPA_UC_MAILBOX_m_n doorbell instead of IPA_IRQ_EE_UC_0
- * due to HW limitation.
- *
- */
-#define IPA_CPU_2_HW_CMD_MBOX_m          0
-#define IPA_CPU_2_HW_CMD_MBOX_n         23
-
-/**
- * enum ipa3_cpu_2_hw_commands - Values that represent the commands from the CPU
- * IPA_CPU_2_HW_CMD_NO_OP : No operation is required.
- * IPA_CPU_2_HW_CMD_UPDATE_FLAGS : Update SW flags which defines the behavior
- *                                 of HW.
- * IPA_CPU_2_HW_CMD_DEBUG_RUN_TEST : Launch predefined test over HW.
- * IPA_CPU_2_HW_CMD_DEBUG_GET_INFO : Read HW internal debug information.
- * IPA_CPU_2_HW_CMD_ERR_FATAL : CPU instructs HW to perform error fatal
- *                              handling.
- * IPA_CPU_2_HW_CMD_CLK_GATE : CPU instructs HW to goto Clock Gated state.
- * IPA_CPU_2_HW_CMD_CLK_UNGATE : CPU instructs HW to goto Clock Ungated state.
- * IPA_CPU_2_HW_CMD_MEMCPY : CPU instructs HW to do memcopy using QMB.
- * IPA_CPU_2_HW_CMD_RESET_PIPE : Command to reset a pipe - SW WA for a HW bug.
- * IPA_CPU_2_HW_CMD_GSI_CH_EMPTY : Command to check for GSI channel emptiness.
- * IPA_CPU_2_HW_CMD_REMOTE_IPA_INFO: Command to store remote IPA Info
- */
-enum ipa3_cpu_2_hw_commands {
-	IPA_CPU_2_HW_CMD_NO_OP                     =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_COMMON, 0),
-	IPA_CPU_2_HW_CMD_UPDATE_FLAGS              =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_COMMON, 1),
-	IPA_CPU_2_HW_CMD_DEBUG_RUN_TEST            =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_COMMON, 2),
-	IPA_CPU_2_HW_CMD_DEBUG_GET_INFO            =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_COMMON, 3),
-	IPA_CPU_2_HW_CMD_ERR_FATAL                 =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_COMMON, 4),
-	IPA_CPU_2_HW_CMD_CLK_GATE                  =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_COMMON, 5),
-	IPA_CPU_2_HW_CMD_CLK_UNGATE                =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_COMMON, 6),
-	IPA_CPU_2_HW_CMD_MEMCPY                    =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_COMMON, 7),
-	IPA_CPU_2_HW_CMD_RESET_PIPE                =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_COMMON, 8),
-	IPA_CPU_2_HW_CMD_REG_WRITE                 =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_COMMON, 9),
-	IPA_CPU_2_HW_CMD_GSI_CH_EMPTY              =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_COMMON, 10),
-	IPA_CPU_2_HW_CMD_REMOTE_IPA_INFO           =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_COMMON, 11),
-};
-
-/**
- * enum ipa3_hw_2_cpu_responses -  Values that represent common HW responses
- *  to CPU commands.
- * @IPA_HW_2_CPU_RESPONSE_NO_OP : No operation response
- * @IPA_HW_2_CPU_RESPONSE_INIT_COMPLETED : HW shall send this command once
- *  boot sequence is completed and HW is ready to serve commands from CPU
- * @IPA_HW_2_CPU_RESPONSE_CMD_COMPLETED: Response to CPU commands
- * @IPA_HW_2_CPU_RESPONSE_DEBUG_GET_INFO : Response to
- *  IPA_CPU_2_HW_CMD_DEBUG_GET_INFO command
- */
-enum ipa3_hw_2_cpu_responses {
-	IPA_HW_2_CPU_RESPONSE_NO_OP          =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_COMMON, 0),
-	IPA_HW_2_CPU_RESPONSE_INIT_COMPLETED =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_COMMON, 1),
-	IPA_HW_2_CPU_RESPONSE_CMD_COMPLETED  =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_COMMON, 2),
-	IPA_HW_2_CPU_RESPONSE_DEBUG_GET_INFO =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_COMMON, 3),
-};
-
-/**
- * struct IpaHwMemCopyData_t - Structure holding the parameters
- * for IPA_CPU_2_HW_CMD_MEMCPY command.
- *
- * The parameters are passed as immediate params in the shared memory
- */
-struct IpaHwMemCopyData_t  {
-	u32 destination_addr;
-	u32 source_addr;
-	u32 dest_buffer_size;
-	u32 source_buffer_size;
-};
-
-/**
- * struct IpaHwRegWriteCmdData_t - holds the parameters for
- * IPA_CPU_2_HW_CMD_REG_WRITE command. Parameters are
- * sent as 64b immediate parameters.
- * @RegisterAddress: RG10 register address where the value needs to be written
- * @RegisterValue: 32-Bit value to be written into the register
- */
-struct IpaHwRegWriteCmdData_t {
-	u32 RegisterAddress;
-	u32 RegisterValue;
-};
-
-/**
- * union IpaHwCpuCmdCompletedResponseData_t - Structure holding the parameters
- * for IPA_HW_2_CPU_RESPONSE_CMD_COMPLETED response.
- * @originalCmdOp : The original command opcode
- * @status : 0 for success indication, otherwise failure
- * @responseData : 16b responseData
- *
- * Parameters are sent as 32b immediate parameters.
- */
-union IpaHwCpuCmdCompletedResponseData_t {
-	struct IpaHwCpuCmdCompletedResponseParams_t {
-		u32 originalCmdOp:8;
-		u32 status:8;
-		u32 responseData:16;
-	} __packed params;
-	u32 raw32b;
-} __packed;
-
-/**
- * union IpaHwUpdateFlagsCmdData_t - Structure holding the parameters for
- * IPA_CPU_2_HW_CMD_UPDATE_FLAGS command
- * @newFlags: SW flags defined the behavior of HW.
- *	This field is expected to be used as bitmask for enum ipa3_hw_flags
- */
-union IpaHwUpdateFlagsCmdData_t {
-	struct IpaHwUpdateFlagsCmdParams_t {
-		u32 newFlags;
-	} params;
-	u32 raw32b;
-};
-
-/**
- * union IpaHwChkChEmptyCmdData_t -  Structure holding the parameters for
- *  IPA_CPU_2_HW_CMD_GSI_CH_EMPTY command. Parameters are sent as 32b
- *  immediate parameters.
- * @ee_n : EE owner of the channel
- * @vir_ch_id : GSI virtual channel ID of the channel to checked of emptiness
- * @reserved_02_04 : Reserved
- */
-union IpaHwChkChEmptyCmdData_t {
-	struct IpaHwChkChEmptyCmdParams_t {
-		u8 ee_n;
-		u8 vir_ch_id;
-		u16 reserved_02_04;
-	} __packed params;
-	u32 raw32b;
-} __packed;
-
-
-/**
- * Structure holding the parameters for IPA_CPU_2_HW_CMD_REMOTE_IPA_INFO
- * command.
- * @remoteIPAAddr: 5G IPA address : uC proxies Q6 doorbell to this address
- * @mboxN: mbox on which Q6 will interrupt uC
- */
-struct IpaHwDbAddrInfo_t {
-	u32 remoteIPAAddr;
-	uint32_t mboxN;
-} __packed;
-
-static DEFINE_MUTEX(uc_loaded_nb_lock);
-static BLOCKING_NOTIFIER_HEAD(uc_loaded_notifier);
-
-struct ipa3_uc_hdlrs ipa3_uc_hdlrs[IPA_HW_NUM_FEATURES] = { { 0 } };
-
-const char *ipa_hw_error_str(enum ipa3_hw_errors err_type)
-{
-	const char *str;
-
-	switch (err_type) {
-	case IPA_HW_ERROR_NONE:
-		str = "IPA_HW_ERROR_NONE";
-		break;
-	case IPA_HW_INVALID_DOORBELL_ERROR:
-		str = "IPA_HW_INVALID_DOORBELL_ERROR";
-		break;
-	case IPA_HW_DMA_ERROR:
-		str = "IPA_HW_DMA_ERROR";
-		break;
-	case IPA_HW_FATAL_SYSTEM_ERROR:
-		str = "IPA_HW_FATAL_SYSTEM_ERROR";
-		break;
-	case IPA_HW_INVALID_OPCODE:
-		str = "IPA_HW_INVALID_OPCODE";
-		break;
-	case IPA_HW_INVALID_PARAMS:
-		str = "IPA_HW_INVALID_PARAMS";
-		break;
-	case IPA_HW_CONS_DISABLE_CMD_GSI_STOP_FAILURE:
-		str = "IPA_HW_CONS_DISABLE_CMD_GSI_STOP_FAILURE";
-		break;
-	case IPA_HW_PROD_DISABLE_CMD_GSI_STOP_FAILURE:
-		str = "IPA_HW_PROD_DISABLE_CMD_GSI_STOP_FAILURE";
-		break;
-	case IPA_HW_GSI_CH_NOT_EMPTY_FAILURE:
-		str = "IPA_HW_GSI_CH_NOT_EMPTY_FAILURE";
-		break;
-	default:
-		str = "INVALID ipa_hw_errors type";
-	}
-
-	return str;
-}
-
-static void ipa3_uc_save_dbg_stats(u32 size)
-{
-	u8 protocol_id;
-	u32 addr_offset;
-	void __iomem *mmio;
-
-	protocol_id = IPA_UC_DBG_STATS_GET_PROT_ID(
-		ipa3_ctx->uc_ctx.uc_sram_mmio->responseParams_1);
-	addr_offset = IPA_UC_DBG_STATS_GET_OFFSET(
-		ipa3_ctx->uc_ctx.uc_sram_mmio->responseParams_1);
-	mmio = ioremap(ipa3_ctx->ipa_wrapper_base +
-		addr_offset, sizeof(struct IpaHwRingStats_t) *
-		MAX_CH_STATS_SUPPORTED);
-	if (mmio == NULL) {
-		IPAERR("unexpected NULL mmio\n");
-		return;
-	}
-	switch (protocol_id) {
-	case IPA_HW_PROTOCOL_AQC:
-		break;
-	case IPA_HW_PROTOCOL_11ad:
-		break;
-	case IPA_HW_PROTOCOL_WDI:
-		ipa3_ctx->wdi2_ctx.dbg_stats.uc_dbg_stats_size = size;
-		ipa3_ctx->wdi2_ctx.dbg_stats.uc_dbg_stats_ofst = addr_offset;
-		ipa3_ctx->wdi2_ctx.dbg_stats.uc_dbg_stats_mmio = mmio;
-		break;
-	case IPA_HW_PROTOCOL_WDI3:
-		ipa3_ctx->wdi3_ctx.dbg_stats.uc_dbg_stats_size = size;
-		ipa3_ctx->wdi3_ctx.dbg_stats.uc_dbg_stats_ofst = addr_offset;
-		ipa3_ctx->wdi3_ctx.dbg_stats.uc_dbg_stats_mmio = mmio;
-		break;
-	case IPA_HW_PROTOCOL_ETH:
-		break;
-	case IPA_HW_PROTOCOL_MHIP:
-		ipa3_ctx->mhip_ctx.dbg_stats.uc_dbg_stats_size = size;
-		ipa3_ctx->mhip_ctx.dbg_stats.uc_dbg_stats_ofst = addr_offset;
-		ipa3_ctx->mhip_ctx.dbg_stats.uc_dbg_stats_mmio = mmio;
-		break;
-	case IPA_HW_PROTOCOL_USB:
-		ipa3_ctx->usb_ctx.dbg_stats.uc_dbg_stats_size = size;
-		ipa3_ctx->usb_ctx.dbg_stats.uc_dbg_stats_ofst = addr_offset;
-		ipa3_ctx->usb_ctx.dbg_stats.uc_dbg_stats_mmio = mmio;
-		break;
-	default:
-		IPAERR("unknown protocols %d\n", protocol_id);
-	}
-}
-
-static void ipa3_log_evt_hdlr(void)
-{
-	int i;
-
-	if (!ipa3_ctx->uc_ctx.uc_event_top_ofst) {
-		ipa3_ctx->uc_ctx.uc_event_top_ofst =
-			ipa3_ctx->uc_ctx.uc_sram_mmio->eventParams;
-		if (ipa3_ctx->uc_ctx.uc_event_top_ofst +
-			sizeof(struct IpaHwEventLogInfoData_t) >=
-			ipa3_ctx->ctrl->ipa_reg_base_ofst +
-			ipahal_get_reg_n_ofst(
-				IPA_SW_AREA_RAM_DIRECT_ACCESS_n, 0) +
-			ipa3_ctx->smem_sz) {
-			IPAERR("uc_top 0x%x outside SRAM\n",
-				ipa3_ctx->uc_ctx.uc_event_top_ofst);
-			goto bad_uc_top_ofst;
-		}
-
-		ipa3_ctx->uc_ctx.uc_event_top_mmio = ioremap(
-			ipa3_ctx->ipa_wrapper_base +
-			ipa3_ctx->uc_ctx.uc_event_top_ofst,
-			sizeof(struct IpaHwEventLogInfoData_t));
-		if (!ipa3_ctx->uc_ctx.uc_event_top_mmio) {
-			IPAERR("fail to ioremap uc top\n");
-			goto bad_uc_top_ofst;
-		}
-
-		for (i = 0; i < IPA_HW_NUM_FEATURES; i++) {
-			if (ipa3_uc_hdlrs[i].ipa_uc_event_log_info_hdlr)
-				ipa3_uc_hdlrs[i].ipa_uc_event_log_info_hdlr
-					(ipa3_ctx->uc_ctx.uc_event_top_mmio);
-		}
-	} else {
-
-		if (ipa3_ctx->uc_ctx.uc_sram_mmio->eventParams !=
-			ipa3_ctx->uc_ctx.uc_event_top_ofst) {
-			IPAERR("uc top ofst changed new=%u cur=%u\n",
-				ipa3_ctx->uc_ctx.uc_sram_mmio->eventParams,
-				ipa3_ctx->uc_ctx.uc_event_top_ofst);
-		}
-	}
-
-	return;
-
-bad_uc_top_ofst:
-	ipa3_ctx->uc_ctx.uc_event_top_ofst = 0;
-}
-
-/**
- * ipa3_uc_state_check() - Check the status of the uC interface
- *
- * Return value: 0 if the uC is loaded, interface is initialized
- *               and there was no recent failure in one of the commands.
- *               A negative value is returned otherwise.
- */
-int ipa3_uc_state_check(void)
-{
-	if (!ipa3_ctx->uc_ctx.uc_inited) {
-		IPAERR("uC interface not initialized\n");
-		return -EFAULT;
-	}
-
-	if (!ipa3_ctx->uc_ctx.uc_loaded) {
-		IPAERR("uC is not loaded\n");
-		return -EFAULT;
-	}
-
-	if (ipa3_ctx->uc_ctx.uc_failed) {
-		IPAERR("uC has failed its last command\n");
-		return -EFAULT;
-	}
-
-	return 0;
-}
-
-/**
- * ipa3_uc_loaded_check() - Check the uC has been loaded
- *
- * Return value: 1 if the uC is loaded, 0 otherwise
- */
-int ipa3_uc_loaded_check(void)
-{
-	return ipa3_ctx->uc_ctx.uc_loaded;
-}
-EXPORT_SYMBOL(ipa3_uc_loaded_check);
-
-/**
- * ipa3_uc_register_ready_cb() - register a uC ready callback notifier block
- * @nb: notifier
- *
- * Register a callback to be called when uC is ready to receive commands. uC is
- * considered to be ready when it sends %IPA_HW_2_CPU_RESPONSE_INIT_COMPLETED.
- *
- * Return: 0 on successful registration, negative errno otherwise
- *
- * See blocking_notifier_chain_register() for possible errno values
- */
-int ipa3_uc_register_ready_cb(struct notifier_block *nb)
-{
-	int rc;
-
-	mutex_lock(&uc_loaded_nb_lock);
-
-	rc = blocking_notifier_chain_register(&uc_loaded_notifier, nb);
-	if (!rc && ipa3_ctx->uc_ctx.uc_loaded)
-		(void) nb->notifier_call(nb, false, ipa3_ctx);
-
-	mutex_unlock(&uc_loaded_nb_lock);
-
-	return rc;
-}
-EXPORT_SYMBOL(ipa3_uc_register_ready_cb);
-
-/**
- * ipa3_uc_unregister_ready_cb() - unregister a uC ready callback
- * @nb: notifier
- *
- * Unregister a uC loaded notifier block that was previously registered by
- * ipa3_uc_register_ready_cb().
- *
- * Return: 0 on successful unregistration, negative errno otherwise
- *
- * See blocking_notifier_chain_unregister() for possible errno values
- */
-int ipa3_uc_unregister_ready_cb(struct notifier_block *nb)
-{
-	return blocking_notifier_chain_unregister(&uc_loaded_notifier, nb);
-}
-EXPORT_SYMBOL(ipa3_uc_unregister_ready_cb);
-
-static void ipa3_uc_event_handler(enum ipa_irq_type interrupt,
-				 void *private_data,
-				 void *interrupt_data)
-{
-	union IpaHwErrorEventData_t evt;
-	u8 feature;
-
-	WARN_ON(private_data != ipa3_ctx);
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	IPADBG("uC evt opcode=%u\n",
-		ipa3_ctx->uc_ctx.uc_sram_mmio->eventOp);
-
-
-	feature = EXTRACT_UC_FEATURE(ipa3_ctx->uc_ctx.uc_sram_mmio->eventOp);
-
-	if (feature >= IPA_HW_FEATURE_MAX) {
-		IPAERR("Invalid feature %u for event %u\n",
-			feature, ipa3_ctx->uc_ctx.uc_sram_mmio->eventOp);
-		IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-		return;
-	}
-	/* Feature specific handling */
-	if (ipa3_uc_hdlrs[feature].ipa_uc_event_hdlr)
-		ipa3_uc_hdlrs[feature].ipa_uc_event_hdlr
-			(ipa3_ctx->uc_ctx.uc_sram_mmio);
-
-	/* General handling */
-	if (ipa3_ctx->uc_ctx.uc_sram_mmio->eventOp ==
-	    IPA_HW_2_CPU_EVENT_ERROR) {
-		evt.raw32b = ipa3_ctx->uc_ctx.uc_sram_mmio->eventParams;
-		IPAERR("uC Error, evt errorType = %s\n",
-			ipa_hw_error_str(evt.params.errorType));
-		ipa3_ctx->uc_ctx.uc_failed = true;
-		ipa3_ctx->uc_ctx.uc_error_type = evt.params.errorType;
-		ipa3_ctx->uc_ctx.uc_error_timestamp =
-			ipahal_read_reg(IPA_TAG_TIMER);
-		/* Unexpected UC hardware state */
-		ipa_assert();
-	} else if (ipa3_ctx->uc_ctx.uc_sram_mmio->eventOp ==
-		IPA_HW_2_CPU_EVENT_LOG_INFO) {
-		IPADBG("uC evt log info ofst=0x%x\n",
-			ipa3_ctx->uc_ctx.uc_sram_mmio->eventParams);
-		ipa3_log_evt_hdlr();
-	} else {
-		IPADBG("unsupported uC evt opcode=%u\n",
-				ipa3_ctx->uc_ctx.uc_sram_mmio->eventOp);
-	}
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-
-}
-
-int ipa3_uc_panic_notifier(struct notifier_block *this,
-		unsigned long event, void *ptr)
-{
-	int result = 0;
-	struct ipa_active_client_logging_info log_info;
-
-	IPADBG("this=%pK evt=%lu ptr=%pK\n", this, event, ptr);
-
-	result = ipa3_uc_state_check();
-	if (result)
-		goto fail;
-
-	IPA_ACTIVE_CLIENTS_PREP_SIMPLE(log_info);
-	if (ipa3_inc_client_enable_clks_no_block(&log_info))
-		goto fail;
-
-	ipa3_ctx->uc_ctx.uc_sram_mmio->cmdOp =
-		IPA_CPU_2_HW_CMD_ERR_FATAL;
-	ipa3_ctx->uc_ctx.pending_cmd = ipa3_ctx->uc_ctx.uc_sram_mmio->cmdOp;
-	/* ensure write to shared memory is done before triggering uc */
-	wmb();
-	ipahal_write_reg_n(IPA_IRQ_EE_UC_n, 0, 0x1);
-
-	/* give uc enough time to save state */
-	udelay(IPA_PKT_FLUSH_TO_US);
-
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-	IPADBG("err_fatal issued\n");
-
-fail:
-	return NOTIFY_DONE;
-}
-
-static void ipa3_uc_response_hdlr(enum ipa_irq_type interrupt,
-				void *private_data,
-				void *interrupt_data)
-{
-	union IpaHwCpuCmdCompletedResponseData_t uc_rsp;
-	u8 feature;
-	int res;
-	int i;
-
-	WARN_ON(private_data != ipa3_ctx);
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-	IPADBG("uC rsp opcode=%u\n",
-			ipa3_ctx->uc_ctx.uc_sram_mmio->responseOp);
-
-	feature = EXTRACT_UC_FEATURE(ipa3_ctx->uc_ctx.uc_sram_mmio->responseOp);
-
-	if (feature >= IPA_HW_FEATURE_MAX) {
-		IPAERR("Invalid feature %u for event %u\n",
-			feature, ipa3_ctx->uc_ctx.uc_sram_mmio->eventOp);
-		IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-		return;
-	}
-
-	/* Feature specific handling */
-	if (ipa3_uc_hdlrs[feature].ipa3_uc_response_hdlr) {
-		res = ipa3_uc_hdlrs[feature].ipa3_uc_response_hdlr(
-			ipa3_ctx->uc_ctx.uc_sram_mmio,
-			&ipa3_ctx->uc_ctx.uc_status);
-		if (res == 0) {
-			IPADBG("feature %d specific response handler\n",
-				feature);
-			complete_all(&ipa3_ctx->uc_ctx.uc_completion);
-			IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-			return;
-		}
-	}
-
-	/* General handling */
-	if (ipa3_ctx->uc_ctx.uc_sram_mmio->responseOp ==
-			IPA_HW_2_CPU_RESPONSE_INIT_COMPLETED) {
-
-		if (ipa3_ctx->uc_ctx.uc_loaded) {
-			IPADBG("uC resp op INIT_COMPLETED is unexpected\n");
-			IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-			return;
-		}
-
-		mutex_lock(&uc_loaded_nb_lock);
-
-		ipa3_ctx->uc_ctx.uc_loaded = true;
-
-		(void) blocking_notifier_call_chain(&uc_loaded_notifier, true,
-			ipa3_ctx);
-
-		mutex_unlock(&uc_loaded_nb_lock);
-
-		IPADBG("IPA uC loaded\n");
-		/*
-		 * The proxy vote is held until uC is loaded to ensure that
-		 * IPA_HW_2_CPU_RESPONSE_INIT_COMPLETED is received.
-		 */
-		ipa3_proxy_clk_unvote();
-
-		/*
-		 * To enable ipa power collapse we need to enable rpmh and uc
-		 * handshake So that uc can do register retention. To enable
-		 * this handshake we need to send the below message to rpmh.
-		 */
-		ipa_pc_qmp_enable();
-
-		for (i = 0; i < IPA_HW_NUM_FEATURES; i++) {
-			if (ipa3_uc_hdlrs[i].ipa_uc_loaded_hdlr)
-				ipa3_uc_hdlrs[i].ipa_uc_loaded_hdlr();
-		}
-	} else if (ipa3_ctx->uc_ctx.uc_sram_mmio->responseOp ==
-		   IPA_HW_2_CPU_RESPONSE_CMD_COMPLETED) {
-		uc_rsp.raw32b = ipa3_ctx->uc_ctx.uc_sram_mmio->responseParams;
-		IPADBG("uC cmd response opcode=%u status=%u\n",
-		       uc_rsp.params.originalCmdOp,
-		       uc_rsp.params.status);
-		if (uc_rsp.params.originalCmdOp ==
-		    ipa3_ctx->uc_ctx.pending_cmd) {
-			ipa3_ctx->uc_ctx.uc_status = uc_rsp.params.status;
-			if (uc_rsp.params.originalCmdOp ==
-				IPA_CPU_2_HW_CMD_OFFLOAD_STATS_ALLOC)
-				ipa3_uc_save_dbg_stats(
-					uc_rsp.params.responseData);
-			complete_all(&ipa3_ctx->uc_ctx.uc_completion);
-		} else {
-			IPAERR("Expected cmd=%u rcvd cmd=%u\n",
-			       ipa3_ctx->uc_ctx.pending_cmd,
-			       uc_rsp.params.originalCmdOp);
-		}
-	} else {
-		IPAERR("Unsupported uC rsp opcode = %u\n",
-		       ipa3_ctx->uc_ctx.uc_sram_mmio->responseOp);
-	}
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-}
-
-static void ipa3_uc_wigig_misc_int_handler(enum ipa_irq_type interrupt,
-	void *private_data,
-	void *interrupt_data)
-{
-	IPADBG("\n");
-
-	WARN_ON(private_data != ipa3_ctx);
-
-	if (ipa3_ctx->uc_wigig_ctx.misc_notify_cb)
-		ipa3_ctx->uc_wigig_ctx.misc_notify_cb(
-			ipa3_ctx->uc_wigig_ctx.priv);
-
-	IPADBG("exit\n");
-}
-
-static int ipa3_uc_send_cmd_64b_param(u32 cmd_lo, u32 cmd_hi, u32 opcode,
-	u32 expected_status, bool polling_mode, unsigned long timeout_jiffies)
-{
-	int index;
-	union IpaHwCpuCmdCompletedResponseData_t uc_rsp;
-	int retries = 0;
-	u32 uc_error_type;
-
-send_cmd_lock:
-	mutex_lock(&ipa3_ctx->uc_ctx.uc_lock);
-
-	if (ipa3_uc_state_check()) {
-		IPADBG("uC send command aborted\n");
-		mutex_unlock(&ipa3_ctx->uc_ctx.uc_lock);
-		return -EBADF;
-	}
-send_cmd:
-	init_completion(&ipa3_ctx->uc_ctx.uc_completion);
-
-	ipa3_ctx->uc_ctx.uc_sram_mmio->cmdParams = cmd_lo;
-	ipa3_ctx->uc_ctx.uc_sram_mmio->cmdParams_hi = cmd_hi;
-	ipa3_ctx->uc_ctx.uc_sram_mmio->cmdOp = opcode;
-	ipa3_ctx->uc_ctx.pending_cmd = opcode;
-	ipa3_ctx->uc_ctx.uc_sram_mmio->responseOp = 0;
-	ipa3_ctx->uc_ctx.uc_sram_mmio->responseParams = 0;
-
-	ipa3_ctx->uc_ctx.uc_status = 0;
-
-	/* ensure write to shared memory is done before triggering uc */
-	wmb();
-
-	ipahal_write_reg_n(IPA_IRQ_EE_UC_n, 0, 0x1);
-
-	if (polling_mode) {
-		struct IpaHwSharedMemCommonMapping_t *uc_sram_ptr =
-			ipa3_ctx->uc_ctx.uc_sram_mmio;
-		for (index = 0; index < IPA_UC_POLL_MAX_RETRY; index++) {
-			if (uc_sram_ptr->responseOp ==
-			    IPA_HW_2_CPU_RESPONSE_CMD_COMPLETED) {
-				uc_rsp.raw32b = uc_sram_ptr->responseParams;
-				if (uc_rsp.params.originalCmdOp ==
-					ipa3_ctx->uc_ctx.pending_cmd) {
-					ipa3_ctx->uc_ctx.uc_status =
-						uc_rsp.params.status;
-					break;
-				}
-			}
-			usleep_range(IPA_UC_POLL_SLEEP_USEC,
-				IPA_UC_POLL_SLEEP_USEC);
-		}
-
-		if (index == IPA_UC_POLL_MAX_RETRY) {
-			IPAERR("uC max polling retries reached\n");
-			if (ipa3_ctx->uc_ctx.uc_failed) {
-				uc_error_type = ipa3_ctx->uc_ctx.uc_error_type;
-				IPAERR("uC reported on Error, errorType = %s\n",
-					ipa_hw_error_str(uc_error_type));
-			}
-			mutex_unlock(&ipa3_ctx->uc_ctx.uc_lock);
-			/* Unexpected UC hardware state */
-			ipa_assert();
-		}
-	} else {
-		if (wait_for_completion_timeout(&ipa3_ctx->uc_ctx.uc_completion,
-			timeout_jiffies) == 0) {
-			IPAERR("uC timed out\n");
-			if (ipa3_ctx->uc_ctx.uc_failed) {
-				uc_error_type = ipa3_ctx->uc_ctx.uc_error_type;
-				IPAERR("uC reported on Error, errorType = %s\n",
-					ipa_hw_error_str(uc_error_type));
-			}
-			mutex_unlock(&ipa3_ctx->uc_ctx.uc_lock);
-			/* Unexpected UC hardware state */
-			ipa_assert();
-		}
-	}
-
-	if (ipa3_ctx->uc_ctx.uc_status != expected_status) {
-		if (ipa3_ctx->uc_ctx.uc_status ==
-		    IPA_HW_PROD_DISABLE_CMD_GSI_STOP_FAILURE ||
-		    ipa3_ctx->uc_ctx.uc_status ==
-		    IPA_HW_CONS_DISABLE_CMD_GSI_STOP_FAILURE ||
-		    ipa3_ctx->uc_ctx.uc_status ==
-		    IPA_HW_CONS_STOP_FAILURE ||
-		    ipa3_ctx->uc_ctx.uc_status ==
-		    IPA_HW_PROD_STOP_FAILURE) {
-			retries++;
-			if (retries == IPA_GSI_CHANNEL_STOP_MAX_RETRY) {
-				IPAERR("Failed after %d tries\n", retries);
-				mutex_unlock(&ipa3_ctx->uc_ctx.uc_lock);
-				/* Unexpected UC hardware state */
-				ipa_assert();
-			}
-			mutex_unlock(&ipa3_ctx->uc_ctx.uc_lock);
-			if (ipa3_ctx->uc_ctx.uc_status ==
-			    IPA_HW_PROD_DISABLE_CMD_GSI_STOP_FAILURE)
-				ipa3_inject_dma_task_for_gsi();
-			/* sleep for short period to flush IPA */
-			usleep_range(IPA_GSI_CHANNEL_STOP_SLEEP_MIN_USEC,
-				IPA_GSI_CHANNEL_STOP_SLEEP_MAX_USEC);
-			goto send_cmd_lock;
-		}
-
-		if (ipa3_ctx->uc_ctx.uc_status ==
-			IPA_HW_GSI_CH_NOT_EMPTY_FAILURE) {
-			retries++;
-			if (retries >= IPA_GSI_CHANNEL_EMPTY_MAX_RETRY) {
-				IPAERR("Failed after %d tries\n", retries);
-				mutex_unlock(&ipa3_ctx->uc_ctx.uc_lock);
-				return -EFAULT;
-			}
-			usleep_range(
-			IPA_GSI_CHANNEL_EMPTY_SLEEP_MIN_USEC,
-			IPA_GSI_CHANNEL_EMPTY_SLEEP_MAX_USEC);
-			goto send_cmd;
-		}
-
-		IPAERR("Recevied status %u, Expected status %u\n",
-			ipa3_ctx->uc_ctx.uc_status, expected_status);
-		mutex_unlock(&ipa3_ctx->uc_ctx.uc_lock);
-		return -EFAULT;
-	}
-
-	mutex_unlock(&ipa3_ctx->uc_ctx.uc_lock);
-
-	IPADBG("uC cmd %u send succeeded\n", opcode);
-
-	return 0;
-}
-
-/**
- * ipa3_uc_interface_init() - Initialize the interface with the uC
- *
- * Return value: 0 on success, negative value otherwise
- */
-int ipa3_uc_interface_init(void)
-{
-	int result;
-	unsigned long phys_addr;
-
-	if (ipa3_ctx->uc_ctx.uc_inited) {
-		IPADBG("uC interface already initialized\n");
-		return 0;
-	}
-
-	mutex_init(&ipa3_ctx->uc_ctx.uc_lock);
-	spin_lock_init(&ipa3_ctx->uc_ctx.uc_spinlock);
-
-	phys_addr = ipa3_ctx->ipa_wrapper_base +
-		ipa3_ctx->ctrl->ipa_reg_base_ofst +
-		ipahal_get_reg_n_ofst(IPA_SW_AREA_RAM_DIRECT_ACCESS_n, 0);
-	ipa3_ctx->uc_ctx.uc_sram_mmio = ioremap(phys_addr,
-					       IPA_RAM_UC_SMEM_SIZE);
-	if (!ipa3_ctx->uc_ctx.uc_sram_mmio) {
-		IPAERR("Fail to ioremap IPA uC SRAM\n");
-		result = -ENOMEM;
-		goto remap_fail;
-	}
-
-	result = ipa3_add_interrupt_handler(IPA_UC_IRQ_0,
-		ipa3_uc_event_handler, true,
-		ipa3_ctx);
-	if (result) {
-		IPAERR("Fail to register for UC_IRQ0 event interrupt\n");
-		result = -EFAULT;
-		goto irq_fail0;
-	}
-
-	result = ipa3_add_interrupt_handler(IPA_UC_IRQ_1,
-		ipa3_uc_response_hdlr, true,
-		ipa3_ctx);
-	if (result) {
-		IPAERR("fail to register for UC_IRQ1 rsp interrupt\n");
-		result = -EFAULT;
-		goto irq_fail1;
-	}
-
-	result = ipa3_add_interrupt_handler(IPA_UC_IRQ_2,
-		ipa3_uc_wigig_misc_int_handler, true,
-		ipa3_ctx);
-	if (result) {
-		IPAERR("fail to register for UC_IRQ2 wigig misc interrupt\n");
-		result = -EFAULT;
-		goto irq_fail2;
-	}
-
-	ipa3_ctx->uc_ctx.uc_inited = true;
-
-	IPADBG("IPA uC interface is initialized\n");
-	return 0;
-irq_fail2:
-	ipa3_remove_interrupt_handler(IPA_UC_IRQ_1);
-irq_fail1:
-	ipa3_remove_interrupt_handler(IPA_UC_IRQ_0);
-irq_fail0:
-	iounmap(ipa3_ctx->uc_ctx.uc_sram_mmio);
-remap_fail:
-	return result;
-}
-
-/**
- * ipa3_uc_send_cmd() - Send a command to the uC
- *
- * Note1: This function sends command with 32bit parameter and do not
- *	use the higher 32bit of the command parameter (set to zero).
- *
- * Note2: In case the operation times out (No response from the uC) or
- *       polling maximal amount of retries has reached, the logic
- *       considers it as an invalid state of the uC/IPA, and
- *       issues a kernel panic.
- *
- * Returns: 0 on success.
- *          -EINVAL in case of invalid input.
- *          -EBADF in case uC interface is not initialized /
- *                 or the uC has failed previously.
- *          -EFAULT in case the received status doesn't match
- *                  the expected.
- */
-int ipa3_uc_send_cmd(u32 cmd, u32 opcode, u32 expected_status,
-		    bool polling_mode, unsigned long timeout_jiffies)
-{
-	return ipa3_uc_send_cmd_64b_param(cmd, 0, opcode,
-		expected_status, polling_mode, timeout_jiffies);
-}
-
-/**
- * ipa3_uc_register_handlers() - Registers event, response and log event
- *                              handlers for a specific feature.Please note
- *                              that currently only one handler can be
- *                              registered per feature.
- *
- * Return value: None
- */
-void ipa3_uc_register_handlers(enum ipa3_hw_features feature,
-			      struct ipa3_uc_hdlrs *hdlrs)
-{
-	if (0 > feature || IPA_HW_FEATURE_MAX <= feature) {
-		IPAERR("Feature %u is invalid, not registering hdlrs\n",
-		       feature);
-		return;
-	}
-
-	mutex_lock(&ipa3_ctx->uc_ctx.uc_lock);
-	ipa3_uc_hdlrs[feature] = *hdlrs;
-	mutex_unlock(&ipa3_ctx->uc_ctx.uc_lock);
-
-	IPADBG("uC handlers registered for feature %u\n", feature);
-}
-
-int ipa3_uc_is_gsi_channel_empty(enum ipa_client_type ipa_client)
-{
-	const struct ipa_gsi_ep_config *gsi_ep_info;
-	union IpaHwChkChEmptyCmdData_t cmd;
-	int ret;
-
-	gsi_ep_info = ipa3_get_gsi_ep_info(ipa_client);
-	if (!gsi_ep_info) {
-		IPAERR("Failed getting GSI EP info for client=%d\n",
-		       ipa_client);
-		return 0;
-	}
-
-	if (ipa3_uc_state_check()) {
-		IPADBG("uC cannot be used to validate ch emptiness clnt=%d\n"
-			, ipa_client);
-		return 0;
-	}
-
-	cmd.params.ee_n = gsi_ep_info->ee;
-	cmd.params.vir_ch_id = gsi_ep_info->ipa_gsi_chan_num;
-
-	IPADBG("uC emptiness check for IPA GSI Channel %d\n",
-	       gsi_ep_info->ipa_gsi_chan_num);
-
-	ret = ipa3_uc_send_cmd(cmd.raw32b, IPA_CPU_2_HW_CMD_GSI_CH_EMPTY, 0,
-			      false, 10*HZ);
-
-	return ret;
-}
-
-
-/**
- * ipa3_uc_notify_clk_state() - notify to uC of clock enable / disable
- * @enabled: true if clock are enabled
- *
- * The function uses the uC interface in order to notify uC before IPA clocks
- * are disabled to make sure uC is not in the middle of operation.
- * Also after clocks are enabled ned to notify uC to start processing.
- *
- * Returns: 0 on success, negative on failure
- */
-int ipa3_uc_notify_clk_state(bool enabled)
-{
-	u32 opcode;
-
-	if (ipa3_ctx->ipa_hw_type > IPA_HW_v4_0) {
-		IPADBG_LOW("not supported past IPA v4.0\n");
-		return 0;
-	}
-
-	/*
-	 * If the uC interface has not been initialized yet,
-	 * don't notify the uC on the enable/disable
-	 */
-	if (ipa3_uc_state_check()) {
-		IPADBG("uC interface will not notify the UC on clock state\n");
-		return 0;
-	}
-
-	IPADBG("uC clock %s notification\n", (enabled) ? "UNGATE" : "GATE");
-
-	opcode = (enabled) ? IPA_CPU_2_HW_CMD_CLK_UNGATE :
-			     IPA_CPU_2_HW_CMD_CLK_GATE;
-
-	return ipa3_uc_send_cmd(0, opcode, 0, true, 0);
-}
-
-/**
- * ipa3_uc_update_hw_flags() - send uC the HW flags to be used
- * @flags: This field is expected to be used as bitmask for enum ipa3_hw_flags
- *
- * Returns: 0 on success, negative on failure
- */
-int ipa3_uc_update_hw_flags(u32 flags)
-{
-	union IpaHwUpdateFlagsCmdData_t cmd;
-
-	memset(&cmd, 0, sizeof(cmd));
-	cmd.params.newFlags = flags;
-	return ipa3_uc_send_cmd(cmd.raw32b, IPA_CPU_2_HW_CMD_UPDATE_FLAGS, 0,
-		false, HZ);
-}
-
-/**
- * ipa3_uc_memcpy() - Perform a memcpy action using IPA uC
- * @dest: physical address to store the copied data.
- * @src: physical address of the source data to copy.
- * @len: number of bytes to copy.
- *
- * Returns: 0 on success, negative on failure
- */
-int ipa3_uc_memcpy(phys_addr_t dest, phys_addr_t src, int len)
-{
-	int res;
-	struct ipa_mem_buffer mem;
-	struct IpaHwMemCopyData_t *cmd;
-
-	IPADBG("dest 0x%pa src 0x%pa len %d\n", &dest, &src, len);
-	mem.size = sizeof(cmd);
-	mem.base = dma_alloc_coherent(ipa3_ctx->pdev, mem.size, &mem.phys_base,
-		GFP_KERNEL);
-	if (!mem.base) {
-		IPAERR("fail to alloc DMA buff of size %d\n", mem.size);
-		return -ENOMEM;
-	}
-	cmd = (struct IpaHwMemCopyData_t *)mem.base;
-	memset(cmd, 0, sizeof(*cmd));
-	cmd->destination_addr = dest;
-	cmd->dest_buffer_size = len;
-	cmd->source_addr = src;
-	cmd->source_buffer_size = len;
-	res = ipa3_uc_send_cmd((u32)mem.phys_base, IPA_CPU_2_HW_CMD_MEMCPY, 0,
-		true, 10 * HZ);
-	if (res) {
-		IPAERR("ipa3_uc_send_cmd failed %d\n", res);
-		goto free_coherent;
-	}
-
-	res = 0;
-free_coherent:
-	dma_free_coherent(ipa3_ctx->pdev, mem.size, mem.base, mem.phys_base);
-	return res;
-}
-
-int ipa3_uc_send_remote_ipa_info(u32 remote_addr, uint32_t mbox_n)
-{
-	int res;
-	struct ipa_mem_buffer cmd;
-	struct IpaHwDbAddrInfo_t *uc_info;
-
-	cmd.size = sizeof(*uc_info);
-	cmd.base = dma_alloc_coherent(ipa3_ctx->uc_pdev, cmd.size,
-		&cmd.phys_base, GFP_KERNEL);
-	if (cmd.base == NULL)
-		return -ENOMEM;
-
-	uc_info = (struct IpaHwDbAddrInfo_t *) cmd.base;
-	uc_info->remoteIPAAddr = remote_addr;
-	uc_info->mboxN = mbox_n;
-
-	res = ipa3_uc_send_cmd((u32)(cmd.phys_base),
-		IPA_CPU_2_HW_CMD_REMOTE_IPA_INFO, 0,
-		false, 10 * HZ);
-
-	if (res) {
-		IPAERR("fail to map 0x%x to mbox %d\n",
-			uc_info->remoteIPAAddr,
-			uc_info->mboxN);
-		goto free_coherent;
-	}
-
-	res = 0;
-free_coherent:
-	dma_free_coherent(ipa3_ctx->uc_pdev, cmd.size, cmd.base, cmd.phys_base);
-	return res;
-}
-
-int ipa3_uc_debug_stats_alloc(
-	struct IpaHwOffloadStatsAllocCmdData_t cmdinfo)
-{
-	int result;
-	struct ipa_mem_buffer cmd;
-	enum ipa_cpu_2_hw_offload_commands command;
-	struct IpaHwOffloadStatsAllocCmdData_t *cmd_data;
-
-	cmd.size = sizeof(*cmd_data);
-	cmd.base = dma_alloc_coherent(ipa3_ctx->uc_pdev, cmd.size,
-		&cmd.phys_base, GFP_KERNEL);
-	if (cmd.base == NULL) {
-		result = -ENOMEM;
-		return result;
-	}
-	cmd_data = (struct IpaHwOffloadStatsAllocCmdData_t *)cmd.base;
-	memcpy(cmd_data, &cmdinfo,
-		sizeof(struct IpaHwOffloadStatsAllocCmdData_t));
-	command = IPA_CPU_2_HW_CMD_OFFLOAD_STATS_ALLOC;
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	result = ipa3_uc_send_cmd((u32)(cmd.phys_base),
-		command,
-		IPA_HW_2_CPU_OFFLOAD_CMD_STATUS_SUCCESS,
-		false, 10 * HZ);
-	if (result) {
-		IPAERR("fail to alloc offload stats\n");
-		goto cleanup;
-	}
-	result = 0;
-cleanup:
-	dma_free_coherent(ipa3_ctx->uc_pdev,
-		cmd.size,
-		cmd.base, cmd.phys_base);
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-	IPADBG("exit\n");
-	return result;
-}
-
-int ipa3_uc_debug_stats_dealloc(uint32_t protocol)
-{
-	int result;
-	struct ipa_mem_buffer cmd;
-	enum ipa_cpu_2_hw_offload_commands command;
-	struct IpaHwOffloadStatsDeAllocCmdData_t *cmd_data;
-
-	cmd.size = sizeof(*cmd_data);
-	cmd.base = dma_alloc_coherent(ipa3_ctx->uc_pdev, cmd.size,
-		&cmd.phys_base, GFP_KERNEL);
-	if (cmd.base == NULL) {
-		result = -ENOMEM;
-		return result;
-	}
-	cmd_data = (struct IpaHwOffloadStatsDeAllocCmdData_t *)
-		cmd.base;
-	cmd_data->protocol = protocol;
-	command = IPA_CPU_2_HW_CMD_OFFLOAD_STATS_DEALLOC;
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	result = ipa3_uc_send_cmd((u32)(cmd.phys_base),
-		command,
-		IPA_HW_2_CPU_OFFLOAD_CMD_STATUS_SUCCESS,
-		false, 10 * HZ);
-	if (result) {
-		IPAERR("fail to dealloc offload stats\n");
-		goto cleanup;
-	}
-	switch (protocol) {
-	case IPA_HW_PROTOCOL_AQC:
-		break;
-	case IPA_HW_PROTOCOL_11ad:
-		break;
-	case IPA_HW_PROTOCOL_WDI:
-		iounmap(ipa3_ctx->wdi2_ctx.dbg_stats.uc_dbg_stats_mmio);
-		ipa3_ctx->wdi2_ctx.dbg_stats.uc_dbg_stats_mmio = NULL;
-		break;
-	case IPA_HW_PROTOCOL_WDI3:
-		iounmap(ipa3_ctx->wdi3_ctx.dbg_stats.uc_dbg_stats_mmio);
-		ipa3_ctx->wdi3_ctx.dbg_stats.uc_dbg_stats_mmio = NULL;
-		break;
-	case IPA_HW_PROTOCOL_ETH:
-		break;
-	default:
-		IPAERR("unknown protocols %d\n", protocol);
-	}
-	result = 0;
-cleanup:
-	dma_free_coherent(ipa3_ctx->uc_pdev, cmd.size,
-		cmd.base, cmd.phys_base);
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-	IPADBG("exit\n");
-	return result;
-}

ipa/ipa_v3/ipa_uc_mhi.c

@@ -1,957 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/ipa.h>
-#include "ipa_i.h"
-
-/* MHI uC interface definitions */
-#define IPA_HW_INTERFACE_MHI_VERSION            0x0004
-
-#define IPA_HW_MAX_NUMBER_OF_CHANNELS	2
-#define IPA_HW_MAX_NUMBER_OF_EVENTRINGS	2
-#define IPA_HW_MAX_CHANNEL_HANDLE	(IPA_HW_MAX_NUMBER_OF_CHANNELS-1)
-
-/**
- * Values that represent the MHI commands from CPU to IPA HW.
- * @IPA_CPU_2_HW_CMD_MHI_INIT: Initialize HW to be ready for MHI processing.
- *	Once operation was completed HW shall respond with
- *	IPA_HW_2_CPU_RESPONSE_CMD_COMPLETED.
- * @IPA_CPU_2_HW_CMD_MHI_INIT_CHANNEL: Initialize specific channel to be ready
- *	to serve MHI transfers. Once initialization was completed HW shall
- *	respond with IPA_HW_2_CPU_RESPONSE_MHI_CHANGE_CHANNEL_STATE.
- *		IPA_HW_MHI_CHANNEL_STATE_ENABLE
- * @IPA_CPU_2_HW_CMD_MHI_UPDATE_MSI: Update MHI MSI interrupts data.
- *	Once operation was completed HW shall respond with
- *	IPA_HW_2_CPU_RESPONSE_CMD_COMPLETED.
- * @IPA_CPU_2_HW_CMD_MHI_CHANGE_CHANNEL_STATE: Change specific channel
- *	processing state following host request. Once operation was completed
- *	HW shall respond with IPA_HW_2_CPU_RESPONSE_MHI_CHANGE_CHANNEL_STATE.
- * @IPA_CPU_2_HW_CMD_MHI_DL_UL_SYNC_INFO: Info related to DL UL syncronization.
- * @IPA_CPU_2_HW_CMD_MHI_STOP_EVENT_UPDATE: Cmd to stop event ring processing.
- */
-enum ipa_cpu_2_hw_mhi_commands {
-	IPA_CPU_2_HW_CMD_MHI_INIT
-		= FEATURE_ENUM_VAL(IPA_HW_FEATURE_MHI, 0),
-	IPA_CPU_2_HW_CMD_MHI_INIT_CHANNEL
-		= FEATURE_ENUM_VAL(IPA_HW_FEATURE_MHI, 1),
-	IPA_CPU_2_HW_CMD_MHI_UPDATE_MSI
-		= FEATURE_ENUM_VAL(IPA_HW_FEATURE_MHI, 2),
-	IPA_CPU_2_HW_CMD_MHI_CHANGE_CHANNEL_STATE
-		= FEATURE_ENUM_VAL(IPA_HW_FEATURE_MHI, 3),
-	IPA_CPU_2_HW_CMD_MHI_DL_UL_SYNC_INFO
-		= FEATURE_ENUM_VAL(IPA_HW_FEATURE_MHI, 4),
-	IPA_CPU_2_HW_CMD_MHI_STOP_EVENT_UPDATE
-		= FEATURE_ENUM_VAL(IPA_HW_FEATURE_MHI, 5)
-};
-
-/**
- * Values that represent MHI related HW responses to CPU commands.
- * @IPA_HW_2_CPU_RESPONSE_MHI_CHANGE_CHANNEL_STATE: Response to
- *	IPA_CPU_2_HW_CMD_MHI_INIT_CHANNEL or
- *	IPA_CPU_2_HW_CMD_MHI_CHANGE_CHANNEL_STATE commands.
- */
-enum ipa_hw_2_cpu_mhi_responses {
-	IPA_HW_2_CPU_RESPONSE_MHI_CHANGE_CHANNEL_STATE
-		= FEATURE_ENUM_VAL(IPA_HW_FEATURE_MHI, 0),
-};
-
-/**
- * Values that represent MHI related HW event to be sent to CPU.
- * @IPA_HW_2_CPU_EVENT_MHI_CHANNEL_ERROR: Event specify the device detected an
- *	error in an element from the transfer ring associated with the channel
- * @IPA_HW_2_CPU_EVENT_MHI_CHANNEL_WAKE_UP_REQUEST: Event specify a transport
- *	interrupt was asserted when MHI engine is suspended
- */
-enum ipa_hw_2_cpu_mhi_events {
-	IPA_HW_2_CPU_EVENT_MHI_CHANNEL_ERROR
-		= FEATURE_ENUM_VAL(IPA_HW_FEATURE_MHI, 0),
-	IPA_HW_2_CPU_EVENT_MHI_CHANNEL_WAKE_UP_REQUEST
-		= FEATURE_ENUM_VAL(IPA_HW_FEATURE_MHI, 1),
-};
-
-/**
- * Channel error types.
- * @IPA_HW_CHANNEL_ERROR_NONE: No error persists.
- * @IPA_HW_CHANNEL_INVALID_RE_ERROR: Invalid Ring Element was detected
- */
-enum ipa_hw_channel_errors {
-	IPA_HW_CHANNEL_ERROR_NONE,
-	IPA_HW_CHANNEL_INVALID_RE_ERROR
-};
-
-/**
- * MHI error types.
- * @IPA_HW_INVALID_MMIO_ERROR: Invalid data read from MMIO space
- * @IPA_HW_INVALID_CHANNEL_ERROR: Invalid data read from channel context array
- * @IPA_HW_INVALID_EVENT_ERROR: Invalid data read from event ring context array
- * @IPA_HW_NO_ED_IN_RING_ERROR: No event descriptors are available to report on
- *	secondary event ring
- * @IPA_HW_LINK_ERROR: Link error
- */
-enum ipa_hw_mhi_errors {
-	IPA_HW_INVALID_MMIO_ERROR
-		= FEATURE_ENUM_VAL(IPA_HW_FEATURE_MHI, 0),
-	IPA_HW_INVALID_CHANNEL_ERROR
-		= FEATURE_ENUM_VAL(IPA_HW_FEATURE_MHI, 1),
-	IPA_HW_INVALID_EVENT_ERROR
-		= FEATURE_ENUM_VAL(IPA_HW_FEATURE_MHI, 2),
-	IPA_HW_NO_ED_IN_RING_ERROR
-		= FEATURE_ENUM_VAL(IPA_HW_FEATURE_MHI, 4),
-	IPA_HW_LINK_ERROR
-		= FEATURE_ENUM_VAL(IPA_HW_FEATURE_MHI, 5),
-};
-
-
-/**
- * Structure referring to the common and MHI section of 128B shared memory
- * located in offset zero of SW Partition in IPA SRAM.
- * The shared memory is used for communication between IPA HW and CPU.
- * @common: common section in IPA SRAM
- * @interfaceVersionMhi: The MHI interface version as reported by HW
- * @mhiState: Overall MHI state
- * @reserved_2B: reserved
- * @mhiCnl0State: State of MHI channel 0.
- *	The state carries information regarding the error type.
- *	See IPA_HW_MHI_CHANNEL_STATES.
- * @mhiCnl0State: State of MHI channel 1.
- * @mhiCnl0State: State of MHI channel 2.
- * @mhiCnl0State: State of MHI channel 3
- * @mhiCnl0State: State of MHI channel 4.
- * @mhiCnl0State: State of MHI channel 5.
- * @mhiCnl0State: State of MHI channel 6.
- * @mhiCnl0State: State of MHI channel 7.
- * @reserved_37_34: reserved
- * @reserved_3B_38: reserved
- * @reserved_3F_3C: reserved
- */
-struct IpaHwSharedMemMhiMapping_t {
-	struct IpaHwSharedMemCommonMapping_t common;
-	u16 interfaceVersionMhi;
-	u8 mhiState;
-	u8 reserved_2B;
-	u8 mhiCnl0State;
-	u8 mhiCnl1State;
-	u8 mhiCnl2State;
-	u8 mhiCnl3State;
-	u8 mhiCnl4State;
-	u8 mhiCnl5State;
-	u8 mhiCnl6State;
-	u8 mhiCnl7State;
-	u32 reserved_37_34;
-	u32 reserved_3B_38;
-	u32 reserved_3F_3C;
-};
-
-
-/**
- * Structure holding the parameters for IPA_CPU_2_HW_CMD_MHI_INIT command.
- * Parameters are sent as pointer thus should be reside in address accessible
- * to HW.
- * @msiAddress: The MSI base (in device space) used for asserting the interrupt
- *	(MSI) associated with the event ring
- * mmioBaseAddress: The address (in device space) of MMIO structure in
- *	host space
- * deviceMhiCtrlBaseAddress: Base address of the memory region in the device
- *	address space where the MHI control data structures are allocated by
- *	the host, including channel context array, event context array,
- *	and rings. This value is used for host/device address translation.
- * deviceMhiDataBaseAddress: Base address of the memory region in the device
- *	address space where the MHI data buffers are allocated by the host.
- *	This value is used for host/device address translation.
- * firstChannelIndex: First channel ID. Doorbell 0 is mapped to this channel
- * firstEventRingIndex: First event ring ID. Doorbell 16 is mapped to this
- *	event ring.
- */
-struct IpaHwMhiInitCmdData_t {
-	u32 msiAddress;
-	u32 mmioBaseAddress;
-	u32 deviceMhiCtrlBaseAddress;
-	u32 deviceMhiDataBaseAddress;
-	u32 firstChannelIndex;
-	u32 firstEventRingIndex;
-};
-
-/**
- * Structure holding the parameters for IPA_CPU_2_HW_CMD_MHI_INIT_CHANNEL
- *	command. Parameters are sent as 32b immediate parameters.
- * @hannelHandle: The channel identifier as allocated by driver.
- *	value is within the range 0 to IPA_HW_MAX_CHANNEL_HANDLE
- * @contexArrayIndex: Unique index for channels, between 0 and 255. The index is
- *	used as an index in channel context array structures.
- * @bamPipeId: The IPA pipe number for pipe dedicated for this channel
- * @channelDirection: The direction of the channel as defined in the channel
- *	type field (CHTYPE) in the channel context data structure.
- * @reserved: reserved.
- */
-union IpaHwMhiInitChannelCmdData_t {
-	struct IpaHwMhiInitChannelCmdParams_t {
-		u32 channelHandle:8;
-		u32 contexArrayIndex:8;
-		u32 bamPipeId:6;
-		u32 channelDirection:2;
-		u32 reserved:8;
-	} params;
-	u32 raw32b;
-};
-
-/**
- * Structure holding the parameters for IPA_CPU_2_HW_CMD_MHI_UPDATE_MSI command.
- * @msiAddress_low: The MSI lower base addr (in device space) used for asserting
- *	the interrupt (MSI) associated with the event ring.
- * @msiAddress_hi: The MSI higher base addr (in device space) used for asserting
- *	the interrupt (MSI) associated with the event ring.
- * @msiMask: Mask indicating number of messages assigned by the host to device
- * @msiData: Data Pattern to use when generating the MSI
- */
-struct IpaHwMhiMsiCmdData_t {
-	u32 msiAddress_low;
-	u32 msiAddress_hi;
-	u32 msiMask;
-	u32 msiData;
-};
-
-/**
- * Structure holding the parameters for
- * IPA_CPU_2_HW_CMD_MHI_CHANGE_CHANNEL_STATE command.
- * Parameters are sent as 32b immediate parameters.
- * @requestedState: The requested channel state as was indicated from Host.
- *	Use IPA_HW_MHI_CHANNEL_STATES to specify the requested state
- * @channelHandle: The channel identifier as allocated by driver.
- *	value is within the range 0 to IPA_HW_MAX_CHANNEL_HANDLE
- * @LPTransitionRejected: Indication that low power state transition was
- *	rejected
- * @reserved: reserved
- */
-union IpaHwMhiChangeChannelStateCmdData_t {
-	struct IpaHwMhiChangeChannelStateCmdParams_t {
-		u32 requestedState:8;
-		u32 channelHandle:8;
-		u32 LPTransitionRejected:8;
-		u32 reserved:8;
-	} params;
-	u32 raw32b;
-};
-
-/**
- * Structure holding the parameters for
- *	IPA_CPU_2_HW_CMD_MHI_STOP_EVENT_UPDATE command.
- * Parameters are sent as 32b immediate parameters.
- * @channelHandle: The channel identifier as allocated by driver.
- *	value is within the range 0 to IPA_HW_MAX_CHANNEL_HANDLE
- * @reserved: reserved
- */
-union IpaHwMhiStopEventUpdateData_t {
-	struct IpaHwMhiStopEventUpdateDataParams_t {
-		u32 channelHandle:8;
-		u32 reserved:24;
-	} params;
-	u32 raw32b;
-};
-
-/**
- * Structure holding the parameters for
- *	IPA_HW_2_CPU_RESPONSE_MHI_CHANGE_CHANNEL_STATE response.
- * Parameters are sent as 32b immediate parameters.
- * @state: The new channel state. In case state is not as requested this is
- *	error indication for the last command
- * @channelHandle: The channel identifier
- * @additonalParams: For stop: the number of pending transport descriptors
- * currently queued
- */
-union IpaHwMhiChangeChannelStateResponseData_t {
-	struct IpaHwMhiChangeChannelStateResponseParams_t {
-		u32 state:8;
-		u32 channelHandle:8;
-		u32 additonalParams:16;
-	} params;
-	u32 raw32b;
-};
-
-/**
- * Structure holding the parameters for
- *	IPA_HW_2_CPU_EVENT_MHI_CHANNEL_ERROR event.
- * Parameters are sent as 32b immediate parameters.
- * @errorType: Type of error - IPA_HW_CHANNEL_ERRORS
- * @channelHandle: The channel identifier as allocated by driver.
- *	value is within the range 0 to IPA_HW_MAX_CHANNEL_HANDLE
- * @reserved: reserved
- */
-union IpaHwMhiChannelErrorEventData_t {
-	struct IpaHwMhiChannelErrorEventParams_t {
-		u32 errorType:8;
-		u32 channelHandle:8;
-		u32 reserved:16;
-	} params;
-	u32 raw32b;
-};
-
-/**
- * Structure holding the parameters for
- *	IPA_HW_2_CPU_EVENT_MHI_CHANNEL_WAKE_UP_REQUEST event.
- * Parameters are sent as 32b immediate parameters.
- * @channelHandle: The channel identifier as allocated by driver.
- *	value is within the range 0 to IPA_HW_MAX_CHANNEL_HANDLE
- * @reserved: reserved
- */
-union IpaHwMhiChannelWakeupEventData_t {
-	struct IpaHwMhiChannelWakeupEventParams_t {
-		u32 channelHandle:8;
-		u32 reserved:24;
-	} params;
-	u32 raw32b;
-};
-
-/**
- * Structure holding the MHI Common statistics
- * @numULDLSync: Number of times UL activity trigged due to DL activity
- * @numULTimerExpired: Number of times UL Accm Timer expired
- */
-struct IpaHwStatsMhiCmnInfoData_t {
-	u32 numULDLSync;
-	u32 numULTimerExpired;
-	u32 numChEvCtxWpRead;
-	u32 reserved;
-};
-
-/**
- * Structure holding the MHI Channel statistics
- * @doorbellInt: The number of doorbell int
- * @reProccesed: The number of ring elements processed
- * @bamFifoFull: Number of times Bam Fifo got full
- * @bamFifoEmpty: Number of times Bam Fifo got empty
- * @bamFifoUsageHigh: Number of times Bam fifo usage went above 75%
- * @bamFifoUsageLow: Number of times Bam fifo usage went below 25%
- * @bamInt: Number of BAM Interrupts
- * @ringFull: Number of times Transfer Ring got full
- * @ringEmpty: umber of times Transfer Ring got empty
- * @ringUsageHigh: Number of times Transfer Ring usage went above 75%
- * @ringUsageLow: Number of times Transfer Ring usage went below 25%
- * @delayedMsi: Number of times device triggered MSI to host after
- *	Interrupt Moderation Timer expiry
- * @immediateMsi: Number of times device triggered MSI to host immediately
- * @thresholdMsi: Number of times device triggered MSI due to max pending
- *	events threshold reached
- * @numSuspend: Number of times channel was suspended
- * @numResume: Number of times channel was suspended
- * @num_OOB: Number of times we indicated that we are OOB
- * @num_OOB_timer_expiry: Number of times we indicated that we are OOB
- *	after timer expiry
- * @num_OOB_moderation_timer_start: Number of times we started timer after
- *	sending OOB and hitting OOB again before we processed threshold
- *	number of packets
- * @num_db_mode_evt: Number of times we indicated that we are in Doorbell mode
- */
-struct IpaHwStatsMhiCnlInfoData_t {
-	u32 doorbellInt;
-	u32 reProccesed;
-	u32 bamFifoFull;
-	u32 bamFifoEmpty;
-	u32 bamFifoUsageHigh;
-	u32 bamFifoUsageLow;
-	u32 bamInt;
-	u32 ringFull;
-	u32 ringEmpty;
-	u32 ringUsageHigh;
-	u32 ringUsageLow;
-	u32 delayedMsi;
-	u32 immediateMsi;
-	u32 thresholdMsi;
-	u32 numSuspend;
-	u32 numResume;
-	u32 num_OOB;
-	u32 num_OOB_timer_expiry;
-	u32 num_OOB_moderation_timer_start;
-	u32 num_db_mode_evt;
-};
-
-/**
- * Structure holding the MHI statistics
- * @mhiCmnStats: Stats pertaining to MHI
- * @mhiCnlStats: Stats pertaining to each channel
- */
-struct IpaHwStatsMhiInfoData_t {
-	struct IpaHwStatsMhiCmnInfoData_t mhiCmnStats;
-	struct IpaHwStatsMhiCnlInfoData_t mhiCnlStats[
-						IPA_HW_MAX_NUMBER_OF_CHANNELS];
-};
-
-/**
- * Structure holding the MHI Common Config info
- * @isDlUlSyncEnabled: Flag to indicate if DL-UL synchronization is enabled
- * @UlAccmVal: Out Channel(UL) accumulation time in ms when DL UL Sync is
- *	enabled
- * @ulMsiEventThreshold: Threshold at which HW fires MSI to host for UL events
- * @dlMsiEventThreshold: Threshold at which HW fires MSI to host for DL events
- */
-struct IpaHwConfigMhiCmnInfoData_t {
-	u8 isDlUlSyncEnabled;
-	u8 UlAccmVal;
-	u8 ulMsiEventThreshold;
-	u8 dlMsiEventThreshold;
-};
-
-/**
- * Structure holding the parameters for MSI info data
- * @msiAddress_low: The MSI lower base addr (in device space) used for asserting
- *	the interrupt (MSI) associated with the event ring.
- * @msiAddress_hi: The MSI higher base addr (in device space) used for asserting
- *	the interrupt (MSI) associated with the event ring.
- * @msiMask: Mask indicating number of messages assigned by the host to device
- * @msiData: Data Pattern to use when generating the MSI
- */
-struct IpaHwConfigMhiMsiInfoData_t {
-	u32 msiAddress_low;
-	u32 msiAddress_hi;
-	u32 msiMask;
-	u32 msiData;
-};
-
-/**
- * Structure holding the MHI Channel Config info
- * @transferRingSize: The Transfer Ring size in terms of Ring Elements
- * @transferRingIndex: The Transfer Ring channel number as defined by host
- * @eventRingIndex: The Event Ring Index associated with this Transfer Ring
- * @bamPipeIndex: The BAM Pipe associated with this channel
- * @isOutChannel: Indication for the direction of channel
- * @reserved_0: Reserved byte for maintaining 4byte alignment
- * @reserved_1: Reserved byte for maintaining 4byte alignment
- */
-struct IpaHwConfigMhiCnlInfoData_t {
-	u16 transferRingSize;
-	u8  transferRingIndex;
-	u8  eventRingIndex;
-	u8  bamPipeIndex;
-	u8  isOutChannel;
-	u8  reserved_0;
-	u8  reserved_1;
-};
-
-/**
- * Structure holding the MHI Event Config info
- * @msiVec: msi vector to invoke MSI interrupt
- * @intmodtValue: Interrupt moderation timer (in milliseconds)
- * @eventRingSize: The Event Ring size in terms of Ring Elements
- * @eventRingIndex: The Event Ring number as defined by host
- * @reserved_0: Reserved byte for maintaining 4byte alignment
- * @reserved_1: Reserved byte for maintaining 4byte alignment
- * @reserved_2: Reserved byte for maintaining 4byte alignment
- */
-struct IpaHwConfigMhiEventInfoData_t {
-	u32 msiVec;
-	u16 intmodtValue;
-	u16 eventRingSize;
-	u8  eventRingIndex;
-	u8  reserved_0;
-	u8  reserved_1;
-	u8  reserved_2;
-};
-
-/**
- * Structure holding the MHI Config info
- * @mhiCmnCfg: Common Config pertaining to MHI
- * @mhiMsiCfg: Config pertaining to MSI config
- * @mhiCnlCfg: Config pertaining to each channel
- * @mhiEvtCfg: Config pertaining to each event Ring
- */
-struct IpaHwConfigMhiInfoData_t {
-	struct IpaHwConfigMhiCmnInfoData_t mhiCmnCfg;
-	struct IpaHwConfigMhiMsiInfoData_t mhiMsiCfg;
-	struct IpaHwConfigMhiCnlInfoData_t mhiCnlCfg[
-						IPA_HW_MAX_NUMBER_OF_CHANNELS];
-	struct IpaHwConfigMhiEventInfoData_t mhiEvtCfg[
-					IPA_HW_MAX_NUMBER_OF_EVENTRINGS];
-};
-
-
-struct ipa3_uc_mhi_ctx {
-	u8 expected_responseOp;
-	u32 expected_responseParams;
-	void (*ready_cb)(void);
-	void (*wakeup_request_cb)(void);
-	u32 mhi_uc_stats_ofst;
-	struct IpaHwStatsMhiInfoData_t *mhi_uc_stats_mmio;
-};
-
-#define PRINT_COMMON_STATS(x) \
-	(nBytes += scnprintf(&dbg_buff[nBytes], size - nBytes, \
-	#x "=0x%x\n", ipa3_uc_mhi_ctx->mhi_uc_stats_mmio->mhiCmnStats.x))
-
-#define PRINT_CHANNEL_STATS(ch, x) \
-	(nBytes += scnprintf(&dbg_buff[nBytes], size - nBytes, \
-	#x "=0x%x\n", ipa3_uc_mhi_ctx->mhi_uc_stats_mmio->mhiCnlStats[ch].x))
-
-struct ipa3_uc_mhi_ctx *ipa3_uc_mhi_ctx;
-
-static int ipa3_uc_mhi_response_hdlr(struct IpaHwSharedMemCommonMapping_t
-	*uc_sram_mmio, u32 *uc_status)
-{
-	IPADBG("responseOp=%d\n", uc_sram_mmio->responseOp);
-	if (uc_sram_mmio->responseOp == ipa3_uc_mhi_ctx->expected_responseOp &&
-	    uc_sram_mmio->responseParams ==
-	    ipa3_uc_mhi_ctx->expected_responseParams) {
-		*uc_status = 0;
-		return 0;
-	}
-	return -EINVAL;
-}
-
-static void ipa3_uc_mhi_event_hdlr(struct IpaHwSharedMemCommonMapping_t
-	*uc_sram_mmio)
-{
-	if (ipa3_ctx->uc_ctx.uc_sram_mmio->eventOp ==
-	    IPA_HW_2_CPU_EVENT_MHI_CHANNEL_ERROR) {
-		union IpaHwMhiChannelErrorEventData_t evt;
-
-		IPAERR("Channel error\n");
-		evt.raw32b = uc_sram_mmio->eventParams;
-		IPAERR("errorType=%d channelHandle=%d reserved=%d\n",
-			evt.params.errorType, evt.params.channelHandle,
-			evt.params.reserved);
-	} else if (ipa3_ctx->uc_ctx.uc_sram_mmio->eventOp ==
-		   IPA_HW_2_CPU_EVENT_MHI_CHANNEL_WAKE_UP_REQUEST) {
-		union IpaHwMhiChannelWakeupEventData_t evt;
-
-		IPADBG("WakeUp channel request\n");
-		evt.raw32b = uc_sram_mmio->eventParams;
-		IPADBG("channelHandle=%d reserved=%d\n",
-			evt.params.channelHandle, evt.params.reserved);
-		ipa3_uc_mhi_ctx->wakeup_request_cb();
-	}
-}
-
-static void ipa3_uc_mhi_event_log_info_hdlr(
-	struct IpaHwEventLogInfoData_t *uc_event_top_mmio)
-{
-	struct Ipa3HwEventInfoData_t *evt_info_ptr;
-	u32 size;
-
-	if ((uc_event_top_mmio->protocolMask & (1 << IPA_HW_FEATURE_MHI))
-		== 0) {
-		IPAERR("MHI feature missing 0x%x\n",
-			uc_event_top_mmio->protocolMask);
-		return;
-	}
-
-	evt_info_ptr = &uc_event_top_mmio->statsInfo;
-	size = evt_info_ptr->featureInfo[IPA_HW_FEATURE_MHI].params.size;
-	if (size != sizeof(struct IpaHwStatsMhiInfoData_t)) {
-		IPAERR("mhi stats sz invalid exp=%zu is=%u\n",
-			sizeof(struct IpaHwStatsMhiInfoData_t),
-			size);
-		return;
-	}
-
-	ipa3_uc_mhi_ctx->mhi_uc_stats_ofst =
-		evt_info_ptr->baseAddrOffset +
-		evt_info_ptr->featureInfo[IPA_HW_FEATURE_MHI].params.offset;
-	IPAERR("MHI stats ofst=0x%x\n", ipa3_uc_mhi_ctx->mhi_uc_stats_ofst);
-	if (ipa3_uc_mhi_ctx->mhi_uc_stats_ofst +
-		sizeof(struct IpaHwStatsMhiInfoData_t) >=
-		ipa3_ctx->ctrl->ipa_reg_base_ofst +
-		ipahal_get_reg_n_ofst(IPA_SW_AREA_RAM_DIRECT_ACCESS_n, 0) +
-		ipa3_ctx->smem_sz) {
-		IPAERR("uc_mhi_stats 0x%x outside SRAM\n",
-			ipa3_uc_mhi_ctx->mhi_uc_stats_ofst);
-		return;
-	}
-
-	ipa3_uc_mhi_ctx->mhi_uc_stats_mmio =
-		ioremap(ipa3_ctx->ipa_wrapper_base +
-		ipa3_uc_mhi_ctx->mhi_uc_stats_ofst,
-		sizeof(struct IpaHwStatsMhiInfoData_t));
-	if (!ipa3_uc_mhi_ctx->mhi_uc_stats_mmio) {
-		IPAERR("fail to ioremap uc mhi stats\n");
-		return;
-	}
-}
-
-int ipa3_uc_mhi_init(void (*ready_cb)(void), void (*wakeup_request_cb)(void))
-{
-	struct ipa3_uc_hdlrs hdlrs;
-
-	if (ipa3_uc_mhi_ctx) {
-		IPAERR("Already initialized\n");
-		return -EFAULT;
-	}
-
-	ipa3_uc_mhi_ctx = kzalloc(sizeof(*ipa3_uc_mhi_ctx), GFP_KERNEL);
-	if (!ipa3_uc_mhi_ctx) {
-		IPAERR("no mem\n");
-		return -ENOMEM;
-	}
-
-	ipa3_uc_mhi_ctx->ready_cb = ready_cb;
-	ipa3_uc_mhi_ctx->wakeup_request_cb = wakeup_request_cb;
-
-	memset(&hdlrs, 0, sizeof(hdlrs));
-	hdlrs.ipa_uc_loaded_hdlr = ipa3_uc_mhi_ctx->ready_cb;
-	hdlrs.ipa3_uc_response_hdlr = ipa3_uc_mhi_response_hdlr;
-	hdlrs.ipa_uc_event_hdlr = ipa3_uc_mhi_event_hdlr;
-	hdlrs.ipa_uc_event_log_info_hdlr = ipa3_uc_mhi_event_log_info_hdlr;
-	ipa3_uc_register_handlers(IPA_HW_FEATURE_MHI, &hdlrs);
-
-	IPADBG("Done\n");
-	return 0;
-}
-
-void ipa3_uc_mhi_cleanup(void)
-{
-	struct ipa3_uc_hdlrs null_hdlrs = { 0 };
-
-	IPADBG("Enter\n");
-
-	if (!ipa3_uc_mhi_ctx) {
-		IPAERR("ipa3_uc_mhi_ctx is not initialized\n");
-		return;
-	}
-	ipa3_uc_register_handlers(IPA_HW_FEATURE_MHI, &null_hdlrs);
-	kfree(ipa3_uc_mhi_ctx);
-	ipa3_uc_mhi_ctx = NULL;
-
-	IPADBG("Done\n");
-}
-
-int ipa3_uc_mhi_init_engine(struct ipa_mhi_msi_info *msi, u32 mmio_addr,
-	u32 host_ctrl_addr, u32 host_data_addr, u32 first_ch_idx,
-	u32 first_evt_idx)
-{
-	int res;
-	struct ipa_mem_buffer mem;
-	struct IpaHwMhiInitCmdData_t *init_cmd_data;
-	struct IpaHwMhiMsiCmdData_t *msi_cmd;
-
-	if (!ipa3_uc_mhi_ctx) {
-		IPAERR("Not initialized\n");
-		return -EFAULT;
-	}
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	res = ipa3_uc_update_hw_flags(0);
-	if (res) {
-		IPAERR("ipa3_uc_update_hw_flags failed %d\n", res);
-		goto disable_clks;
-	}
-
-	mem.size = sizeof(*init_cmd_data);
-	mem.base = dma_zalloc_coherent(ipa3_ctx->pdev, mem.size, &mem.phys_base,
-		GFP_KERNEL);
-	if (!mem.base) {
-		IPAERR("fail to alloc DMA buff of size %d\n", mem.size);
-		res = -ENOMEM;
-		goto disable_clks;
-	}
-	init_cmd_data = (struct IpaHwMhiInitCmdData_t *)mem.base;
-	init_cmd_data->msiAddress = msi->addr_low;
-	init_cmd_data->mmioBaseAddress = mmio_addr;
-	init_cmd_data->deviceMhiCtrlBaseAddress = host_ctrl_addr;
-	init_cmd_data->deviceMhiDataBaseAddress = host_data_addr;
-	init_cmd_data->firstChannelIndex = first_ch_idx;
-	init_cmd_data->firstEventRingIndex = first_evt_idx;
-	res = ipa3_uc_send_cmd((u32)mem.phys_base, IPA_CPU_2_HW_CMD_MHI_INIT, 0,
-		false, HZ);
-	if (res) {
-		IPAERR("ipa3_uc_send_cmd failed %d\n", res);
-		dma_free_coherent(ipa3_ctx->pdev, mem.size, mem.base,
-			mem.phys_base);
-		goto disable_clks;
-	}
-
-	dma_free_coherent(ipa3_ctx->pdev, mem.size, mem.base, mem.phys_base);
-
-	mem.size = sizeof(*msi_cmd);
-	mem.base = dma_alloc_coherent(ipa3_ctx->pdev, mem.size, &mem.phys_base,
-		GFP_KERNEL);
-	if (!mem.base) {
-		IPAERR("fail to alloc DMA buff of size %d\n", mem.size);
-		res = -ENOMEM;
-		goto disable_clks;
-	}
-
-	msi_cmd = (struct IpaHwMhiMsiCmdData_t *)mem.base;
-	msi_cmd->msiAddress_hi = msi->addr_hi;
-	msi_cmd->msiAddress_low = msi->addr_low;
-	msi_cmd->msiData = msi->data;
-	msi_cmd->msiMask = msi->mask;
-	res = ipa3_uc_send_cmd((u32)mem.phys_base,
-		IPA_CPU_2_HW_CMD_MHI_UPDATE_MSI, 0, false, HZ);
-	if (res) {
-		IPAERR("ipa3_uc_send_cmd failed %d\n", res);
-		dma_free_coherent(ipa3_ctx->pdev, mem.size, mem.base,
-			mem.phys_base);
-		goto disable_clks;
-	}
-
-	dma_free_coherent(ipa3_ctx->pdev, mem.size, mem.base, mem.phys_base);
-
-	res = 0;
-
-disable_clks:
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-	return res;
-
-}
-
-int ipa3_uc_mhi_init_channel(int ipa_ep_idx, int channelHandle,
-	int contexArrayIndex, int channelDirection)
-
-{
-	int res;
-	union IpaHwMhiInitChannelCmdData_t init_cmd;
-	union IpaHwMhiChangeChannelStateResponseData_t uc_rsp;
-
-	if (!ipa3_uc_mhi_ctx) {
-		IPAERR("Not initialized\n");
-		return -EFAULT;
-	}
-
-	if (ipa_ep_idx < 0  || ipa_ep_idx >= ipa3_ctx->ipa_num_pipes) {
-		IPAERR("Invalid ipa_ep_idx.\n");
-		return -EINVAL;
-	}
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	memset(&uc_rsp, 0, sizeof(uc_rsp));
-	uc_rsp.params.state = IPA_HW_MHI_CHANNEL_STATE_RUN;
-	uc_rsp.params.channelHandle = channelHandle;
-	ipa3_uc_mhi_ctx->expected_responseOp =
-		IPA_HW_2_CPU_RESPONSE_MHI_CHANGE_CHANNEL_STATE;
-	ipa3_uc_mhi_ctx->expected_responseParams = uc_rsp.raw32b;
-
-	memset(&init_cmd, 0, sizeof(init_cmd));
-	init_cmd.params.channelHandle = channelHandle;
-	init_cmd.params.contexArrayIndex = contexArrayIndex;
-	init_cmd.params.bamPipeId = ipa_ep_idx;
-	init_cmd.params.channelDirection = channelDirection;
-
-	res = ipa3_uc_send_cmd(init_cmd.raw32b,
-		IPA_CPU_2_HW_CMD_MHI_INIT_CHANNEL, 0, false, HZ);
-	if (res) {
-		IPAERR("ipa3_uc_send_cmd failed %d\n", res);
-		goto disable_clks;
-	}
-
-	res = 0;
-
-disable_clks:
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-	return res;
-}
-
-
-int ipa3_uc_mhi_reset_channel(int channelHandle)
-{
-	union IpaHwMhiChangeChannelStateCmdData_t cmd;
-	union IpaHwMhiChangeChannelStateResponseData_t uc_rsp;
-	int res;
-
-	if (!ipa3_uc_mhi_ctx) {
-		IPAERR("Not initialized\n");
-		return -EFAULT;
-	}
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	memset(&uc_rsp, 0, sizeof(uc_rsp));
-	uc_rsp.params.state = IPA_HW_MHI_CHANNEL_STATE_DISABLE;
-	uc_rsp.params.channelHandle = channelHandle;
-	ipa3_uc_mhi_ctx->expected_responseOp =
-		IPA_HW_2_CPU_RESPONSE_MHI_CHANGE_CHANNEL_STATE;
-	ipa3_uc_mhi_ctx->expected_responseParams = uc_rsp.raw32b;
-
-	memset(&cmd, 0, sizeof(cmd));
-	cmd.params.requestedState = IPA_HW_MHI_CHANNEL_STATE_DISABLE;
-	cmd.params.channelHandle = channelHandle;
-	res = ipa3_uc_send_cmd(cmd.raw32b,
-		IPA_CPU_2_HW_CMD_MHI_CHANGE_CHANNEL_STATE, 0, false, HZ);
-	if (res) {
-		IPAERR("ipa3_uc_send_cmd failed %d\n", res);
-		goto disable_clks;
-	}
-
-	res = 0;
-
-disable_clks:
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-	return res;
-}
-
-int ipa3_uc_mhi_suspend_channel(int channelHandle)
-{
-	union IpaHwMhiChangeChannelStateCmdData_t cmd;
-	union IpaHwMhiChangeChannelStateResponseData_t uc_rsp;
-	int res;
-
-	if (!ipa3_uc_mhi_ctx) {
-		IPAERR("Not initialized\n");
-		return -EFAULT;
-	}
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	memset(&uc_rsp, 0, sizeof(uc_rsp));
-	uc_rsp.params.state = IPA_HW_MHI_CHANNEL_STATE_SUSPEND;
-	uc_rsp.params.channelHandle = channelHandle;
-	ipa3_uc_mhi_ctx->expected_responseOp =
-		IPA_HW_2_CPU_RESPONSE_MHI_CHANGE_CHANNEL_STATE;
-	ipa3_uc_mhi_ctx->expected_responseParams = uc_rsp.raw32b;
-
-	memset(&cmd, 0, sizeof(cmd));
-	cmd.params.requestedState = IPA_HW_MHI_CHANNEL_STATE_SUSPEND;
-	cmd.params.channelHandle = channelHandle;
-	res = ipa3_uc_send_cmd(cmd.raw32b,
-		IPA_CPU_2_HW_CMD_MHI_CHANGE_CHANNEL_STATE, 0, false, HZ);
-	if (res) {
-		IPAERR("ipa3_uc_send_cmd failed %d\n", res);
-		goto disable_clks;
-	}
-
-	res = 0;
-
-disable_clks:
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-	return res;
-}
-
-int ipa3_uc_mhi_resume_channel(int channelHandle, bool LPTransitionRejected)
-{
-	union IpaHwMhiChangeChannelStateCmdData_t cmd;
-	union IpaHwMhiChangeChannelStateResponseData_t uc_rsp;
-	int res;
-
-	if (!ipa3_uc_mhi_ctx) {
-		IPAERR("Not initialized\n");
-		return -EFAULT;
-	}
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	memset(&uc_rsp, 0, sizeof(uc_rsp));
-	uc_rsp.params.state = IPA_HW_MHI_CHANNEL_STATE_RUN;
-	uc_rsp.params.channelHandle = channelHandle;
-	ipa3_uc_mhi_ctx->expected_responseOp =
-		IPA_HW_2_CPU_RESPONSE_MHI_CHANGE_CHANNEL_STATE;
-	ipa3_uc_mhi_ctx->expected_responseParams = uc_rsp.raw32b;
-
-	memset(&cmd, 0, sizeof(cmd));
-	cmd.params.requestedState = IPA_HW_MHI_CHANNEL_STATE_RUN;
-	cmd.params.channelHandle = channelHandle;
-	cmd.params.LPTransitionRejected = LPTransitionRejected;
-	res = ipa3_uc_send_cmd(cmd.raw32b,
-		IPA_CPU_2_HW_CMD_MHI_CHANGE_CHANNEL_STATE, 0, false, HZ);
-	if (res) {
-		IPAERR("ipa3_uc_send_cmd failed %d\n", res);
-		goto disable_clks;
-	}
-
-	res = 0;
-
-disable_clks:
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-	return res;
-}
-
-int ipa3_uc_mhi_stop_event_update_channel(int channelHandle)
-{
-	union IpaHwMhiStopEventUpdateData_t cmd;
-	int res;
-
-	if (!ipa3_uc_mhi_ctx) {
-		IPAERR("Not initialized\n");
-		return -EFAULT;
-	}
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	memset(&cmd, 0, sizeof(cmd));
-	cmd.params.channelHandle = channelHandle;
-
-	ipa3_uc_mhi_ctx->expected_responseOp =
-		IPA_CPU_2_HW_CMD_MHI_STOP_EVENT_UPDATE;
-	ipa3_uc_mhi_ctx->expected_responseParams = cmd.raw32b;
-
-	res = ipa3_uc_send_cmd(cmd.raw32b,
-		IPA_CPU_2_HW_CMD_MHI_STOP_EVENT_UPDATE, 0, false, HZ);
-	if (res) {
-		IPAERR("ipa3_uc_send_cmd failed %d\n", res);
-		goto disable_clks;
-	}
-
-	res = 0;
-disable_clks:
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-	return res;
-}
-
-int ipa3_uc_mhi_send_dl_ul_sync_info(union IpaHwMhiDlUlSyncCmdData_t *cmd)
-{
-	int res;
-
-	if (!ipa3_uc_mhi_ctx) {
-		IPAERR("Not initialized\n");
-		return -EFAULT;
-	}
-
-	IPADBG("isDlUlSyncEnabled=0x%x UlAccmVal=0x%x\n",
-		cmd->params.isDlUlSyncEnabled, cmd->params.UlAccmVal);
-	IPADBG("ulMsiEventThreshold=0x%x dlMsiEventThreshold=0x%x\n",
-		cmd->params.ulMsiEventThreshold,
-		cmd->params.dlMsiEventThreshold);
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	res = ipa3_uc_send_cmd(cmd->raw32b,
-		IPA_CPU_2_HW_CMD_MHI_DL_UL_SYNC_INFO, 0, false, HZ);
-	if (res) {
-		IPAERR("ipa3_uc_send_cmd failed %d\n", res);
-		goto disable_clks;
-	}
-
-	res = 0;
-disable_clks:
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-	return res;
-}
-
-int ipa3_uc_mhi_print_stats(char *dbg_buff, int size)
-{
-	int nBytes = 0;
-	int i;
-
-	if (!ipa3_uc_mhi_ctx->mhi_uc_stats_mmio) {
-		IPAERR("MHI uc stats is not valid\n");
-		return 0;
-	}
-
-	nBytes += scnprintf(&dbg_buff[nBytes], size - nBytes,
-		"Common Stats:\n");
-	PRINT_COMMON_STATS(numULDLSync);
-	PRINT_COMMON_STATS(numULTimerExpired);
-	PRINT_COMMON_STATS(numChEvCtxWpRead);
-
-	for (i = 0; i < IPA_HW_MAX_NUMBER_OF_CHANNELS; i++) {
-		nBytes += scnprintf(&dbg_buff[nBytes], size - nBytes,
-			"Channel %d Stats:\n", i);
-		PRINT_CHANNEL_STATS(i, doorbellInt);
-		PRINT_CHANNEL_STATS(i, reProccesed);
-		PRINT_CHANNEL_STATS(i, bamFifoFull);
-		PRINT_CHANNEL_STATS(i, bamFifoEmpty);
-		PRINT_CHANNEL_STATS(i, bamFifoUsageHigh);
-		PRINT_CHANNEL_STATS(i, bamFifoUsageLow);
-		PRINT_CHANNEL_STATS(i, bamInt);
-		PRINT_CHANNEL_STATS(i, ringFull);
-		PRINT_CHANNEL_STATS(i, ringEmpty);
-		PRINT_CHANNEL_STATS(i, ringUsageHigh);
-		PRINT_CHANNEL_STATS(i, ringUsageLow);
-		PRINT_CHANNEL_STATS(i, delayedMsi);
-		PRINT_CHANNEL_STATS(i, immediateMsi);
-		PRINT_CHANNEL_STATS(i, thresholdMsi);
-		PRINT_CHANNEL_STATS(i, numSuspend);
-		PRINT_CHANNEL_STATS(i, numResume);
-		PRINT_CHANNEL_STATS(i, num_OOB);
-		PRINT_CHANNEL_STATS(i, num_OOB_timer_expiry);
-		PRINT_CHANNEL_STATS(i, num_OOB_moderation_timer_start);
-		PRINT_CHANNEL_STATS(i, num_db_mode_evt);
-	}
-
-	return nBytes;
-}

ipa/ipa_v3/ipa_uc_ntn.c

@@ -1,632 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2016-2019, The Linux Foundation. All rights reserved.
- */
-
-#include "ipa_i.h"
-
-#define IPA_UC_NTN_DB_PA_TX 0x79620DC
-#define IPA_UC_NTN_DB_PA_RX 0x79620D8
-
-static void ipa3_uc_ntn_event_log_info_handler(
-struct IpaHwEventLogInfoData_t *uc_event_top_mmio)
-{
-	struct Ipa3HwEventInfoData_t *statsPtr = &uc_event_top_mmio->statsInfo;
-
-	if ((uc_event_top_mmio->protocolMask &
-		(1 << IPA_HW_PROTOCOL_ETH)) == 0) {
-		IPAERR("NTN protocol missing 0x%x\n",
-			uc_event_top_mmio->protocolMask);
-		return;
-	}
-
-	if (statsPtr->featureInfo[IPA_HW_PROTOCOL_ETH].params.size !=
-		sizeof(struct Ipa3HwStatsNTNInfoData_t)) {
-		IPAERR("NTN stats sz invalid exp=%zu is=%u\n",
-			sizeof(struct Ipa3HwStatsNTNInfoData_t),
-			statsPtr->featureInfo[IPA_HW_PROTOCOL_ETH].params.size);
-		return;
-	}
-
-	ipa3_ctx->uc_ntn_ctx.ntn_uc_stats_ofst =
-		uc_event_top_mmio->statsInfo.baseAddrOffset +
-		statsPtr->featureInfo[IPA_HW_PROTOCOL_ETH].params.offset;
-	IPAERR("NTN stats ofst=0x%x\n", ipa3_ctx->uc_ntn_ctx.ntn_uc_stats_ofst);
-	if (ipa3_ctx->uc_ntn_ctx.ntn_uc_stats_ofst +
-		sizeof(struct Ipa3HwStatsNTNInfoData_t) >=
-		ipa3_ctx->ctrl->ipa_reg_base_ofst +
-		ipahal_get_reg_n_ofst(IPA_SW_AREA_RAM_DIRECT_ACCESS_n, 0) +
-		ipa3_ctx->smem_sz) {
-		IPAERR("uc_ntn_stats 0x%x outside SRAM\n",
-			   ipa3_ctx->uc_ntn_ctx.ntn_uc_stats_ofst);
-		return;
-	}
-
-	ipa3_ctx->uc_ntn_ctx.ntn_uc_stats_mmio =
-		ioremap(ipa3_ctx->ipa_wrapper_base +
-		ipa3_ctx->uc_ntn_ctx.ntn_uc_stats_ofst,
-		sizeof(struct Ipa3HwStatsNTNInfoData_t));
-	if (!ipa3_ctx->uc_ntn_ctx.ntn_uc_stats_mmio) {
-		IPAERR("fail to ioremap uc ntn stats\n");
-		return;
-	}
-}
-
-/**
- * ipa2_get_wdi_stats() - Query WDI statistics from uc
- * @stats:	[inout] stats blob from client populated by driver
- *
- * Returns:	0 on success, negative on failure
- *
- * @note Cannot be called from atomic context
- *
- */
-int ipa3_get_ntn_stats(struct Ipa3HwStatsNTNInfoData_t *stats)
-{
-#define TX_STATS(y) stats->tx_ch_stats[0].y = \
-	ipa3_ctx->uc_ntn_ctx.ntn_uc_stats_mmio->tx_ch_stats[0].y
-#define RX_STATS(y) stats->rx_ch_stats[0].y = \
-	ipa3_ctx->uc_ntn_ctx.ntn_uc_stats_mmio->rx_ch_stats[0].y
-
-	if (unlikely(!ipa3_ctx)) {
-		IPAERR("IPA driver was not initialized\n");
-		return -EINVAL;
-	}
-
-	if (!stats || !ipa3_ctx->uc_ntn_ctx.ntn_uc_stats_mmio) {
-		IPAERR("bad parms stats=%pK ntn_stats=%pK\n",
-			stats,
-			ipa3_ctx->uc_ntn_ctx.ntn_uc_stats_mmio);
-		return -EINVAL;
-	}
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	TX_STATS(num_pkts_processed);
-	TX_STATS(ring_stats.ringFull);
-	TX_STATS(ring_stats.ringEmpty);
-	TX_STATS(ring_stats.ringUsageHigh);
-	TX_STATS(ring_stats.ringUsageLow);
-	TX_STATS(ring_stats.RingUtilCount);
-	TX_STATS(gsi_stats.bamFifoFull);
-	TX_STATS(gsi_stats.bamFifoEmpty);
-	TX_STATS(gsi_stats.bamFifoUsageHigh);
-	TX_STATS(gsi_stats.bamFifoUsageLow);
-	TX_STATS(gsi_stats.bamUtilCount);
-	TX_STATS(num_db);
-	TX_STATS(num_qmb_int_handled);
-	TX_STATS(ipa_pipe_number);
-
-	RX_STATS(num_pkts_processed);
-	RX_STATS(ring_stats.ringFull);
-	RX_STATS(ring_stats.ringEmpty);
-	RX_STATS(ring_stats.ringUsageHigh);
-	RX_STATS(ring_stats.ringUsageLow);
-	RX_STATS(ring_stats.RingUtilCount);
-	RX_STATS(gsi_stats.bamFifoFull);
-	RX_STATS(gsi_stats.bamFifoEmpty);
-	RX_STATS(gsi_stats.bamFifoUsageHigh);
-	RX_STATS(gsi_stats.bamFifoUsageLow);
-	RX_STATS(gsi_stats.bamUtilCount);
-	RX_STATS(num_db);
-	RX_STATS(num_qmb_int_handled);
-	RX_STATS(ipa_pipe_number);
-
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-
-	return 0;
-}
-
-
-int ipa3_ntn_uc_reg_rdyCB(void (*ipa_ready_cb)(void *), void *user_data)
-{
-	int ret;
-
-	if (!ipa3_ctx) {
-		IPAERR("IPA ctx is null\n");
-		return -ENXIO;
-	}
-
-	ret = ipa3_uc_state_check();
-	if (ret) {
-		ipa3_ctx->uc_ntn_ctx.uc_ready_cb = ipa_ready_cb;
-		ipa3_ctx->uc_ntn_ctx.priv = user_data;
-		return 0;
-	}
-
-	return -EEXIST;
-}
-
-void ipa3_ntn_uc_dereg_rdyCB(void)
-{
-	ipa3_ctx->uc_ntn_ctx.uc_ready_cb = NULL;
-	ipa3_ctx->uc_ntn_ctx.priv = NULL;
-}
-
-static void ipa3_uc_ntn_loaded_handler(void)
-{
-	if (!ipa3_ctx) {
-		IPAERR("IPA ctx is null\n");
-		return;
-	}
-
-	if (ipa3_ctx->uc_ntn_ctx.uc_ready_cb) {
-		ipa3_ctx->uc_ntn_ctx.uc_ready_cb(
-			ipa3_ctx->uc_ntn_ctx.priv);
-
-		ipa3_ctx->uc_ntn_ctx.uc_ready_cb =
-			NULL;
-		ipa3_ctx->uc_ntn_ctx.priv = NULL;
-	}
-}
-
-int ipa3_ntn_init(void)
-{
-	struct ipa3_uc_hdlrs uc_ntn_cbs = { 0 };
-
-	uc_ntn_cbs.ipa_uc_event_log_info_hdlr =
-		ipa3_uc_ntn_event_log_info_handler;
-	uc_ntn_cbs.ipa_uc_loaded_hdlr =
-		ipa3_uc_ntn_loaded_handler;
-
-	ipa3_uc_register_handlers(IPA_HW_FEATURE_NTN, &uc_ntn_cbs);
-
-	return 0;
-}
-
-static int ipa3_uc_send_ntn_setup_pipe_cmd(
-	struct ipa_ntn_setup_info *ntn_info, u8 dir)
-{
-	int ipa_ep_idx;
-	int result = 0;
-	struct ipa_mem_buffer cmd;
-	struct Ipa3HwNtnSetUpCmdData_t *Ntn_params;
-	struct IpaHwOffloadSetUpCmdData_t *cmd_data;
-	struct IpaHwOffloadSetUpCmdData_t_v4_0 *cmd_data_v4_0;
-
-	if (ntn_info == NULL) {
-		IPAERR("invalid input\n");
-		return -EINVAL;
-	}
-
-	ipa_ep_idx = ipa_get_ep_mapping(ntn_info->client);
-	if (ipa_ep_idx == -1) {
-		IPAERR("fail to get ep idx.\n");
-		return -EFAULT;
-	}
-
-	IPADBG("client=%d ep=%d\n", ntn_info->client, ipa_ep_idx);
-
-	IPADBG("ring_base_pa = 0x%pa\n",
-			&ntn_info->ring_base_pa);
-	IPADBG("ring_base_iova = 0x%pa\n",
-			&ntn_info->ring_base_iova);
-	IPADBG("ntn_ring_size = %d\n", ntn_info->ntn_ring_size);
-	IPADBG("buff_pool_base_pa = 0x%pa\n", &ntn_info->buff_pool_base_pa);
-	IPADBG("buff_pool_base_iova = 0x%pa\n", &ntn_info->buff_pool_base_iova);
-	IPADBG("num_buffers = %d\n", ntn_info->num_buffers);
-	IPADBG("data_buff_size = %d\n", ntn_info->data_buff_size);
-	IPADBG("tail_ptr_base_pa = 0x%pa\n", &ntn_info->ntn_reg_base_ptr_pa);
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0)
-		cmd.size = sizeof(*cmd_data_v4_0);
-	else
-		cmd.size = sizeof(*cmd_data);
-	cmd.base = dma_alloc_coherent(ipa3_ctx->uc_pdev, cmd.size,
-			&cmd.phys_base, GFP_KERNEL);
-	if (cmd.base == NULL) {
-		IPAERR("fail to get DMA memory.\n");
-		return -ENOMEM;
-	}
-
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0) {
-		cmd_data_v4_0 = (struct IpaHwOffloadSetUpCmdData_t_v4_0 *)
-			cmd.base;
-		cmd_data_v4_0->protocol = IPA_HW_PROTOCOL_ETH;
-		Ntn_params = &cmd_data_v4_0->SetupCh_params.NtnSetupCh_params;
-	} else {
-		cmd_data = (struct IpaHwOffloadSetUpCmdData_t *)cmd.base;
-		cmd_data->protocol = IPA_HW_PROTOCOL_ETH;
-		Ntn_params = &cmd_data->SetupCh_params.NtnSetupCh_params;
-	}
-
-	if (ntn_info->smmu_enabled) {
-		Ntn_params->ring_base_pa = (u32)ntn_info->ring_base_iova;
-		Ntn_params->buff_pool_base_pa =
-			(u32)ntn_info->buff_pool_base_iova;
-	} else {
-		Ntn_params->ring_base_pa = ntn_info->ring_base_pa;
-		Ntn_params->buff_pool_base_pa = ntn_info->buff_pool_base_pa;
-	}
-
-	Ntn_params->ntn_ring_size = ntn_info->ntn_ring_size;
-	Ntn_params->num_buffers = ntn_info->num_buffers;
-	Ntn_params->ntn_reg_base_ptr_pa = ntn_info->ntn_reg_base_ptr_pa;
-	Ntn_params->data_buff_size = ntn_info->data_buff_size;
-	Ntn_params->ipa_pipe_number = ipa_ep_idx;
-	Ntn_params->dir = dir;
-
-	result = ipa3_uc_send_cmd((u32)(cmd.phys_base),
-				IPA_CPU_2_HW_CMD_OFFLOAD_CHANNEL_SET_UP,
-				IPA_HW_2_CPU_OFFLOAD_CMD_STATUS_SUCCESS,
-				false, 10*HZ);
-	if (result)
-		result = -EFAULT;
-
-	dma_free_coherent(ipa3_ctx->uc_pdev, cmd.size, cmd.base, cmd.phys_base);
-	return result;
-}
-
-static int ipa3_smmu_map_uc_ntn_pipes(struct ipa_ntn_setup_info *params,
-	bool map)
-{
-	struct iommu_domain *smmu_domain;
-	int result;
-	int i;
-	u64 iova;
-	phys_addr_t pa;
-	u64 iova_p;
-	phys_addr_t pa_p;
-	u32 size_p;
-
-	if (params->data_buff_size > PAGE_SIZE) {
-		IPAERR("invalid data buff size\n");
-		return -EINVAL;
-	}
-
-	result = ipa3_smmu_map_peer_reg(rounddown(params->ntn_reg_base_ptr_pa,
-		PAGE_SIZE), map, IPA_SMMU_CB_UC);
-	if (result) {
-		IPAERR("failed to %s uC regs %d\n",
-			map ? "map" : "unmap", result);
-		goto fail;
-	}
-
-	if (params->smmu_enabled) {
-		IPADBG("smmu is enabled on EMAC\n");
-		result = ipa3_smmu_map_peer_buff((u64)params->ring_base_iova,
-			params->ntn_ring_size, map, params->ring_base_sgt,
-			IPA_SMMU_CB_UC);
-		if (result) {
-			IPAERR("failed to %s ntn ring %d\n",
-				map ? "map" : "unmap", result);
-			goto fail_map_ring;
-		}
-		result = ipa3_smmu_map_peer_buff(
-			(u64)params->buff_pool_base_iova,
-			params->num_buffers * 4, map,
-			params->buff_pool_base_sgt, IPA_SMMU_CB_UC);
-		if (result) {
-			IPAERR("failed to %s pool buffs %d\n",
-				map ? "map" : "unmap", result);
-			goto fail_map_buffer_smmu_enabled;
-		}
-	} else {
-		IPADBG("smmu is disabled on EMAC\n");
-		result = ipa3_smmu_map_peer_buff((u64)params->ring_base_pa,
-			params->ntn_ring_size, map, NULL, IPA_SMMU_CB_UC);
-		if (result) {
-			IPAERR("failed to %s ntn ring %d\n",
-				map ? "map" : "unmap", result);
-			goto fail_map_ring;
-		}
-		result = ipa3_smmu_map_peer_buff(params->buff_pool_base_pa,
-			params->num_buffers * 4, map, NULL, IPA_SMMU_CB_UC);
-		if (result) {
-			IPAERR("failed to %s pool buffs %d\n",
-				map ? "map" : "unmap", result);
-			goto fail_map_buffer_smmu_disabled;
-		}
-	}
-
-	if (ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_AP]) {
-		IPADBG("AP SMMU is set to s1 bypass\n");
-		return 0;
-	}
-
-	smmu_domain = ipa3_get_smmu_domain();
-	if (!smmu_domain) {
-		IPAERR("invalid smmu domain\n");
-		return -EINVAL;
-	}
-
-	for (i = 0; i < params->num_buffers; i++) {
-		iova = (u64)params->data_buff_list[i].iova;
-		pa = (phys_addr_t)params->data_buff_list[i].pa;
-		IPA_SMMU_ROUND_TO_PAGE(iova, pa, params->data_buff_size, iova_p,
-			pa_p, size_p);
-		IPADBG("%s 0x%llx to 0x%pa size %d\n", map ? "mapping" :
-			"unmapping", iova_p, &pa_p, size_p);
-		if (map) {
-			result = ipa3_iommu_map(smmu_domain, iova_p, pa_p,
-				size_p, IOMMU_READ | IOMMU_WRITE);
-			if (result)
-				IPAERR("Fail to map 0x%llx\n", iova);
-		} else {
-			result = iommu_unmap(smmu_domain, iova_p, size_p);
-			if (result != params->data_buff_size)
-				IPAERR("Fail to unmap 0x%llx\n", iova);
-		}
-		if (result) {
-			if (params->smmu_enabled)
-				goto fail_map_data_buff_smmu_enabled;
-			else
-				goto fail_map_data_buff_smmu_disabled;
-		}
-	}
-	return 0;
-
-fail_map_data_buff_smmu_enabled:
-	ipa3_smmu_map_peer_buff((u64)params->buff_pool_base_iova,
-		params->num_buffers * 4, !map, NULL, IPA_SMMU_CB_UC);
-	goto fail_map_buffer_smmu_enabled;
-fail_map_data_buff_smmu_disabled:
-	ipa3_smmu_map_peer_buff(params->buff_pool_base_pa,
-		params->num_buffers * 4, !map, NULL, IPA_SMMU_CB_UC);
-	goto fail_map_buffer_smmu_disabled;
-fail_map_buffer_smmu_enabled:
-	ipa3_smmu_map_peer_buff((u64)params->ring_base_iova,
-		params->ntn_ring_size, !map, params->ring_base_sgt,
-		IPA_SMMU_CB_UC);
-	goto fail_map_ring;
-fail_map_buffer_smmu_disabled:
-	ipa3_smmu_map_peer_buff((u64)params->ring_base_pa,
-			params->ntn_ring_size, !map, NULL, IPA_SMMU_CB_UC);
-fail_map_ring:
-	ipa3_smmu_map_peer_reg(rounddown(params->ntn_reg_base_ptr_pa,
-		PAGE_SIZE), !map, IPA_SMMU_CB_UC);
-fail:
-	return result;
-}
-
-/**
- * ipa3_setup_uc_ntn_pipes() - setup uc offload pipes
- */
-int ipa3_setup_uc_ntn_pipes(struct ipa_ntn_conn_in_params *in,
-	ipa_notify_cb notify, void *priv, u8 hdr_len,
-	struct ipa_ntn_conn_out_params *outp)
-{
-	struct ipa3_ep_context *ep_ul;
-	struct ipa3_ep_context *ep_dl;
-	int ipa_ep_idx_ul;
-	int ipa_ep_idx_dl;
-	int result = 0;
-
-	if (in == NULL) {
-		IPAERR("invalid input\n");
-		return -EINVAL;
-	}
-
-	ipa_ep_idx_ul = ipa_get_ep_mapping(in->ul.client);
-	if (ipa_ep_idx_ul == IPA_EP_NOT_ALLOCATED ||
-		ipa_ep_idx_ul >= IPA3_MAX_NUM_PIPES) {
-		IPAERR("fail to alloc UL EP ipa_ep_idx_ul=%d\n",
-			ipa_ep_idx_ul);
-		return -EFAULT;
-	}
-
-	ipa_ep_idx_dl = ipa_get_ep_mapping(in->dl.client);
-	if (ipa_ep_idx_dl == IPA_EP_NOT_ALLOCATED ||
-		ipa_ep_idx_dl >= IPA3_MAX_NUM_PIPES) {
-		IPAERR("fail to alloc DL EP ipa_ep_idx_dl=%d\n",
-			ipa_ep_idx_dl);
-		return -EFAULT;
-	}
-
-	ep_ul = &ipa3_ctx->ep[ipa_ep_idx_ul];
-	ep_dl = &ipa3_ctx->ep[ipa_ep_idx_dl];
-
-	if (ep_ul->valid || ep_dl->valid) {
-		IPAERR("EP already allocated ul:%d dl:%d\n",
-			   ep_ul->valid, ep_dl->valid);
-		return -EFAULT;
-	}
-
-	memset(ep_ul, 0, offsetof(struct ipa3_ep_context, sys));
-	memset(ep_dl, 0, offsetof(struct ipa3_ep_context, sys));
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	/* setup ul ep cfg */
-	ep_ul->valid = 1;
-	ep_ul->client = in->ul.client;
-	ep_ul->client_notify = notify;
-	ep_ul->priv = priv;
-
-	memset(&ep_ul->cfg, 0, sizeof(ep_ul->cfg));
-	ep_ul->cfg.nat.nat_en = IPA_SRC_NAT;
-	ep_ul->cfg.hdr.hdr_len = hdr_len;
-	ep_ul->cfg.mode.mode = IPA_BASIC;
-
-	if (ipa3_cfg_ep(ipa_ep_idx_ul, &ep_ul->cfg)) {
-		IPAERR("fail to setup ul pipe cfg\n");
-		result = -EFAULT;
-		goto fail;
-	}
-
-	result = ipa3_smmu_map_uc_ntn_pipes(&in->ul, true);
-	if (result) {
-		IPAERR("failed to map SMMU for UL %d\n", result);
-		goto fail;
-	}
-
-	result = ipa3_enable_data_path(ipa_ep_idx_ul);
-	if (result) {
-		IPAERR("Enable data path failed res=%d pipe=%d.\n", result,
-			ipa_ep_idx_ul);
-		result = -EFAULT;
-		goto fail_smmu_unmap_ul;
-	}
-
-	if (ipa3_uc_send_ntn_setup_pipe_cmd(&in->ul, IPA_NTN_RX_DIR)) {
-		IPAERR("fail to send cmd to uc for ul pipe\n");
-		result = -EFAULT;
-		goto fail_disable_dp_ul;
-	}
-	ipa3_install_dflt_flt_rules(ipa_ep_idx_ul);
-	outp->ul_uc_db_pa = IPA_UC_NTN_DB_PA_RX;
-	ep_ul->uc_offload_state |= IPA_UC_OFFLOAD_CONNECTED;
-	IPADBG("client %d (ep: %d) connected\n", in->ul.client,
-		ipa_ep_idx_ul);
-
-	/* setup dl ep cfg */
-	ep_dl->valid = 1;
-	ep_dl->client = in->dl.client;
-	memset(&ep_dl->cfg, 0, sizeof(ep_ul->cfg));
-	ep_dl->cfg.nat.nat_en = IPA_BYPASS_NAT;
-	ep_dl->cfg.hdr.hdr_len = hdr_len;
-	ep_dl->cfg.mode.mode = IPA_BASIC;
-
-	if (ipa3_cfg_ep(ipa_ep_idx_dl, &ep_dl->cfg)) {
-		IPAERR("fail to setup dl pipe cfg\n");
-		result = -EFAULT;
-		goto fail_disable_dp_ul;
-	}
-
-	result = ipa3_smmu_map_uc_ntn_pipes(&in->dl, true);
-	if (result) {
-		IPAERR("failed to map SMMU for DL %d\n", result);
-		goto fail_disable_dp_ul;
-	}
-
-	result = ipa3_enable_data_path(ipa_ep_idx_dl);
-	if (result) {
-		IPAERR("Enable data path failed res=%d pipe=%d.\n", result,
-			ipa_ep_idx_dl);
-		result = -EFAULT;
-		goto fail_smmu_unmap_dl;
-	}
-
-	if (ipa3_uc_send_ntn_setup_pipe_cmd(&in->dl, IPA_NTN_TX_DIR)) {
-		IPAERR("fail to send cmd to uc for dl pipe\n");
-		result = -EFAULT;
-		goto fail_disable_dp_dl;
-	}
-	outp->dl_uc_db_pa = IPA_UC_NTN_DB_PA_TX;
-	ep_dl->uc_offload_state |= IPA_UC_OFFLOAD_CONNECTED;
-
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-	IPADBG("client %d (ep: %d) connected\n", in->dl.client,
-		ipa_ep_idx_dl);
-
-	return 0;
-
-fail_disable_dp_dl:
-	ipa3_disable_data_path(ipa_ep_idx_dl);
-fail_smmu_unmap_dl:
-	ipa3_smmu_map_uc_ntn_pipes(&in->dl, false);
-fail_disable_dp_ul:
-	ipa3_disable_data_path(ipa_ep_idx_ul);
-fail_smmu_unmap_ul:
-	ipa3_smmu_map_uc_ntn_pipes(&in->ul, false);
-fail:
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-	return result;
-}
-
-/**
- * ipa3_tear_down_uc_offload_pipes() - tear down uc offload pipes
- */
-
-int ipa3_tear_down_uc_offload_pipes(int ipa_ep_idx_ul,
-		int ipa_ep_idx_dl, struct ipa_ntn_conn_in_params *params)
-{
-	struct ipa_mem_buffer cmd;
-	struct ipa3_ep_context *ep_ul, *ep_dl;
-	struct IpaHwOffloadCommonChCmdData_t *cmd_data;
-	struct IpaHwOffloadCommonChCmdData_t_v4_0 *cmd_data_v4_0;
-	union Ipa3HwNtnCommonChCmdData_t *tear;
-	int result = 0;
-
-	IPADBG("ep_ul = %d\n", ipa_ep_idx_ul);
-	IPADBG("ep_dl = %d\n", ipa_ep_idx_dl);
-
-	ep_ul = &ipa3_ctx->ep[ipa_ep_idx_ul];
-	ep_dl = &ipa3_ctx->ep[ipa_ep_idx_dl];
-
-	if (ep_ul->uc_offload_state != IPA_UC_OFFLOAD_CONNECTED ||
-		ep_dl->uc_offload_state != IPA_UC_OFFLOAD_CONNECTED) {
-		IPAERR("channel bad state: ul %d dl %d\n",
-			ep_ul->uc_offload_state, ep_dl->uc_offload_state);
-		return -EFAULT;
-	}
-
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0)
-		cmd.size = sizeof(*cmd_data_v4_0);
-	else
-		cmd.size = sizeof(*cmd_data);
-	cmd.base = dma_alloc_coherent(ipa3_ctx->uc_pdev, cmd.size,
-		&cmd.phys_base, GFP_KERNEL);
-	if (cmd.base == NULL) {
-		IPAERR("fail to get DMA memory.\n");
-		return -ENOMEM;
-	}
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0) {
-		cmd_data_v4_0 = (struct IpaHwOffloadCommonChCmdData_t_v4_0 *)
-			cmd.base;
-		cmd_data_v4_0->protocol = IPA_HW_PROTOCOL_ETH;
-		tear = &cmd_data_v4_0->CommonCh_params.NtnCommonCh_params;
-	} else {
-		cmd_data = (struct IpaHwOffloadCommonChCmdData_t *)cmd.base;
-		cmd_data->protocol = IPA_HW_PROTOCOL_ETH;
-		tear = &cmd_data->CommonCh_params.NtnCommonCh_params;
-	}
-
-	/* teardown the DL pipe */
-	ipa3_disable_data_path(ipa_ep_idx_dl);
-	/*
-	 * Reset ep before sending cmd otherwise disconnect
-	 * during data transfer will result into
-	 * enormous suspend interrupts
-	 */
-	memset(&ipa3_ctx->ep[ipa_ep_idx_dl], 0, sizeof(struct ipa3_ep_context));
-	IPADBG("dl client (ep: %d) disconnected\n", ipa_ep_idx_dl);
-	tear->params.ipa_pipe_number = ipa_ep_idx_dl;
-	result = ipa3_uc_send_cmd((u32)(cmd.phys_base),
-				IPA_CPU_2_HW_CMD_OFFLOAD_TEAR_DOWN,
-				IPA_HW_2_CPU_OFFLOAD_CMD_STATUS_SUCCESS,
-				false, 10*HZ);
-	if (result) {
-		IPAERR("fail to tear down dl pipe\n");
-		result = -EFAULT;
-		goto fail;
-	}
-
-	/* unmap the DL pipe */
-	result = ipa3_smmu_map_uc_ntn_pipes(&params->dl, false);
-	if (result) {
-		IPAERR("failed to unmap SMMU for DL %d\n", result);
-		goto fail;
-	}
-
-	/* teardown the UL pipe */
-	ipa3_disable_data_path(ipa_ep_idx_ul);
-
-	tear->params.ipa_pipe_number = ipa_ep_idx_ul;
-	result = ipa3_uc_send_cmd((u32)(cmd.phys_base),
-				IPA_CPU_2_HW_CMD_OFFLOAD_TEAR_DOWN,
-				IPA_HW_2_CPU_OFFLOAD_CMD_STATUS_SUCCESS,
-				false, 10*HZ);
-	if (result) {
-		IPAERR("fail to tear down ul pipe\n");
-		result = -EFAULT;
-		goto fail;
-	}
-
-	/* unmap the UL pipe */
-	result = ipa3_smmu_map_uc_ntn_pipes(&params->ul, false);
-	if (result) {
-		IPAERR("failed to unmap SMMU for UL %d\n", result);
-		goto fail;
-	}
-
-	ipa3_delete_dflt_flt_rules(ipa_ep_idx_ul);
-	memset(&ipa3_ctx->ep[ipa_ep_idx_ul], 0, sizeof(struct ipa3_ep_context));
-	IPADBG("ul client (ep: %d) disconnected\n", ipa_ep_idx_ul);
-
-fail:
-	dma_free_coherent(ipa3_ctx->uc_pdev, cmd.size, cmd.base, cmd.phys_base);
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-	return result;
-}

ipa/ipa_v3/ipa_uc_offload_i.h

@@ -1,682 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2016-2019, The Linux Foundation. All rights reserved.
- */
-
-#ifndef _IPA_UC_OFFLOAD_I_H_
-#define _IPA_UC_OFFLOAD_I_H_
-
-#include <linux/ipa.h>
-#include "ipa_i.h"
-
-/*
- * Neutrino protocol related data structures
- */
-
-#define IPA_UC_MAX_NTN_TX_CHANNELS 1
-#define IPA_UC_MAX_NTN_RX_CHANNELS 1
-
-#define IPA_NTN_TX_DIR 1
-#define IPA_NTN_RX_DIR 2
-
-#define MAX_CH_STATS_SUPPORTED 5
-#define DIR_CONSUMER 0
-#define DIR_PRODUCER 1
-
-#define MAX_AQC_CHANNELS 2
-#define MAX_11AD_CHANNELS 5
-#define MAX_WDI2_CHANNELS 2
-#define MAX_WDI3_CHANNELS 2
-#define MAX_MHIP_CHANNELS 4
-#define MAX_USB_CHANNELS 2
-
-/**
- *  @brief   Enum value determined based on the feature it
- *           corresponds to
- *  +----------------+----------------+
- *  |    3 bits      |     5 bits     |
- *  +----------------+----------------+
- *  |   HW_FEATURE   |     OPCODE     |
- *  +----------------+----------------+
- *
- */
-#define FEATURE_ENUM_VAL(feature, opcode) ((feature << 5) | opcode)
-#define EXTRACT_UC_FEATURE(value) (value >> 5)
-
-#define IPA_HW_NUM_FEATURES 0x8
-
-/**
- * enum ipa3_hw_features - Values that represent the features supported
- * in IPA HW
- * @IPA_HW_FEATURE_COMMON : Feature related to common operation of IPA HW
- * @IPA_HW_FEATURE_MHI : Feature related to MHI operation in IPA HW
- * @IPA_HW_FEATURE_POWER_COLLAPSE: Feature related to IPA Power collapse
- * @IPA_HW_FEATURE_WDI : Feature related to WDI operation in IPA HW
- * @IPA_HW_FEATURE_NTN : Feature related to NTN operation in IPA HW
- * @IPA_HW_FEATURE_OFFLOAD : Feature related to several protocols operation in
- *				IPA HW. use protocol field to
- *				 determine (e.g. IPA_HW_PROTOCOL_11ad).
- */
-enum ipa3_hw_features {
-	IPA_HW_FEATURE_COMMON		=	0x0,
-	IPA_HW_FEATURE_MHI		=	0x1,
-	IPA_HW_FEATURE_POWER_COLLAPSE	=	0x2,
-	IPA_HW_FEATURE_WDI		=	0x3,
-	IPA_HW_FEATURE_ZIP		=	0x4,
-	IPA_HW_FEATURE_NTN		=	0x5,
-	IPA_HW_FEATURE_OFFLOAD		=	0x6,
-	IPA_HW_FEATURE_MAX		=	IPA_HW_NUM_FEATURES
-};
-
-/**
-* enum ipa4_hw_protocol - Values that represent the protocols supported
-* in IPA HW when using the IPA_HW_FEATURE_OFFLOAD feature.
-* @IPA_HW_FEATURE_COMMON : protocol related to common operation of IPA HW
-* @IPA_HW_PROTOCOL_AQC : protocol related to AQC operation in IPA HW
-* @IPA_HW_PROTOCOL_11ad: protocol related to 11ad operation in IPA HW
-* @IPA_HW_PROTOCOL_WDI : protocol related to WDI operation in IPA HW
-* @IPA_HW_PROTOCOL_WDI3: protocol related to WDI3 operation in IPA HW
-* @IPA_HW_PROTOCOL_ETH : protocol related to ETH operation in IPA HW
-* @IPA_HW_PROTOCOL_MHIP: protocol related to MHIP operation in IPA HW
-* @IPA_HW_PROTOCOL_USB : protocol related to USB operation in IPA HW
-*/
-enum ipa4_hw_protocol {
-	IPA_HW_PROTOCOL_COMMON = 0x0,
-	IPA_HW_PROTOCOL_AQC = 0x1,
-	IPA_HW_PROTOCOL_11ad = 0x2,
-	IPA_HW_PROTOCOL_WDI = 0x3,
-	IPA_HW_PROTOCOL_WDI3 = 0x4,
-	IPA_HW_PROTOCOL_ETH = 0x5,
-	IPA_HW_PROTOCOL_MHIP = 0x6,
-	IPA_HW_PROTOCOL_USB = 0x7,
-	IPA_HW_PROTOCOL_MAX
-};
-
-/**
- * enum ipa3_hw_2_cpu_events - Values that represent HW event to be sent to CPU.
- * @IPA_HW_2_CPU_EVENT_NO_OP : No event present
- * @IPA_HW_2_CPU_EVENT_ERROR : Event specify a system error is detected by the
- *  device
- * @IPA_HW_2_CPU_EVENT_LOG_INFO : Event providing logging specific information
- */
-enum ipa3_hw_2_cpu_events {
-	IPA_HW_2_CPU_EVENT_NO_OP     =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_COMMON, 0),
-	IPA_HW_2_CPU_EVENT_ERROR     =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_COMMON, 1),
-	IPA_HW_2_CPU_EVENT_LOG_INFO  =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_COMMON, 2),
-};
-
-/**
- * enum ipa3_hw_errors - Common error types.
- * @IPA_HW_ERROR_NONE : No error persists
- * @IPA_HW_INVALID_DOORBELL_ERROR : Invalid data read from doorbell
- * @IPA_HW_DMA_ERROR : Unexpected DMA error
- * @IPA_HW_FATAL_SYSTEM_ERROR : HW has crashed and requires reset.
- * @IPA_HW_INVALID_OPCODE : Invalid opcode sent
- * @IPA_HW_INVALID_PARAMS : Invalid params for the requested command
- * @IPA_HW_GSI_CH_NOT_EMPTY_FAILURE : GSI channel emptiness validation failed
- * @IPA_HW_CONS_STOP_FAILURE : NTN/ETH CONS stop failed
- * @IPA_HW_PROD_STOP_FAILURE : NTN/ETH PROD stop failed
- */
-enum ipa3_hw_errors {
-	IPA_HW_ERROR_NONE              =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_COMMON, 0),
-	IPA_HW_INVALID_DOORBELL_ERROR  =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_COMMON, 1),
-	IPA_HW_DMA_ERROR               =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_COMMON, 2),
-	IPA_HW_FATAL_SYSTEM_ERROR      =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_COMMON, 3),
-	IPA_HW_INVALID_OPCODE          =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_COMMON, 4),
-	IPA_HW_INVALID_PARAMS        =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_COMMON, 5),
-	IPA_HW_CONS_DISABLE_CMD_GSI_STOP_FAILURE =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_COMMON, 6),
-	IPA_HW_PROD_DISABLE_CMD_GSI_STOP_FAILURE =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_COMMON, 7),
-	IPA_HW_GSI_CH_NOT_EMPTY_FAILURE =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_COMMON, 8),
-	IPA_HW_CONS_STOP_FAILURE =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_COMMON, 9),
-	IPA_HW_PROD_STOP_FAILURE =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_COMMON, 10)
-};
-
-/**
- * struct IpaHwSharedMemCommonMapping_t - Structure referring to the common
- * section in 128B shared memory located in offset zero of SW Partition in IPA
- * SRAM.
- * @cmdOp : CPU->HW command opcode. See IPA_CPU_2_HW_COMMANDS
- * @cmdParams : CPU->HW command parameter lower 32bit.
- * @cmdParams_hi : CPU->HW command parameter higher 32bit.
- * of parameters (immediate parameters) and point on structure in system memory
- * (in such case the address must be accessible for HW)
- * @responseOp : HW->CPU response opcode. See IPA_HW_2_CPU_RESPONSES
- * @responseParams : HW->CPU response parameter. The parameter filed can hold 32
- * bits of parameters (immediate parameters) and point on structure in system
- * memory
- * @eventOp : HW->CPU event opcode. See IPA_HW_2_CPU_EVENTS
- * @eventParams : HW->CPU event parameter. The parameter filed can hold 32
- *		bits of parameters (immediate parameters) and point on
- *		structure in system memory
- * @firstErrorAddress : Contains the address of first error-source on SNOC
- * @hwState : State of HW. The state carries information regarding the
- *				error type.
- * @warningCounter : The warnings counter. The counter carries information
- *						regarding non fatal errors in HW
- * @interfaceVersionCommon : The Common interface version as reported by HW
- * @responseParams_1: offset addr for uC stats
- *
- * The shared memory is used for communication between IPA HW and CPU.
- */
-struct IpaHwSharedMemCommonMapping_t {
-	u8  cmdOp;
-	u8  reserved_01;
-	u16 reserved_03_02;
-	u32 cmdParams;
-	u32 cmdParams_hi;
-	u8  responseOp;
-	u8  reserved_0D;
-	u16 reserved_0F_0E;
-	u32 responseParams;
-	u8  eventOp;
-	u8  reserved_15;
-	u16 reserved_17_16;
-	u32 eventParams;
-	u32 firstErrorAddress;
-	u8  hwState;
-	u8  warningCounter;
-	u16 reserved_23_22;
-	u16 interfaceVersionCommon;
-	u16 reserved_27_26;
-	u32 responseParams_1;
-} __packed;
-
-/**
- * union Ipa3HwFeatureInfoData_t - parameters for stats/config blob
- *
- * @offset : Location of a feature within the EventInfoData
- * @size : Size of the feature
- */
-union Ipa3HwFeatureInfoData_t {
-	struct IpaHwFeatureInfoParams_t {
-		u32 offset:16;
-		u32 size:16;
-	} __packed params;
-	u32 raw32b;
-} __packed;
-
-/**
- * union IpaHwErrorEventData_t - HW->CPU Common Events
- * @errorType : Entered when a system error is detected by the HW. Type of
- * error is specified by IPA_HW_ERRORS
- * @reserved : Reserved
- */
-union IpaHwErrorEventData_t {
-	struct IpaHwErrorEventParams_t {
-		u32 errorType:8;
-		u32 reserved:24;
-	} __packed params;
-	u32 raw32b;
-} __packed;
-
-/**
- * struct Ipa3HwEventInfoData_t - Structure holding the parameters for
- * statistics and config info
- *
- * @baseAddrOffset : Base Address Offset of the statistics or config
- * structure from IPA_WRAPPER_BASE
- * @Ipa3HwFeatureInfoData_t : Location and size of each feature within
- * the statistics or config structure
- *
- * @note    Information about each feature in the featureInfo[]
- * array is populated at predefined indices per the IPA_HW_FEATURES
- * enum definition
- */
-struct Ipa3HwEventInfoData_t {
-	u32 baseAddrOffset;
-	union Ipa3HwFeatureInfoData_t featureInfo[IPA_HW_NUM_FEATURES];
-} __packed;
-
-/**
- * struct IpaHwEventLogInfoData_t - Structure holding the parameters for
- * IPA_HW_2_CPU_EVENT_LOG_INFO Event
- *
- * @protocolMask : Mask indicating the protocols enabled in HW.
- * Refer IPA_HW_FEATURE_MASK
- * @circBuffBaseAddrOffset : Base Address Offset of the Circular Event
- * Log Buffer structure
- * @statsInfo : Statistics related information
- * @configInfo : Configuration related information
- *
- * @note    The offset location of this structure from IPA_WRAPPER_BASE
- * will be provided as Event Params for the IPA_HW_2_CPU_EVENT_LOG_INFO
- * Event
- */
-struct IpaHwEventLogInfoData_t {
-	u32 protocolMask;
-	u32 circBuffBaseAddrOffset;
-	struct Ipa3HwEventInfoData_t statsInfo;
-	struct Ipa3HwEventInfoData_t configInfo;
-
-} __packed;
-
-/**
- * struct ipa3_uc_ntn_ctx
- * @ntn_uc_stats_ofst: Neutrino stats offset
- * @ntn_uc_stats_mmio: Neutrino stats
- * @priv: private data of client
- * @uc_ready_cb: uc Ready cb
- */
-struct ipa3_uc_ntn_ctx {
-	u32 ntn_uc_stats_ofst;
-	struct Ipa3HwStatsNTNInfoData_t *ntn_uc_stats_mmio;
-	void *priv;
-	ipa_uc_ready_cb uc_ready_cb;
-};
-
-/**
- * enum ipa3_hw_ntn_channel_states - Values that represent NTN
- * channel state machine.
- * @IPA_HW_NTN_CHANNEL_STATE_INITED_DISABLED : Channel is
- *			initialized but disabled
- * @IPA_HW_NTN_CHANNEL_STATE_RUNNING : Channel is running.
- *     Entered after SET_UP_COMMAND is processed successfully
- * @IPA_HW_NTN_CHANNEL_STATE_ERROR : Channel is in error state
- * @IPA_HW_NTN_CHANNEL_STATE_INVALID : Invalid state. Shall not
- * be in use in operational scenario
- *
- * These states apply to both Tx and Rx paths. These do not reflect the
- * sub-state the state machine may be in.
- */
-enum ipa3_hw_ntn_channel_states {
-	IPA_HW_NTN_CHANNEL_STATE_INITED_DISABLED = 1,
-	IPA_HW_NTN_CHANNEL_STATE_RUNNING  = 2,
-	IPA_HW_NTN_CHANNEL_STATE_ERROR    = 3,
-	IPA_HW_NTN_CHANNEL_STATE_INVALID  = 0xFF
-};
-
-/**
- * enum ipa3_hw_ntn_channel_errors - List of NTN Channel error
- * types. This is present in the event param
- * @IPA_HW_NTN_CH_ERR_NONE: No error persists
- * @IPA_HW_NTN_TX_FSM_ERROR: Error in the state machine
- *		transition
- * @IPA_HW_NTN_TX_COMP_RE_FETCH_FAIL: Error while calculating
- *		num RE to bring
- * @IPA_HW_NTN_RX_RING_WP_UPDATE_FAIL: Write pointer update
- *		failed in Rx ring
- * @IPA_HW_NTN_RX_FSM_ERROR: Error in the state machine
- *		transition
- * @IPA_HW_NTN_RX_CACHE_NON_EMPTY:
- * @IPA_HW_NTN_CH_ERR_RESERVED:
- *
- * These states apply to both Tx and Rx paths. These do not
- * reflect the sub-state the state machine may be in.
- */
-enum ipa3_hw_ntn_channel_errors {
-	IPA_HW_NTN_CH_ERR_NONE            = 0,
-	IPA_HW_NTN_TX_RING_WP_UPDATE_FAIL = 1,
-	IPA_HW_NTN_TX_FSM_ERROR           = 2,
-	IPA_HW_NTN_TX_COMP_RE_FETCH_FAIL  = 3,
-	IPA_HW_NTN_RX_RING_WP_UPDATE_FAIL = 4,
-	IPA_HW_NTN_RX_FSM_ERROR           = 5,
-	IPA_HW_NTN_RX_CACHE_NON_EMPTY     = 6,
-	IPA_HW_NTN_CH_ERR_RESERVED        = 0xFF
-};
-
-
-/**
- * struct Ipa3HwNtnSetUpCmdData_t  - Ntn setup command data
- * @ring_base_pa: physical address of the base of the Tx/Rx NTN
- *  ring
- * @buff_pool_base_pa: physical address of the base of the Tx/Rx
- *  buffer pool
- * @ntn_ring_size: size of the Tx/Rx NTN ring
- * @num_buffers: Rx/tx buffer pool size
- * @ntn_reg_base_ptr_pa: physical address of the Tx/Rx NTN
- *  Ring's tail pointer
- * @ipa_pipe_number: IPA pipe number that has to be used for the
- *  Tx/Rx path
- * @dir: Tx/Rx Direction
- * @data_buff_size: size of the each data buffer allocated in
- *  DDR
- */
-struct Ipa3HwNtnSetUpCmdData_t {
-	u32 ring_base_pa;
-	u32 buff_pool_base_pa;
-	u16 ntn_ring_size;
-	u16 num_buffers;
-	u32 ntn_reg_base_ptr_pa;
-	u8  ipa_pipe_number;
-	u8  dir;
-	u16 data_buff_size;
-
-} __packed;
-
-/**
- * struct Ipa3HwNtnCommonChCmdData_t - Structure holding the
- * parameters for Ntn Tear down command data params
- *
- *@ipa_pipe_number: IPA pipe number. This could be Tx or an Rx pipe
- */
-union Ipa3HwNtnCommonChCmdData_t {
-	struct IpaHwNtnCommonChCmdParams_t {
-		u32  ipa_pipe_number :8;
-		u32  reserved        :24;
-	} __packed params;
-	uint32_t raw32b;
-} __packed;
-
-/**
- * struct NTN3RxInfoData_t - NTN Structure holding the Rx pipe
- * information
- *
- *@num_pkts_processed: Number of packets processed - cumulative
- *
- *@ring_stats:
- *@gsi_stats:
- *@num_db: Number of times the doorbell was rung
- *@num_qmb_int_handled: Number of QMB interrupts handled
- *@ipa_pipe_number: The IPA Rx/Tx pipe number.
- */
-struct NTN3RxInfoData_t {
-	u32  num_pkts_processed;
-	struct IpaHwRingStats_t ring_stats;
-	struct IpaHwBamStats_t gsi_stats;
-	u32 num_db;
-	u32 num_qmb_int_handled;
-	u32 ipa_pipe_number;
-} __packed;
-
-
-/**
- * struct NTN3TxInfoData_t - Structure holding the NTN Tx channel
- * Ensure that this is always word aligned
- *
- *@num_pkts_processed: Number of packets processed - cumulative
- *@tail_ptr_val: Latest value of doorbell written to copy engine
- *@num_db_fired: Number of DB from uC FW to Copy engine
- *
- *@tx_comp_ring_stats:
- *@bam_stats:
- *@num_db: Number of times the doorbell was rung
- *@num_qmb_int_handled: Number of QMB interrupts handled
- */
-struct NTN3TxInfoData_t {
-	u32  num_pkts_processed;
-	struct IpaHwRingStats_t ring_stats;
-	struct IpaHwBamStats_t gsi_stats;
-	u32 num_db;
-	u32 num_qmb_int_handled;
-	u32 ipa_pipe_number;
-} __packed;
-
-
-/**
- * struct Ipa3HwStatsNTNInfoData_t - Structure holding the NTN Tx
- * channel Ensure that this is always word aligned
- *
- */
-struct Ipa3HwStatsNTNInfoData_t {
-	struct NTN3RxInfoData_t rx_ch_stats[IPA_UC_MAX_NTN_RX_CHANNELS];
-	struct NTN3TxInfoData_t tx_ch_stats[IPA_UC_MAX_NTN_TX_CHANNELS];
-} __packed;
-
-
-/*
- * uC offload related data structures
- */
-#define IPA_UC_OFFLOAD_CONNECTED BIT(0)
-#define IPA_UC_OFFLOAD_ENABLED BIT(1)
-#define IPA_UC_OFFLOAD_RESUMED BIT(2)
-
-/**
- * enum ipa_cpu_2_hw_offload_commands -  Values that represent
- * the offload commands from CPU
- * @IPA_CPU_2_HW_CMD_OFFLOAD_CHANNEL_SET_UP : Command to set up
- * Offload protocol's Tx/Rx Path
- * @IPA_CPU_2_HW_CMD_OFFLOAD_TEAR_DOWN : Command to tear down
- * Offload protocol's Tx/ Rx Path
- * @IPA_CPU_2_HW_CMD_PERIPHERAL_INIT :Command to initialize peripheral
- * @IPA_CPU_2_HW_CMD_PERIPHERAL_DEINIT : Command to deinitialize peripheral
- * @IPA_CPU_2_HW_CMD_OFFLOAD_STATS_ALLOC: Command to start the
- * uC stats calculation for a particular protocol
- * @IPA_CPU_2_HW_CMD_OFFLOAD_STATS_DEALLOC: Command to stop the
- * uC stats calculation for a particular protocol
- */
-enum ipa_cpu_2_hw_offload_commands {
-	IPA_CPU_2_HW_CMD_OFFLOAD_CHANNEL_SET_UP  =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_OFFLOAD, 1),
-	IPA_CPU_2_HW_CMD_OFFLOAD_TEAR_DOWN =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_OFFLOAD, 2),
-	IPA_CPU_2_HW_CMD_PERIPHERAL_INIT =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_OFFLOAD, 3),
-	IPA_CPU_2_HW_CMD_PERIPHERAL_DEINIT =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_OFFLOAD, 4),
-	IPA_CPU_2_HW_CMD_OFFLOAD_STATS_ALLOC =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_OFFLOAD, 5),
-	IPA_CPU_2_HW_CMD_OFFLOAD_STATS_DEALLOC =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_OFFLOAD, 6),
-};
-
-/**
- * struct IpaOffloadStatschannel_info - channel info for uC
- * stats
- * @dir: Director of the channel ID DIR_CONSUMER =0,
- * DIR_PRODUCER = 1
- * @ch_id: Channel id of the IPA endpoint for which stats need
- * to be calculated, 0xFF means invalid channel or disable stats
- * on already stats enabled channel
- */
-struct IpaOffloadStatschannel_info {
-	uint8_t dir;
-	uint8_t ch_id;
-} __packed;
-
-/**
- * struct IpaHwOffloadStatsAllocCmdData_t - protocol info for uC
- * stats start
- * @protocol: Enum that indicates the protocol type
- * @ch_id_info: Channel id of the IPA endpoint for which stats
- * need to be calculated
- */
-struct IpaHwOffloadStatsAllocCmdData_t {
-	uint32_t protocol;
-	struct IpaOffloadStatschannel_info
-		ch_id_info[MAX_CH_STATS_SUPPORTED];
-} __packed;
-
-/**
- * struct IpaHwOffloadStatsDeAllocCmdData_t - protocol info for
- * uC stats stop
- * @protocol: Enum that indicates the protocol type
- */
-struct IpaHwOffloadStatsDeAllocCmdData_t {
-	uint32_t protocol;
-} __packed;
-
-/**
- * enum ipa3_hw_offload_channel_states - Values that represent
- * offload channel state machine.
- * @IPA_HW_OFFLOAD_CHANNEL_STATE_INITED_DISABLED : Channel is
- *			initialized but disabled
- * @IPA_HW_OFFLOAD_CHANNEL_STATE_RUNNING : Channel is running.
- *			Entered after SET_UP_COMMAND is processed successfully
- * @IPA_HW_OFFLOAD_CHANNEL_STATE_ERROR : Channel is in error state
- * @IPA_HW_OFFLOAD_CHANNEL_STATE_INVALID : Invalid state. Shall not
- *				be in use in operational scenario
- *
- * These states apply to both Tx and Rx paths. These do not
- * reflect the sub-state the state machine may be in
- */
-enum ipa3_hw_offload_channel_states {
-	IPA_HW_OFFLOAD_CHANNEL_STATE_INITED_DISABLED = 1,
-	IPA_HW_OFFLOAD_CHANNEL_STATE_RUNNING  = 2,
-	IPA_HW_OFFLOAD_CHANNEL_STATE_ERROR    = 3,
-	IPA_HW_OFFLOAD_CHANNEL_STATE_INVALID  = 0xFF
-};
-
-
-/**
- * enum ipa3_hw_2_cpu_cmd_resp_status -  Values that represent
- * offload related command response status to be sent to CPU.
- */
-enum ipa3_hw_2_cpu_offload_cmd_resp_status {
-	IPA_HW_2_CPU_OFFLOAD_CMD_STATUS_SUCCESS  =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_OFFLOAD, 0),
-	IPA_HW_2_CPU_OFFLOAD_MAX_TX_CHANNELS  =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_OFFLOAD, 1),
-	IPA_HW_2_CPU_OFFLOAD_TX_RING_OVERRUN_POSSIBILITY  =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_OFFLOAD, 2),
-	IPA_HW_2_CPU_OFFLOAD_TX_RING_SET_UP_FAILURE  =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_OFFLOAD, 3),
-	IPA_HW_2_CPU_OFFLOAD_TX_RING_PARAMS_UNALIGNED  =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_OFFLOAD, 4),
-	IPA_HW_2_CPU_OFFLOAD_UNKNOWN_TX_CHANNEL  =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_OFFLOAD, 5),
-	IPA_HW_2_CPU_OFFLOAD_TX_INVALID_FSM_TRANSITION  =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_OFFLOAD, 6),
-	IPA_HW_2_CPU_OFFLOAD_TX_FSM_TRANSITION_ERROR  =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_OFFLOAD, 7),
-	IPA_HW_2_CPU_OFFLOAD_MAX_RX_CHANNELS  =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_OFFLOAD, 8),
-	IPA_HW_2_CPU_OFFLOAD_RX_RING_PARAMS_UNALIGNED  =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_OFFLOAD, 9),
-	IPA_HW_2_CPU_OFFLOAD_RX_RING_SET_UP_FAILURE  =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_OFFLOAD, 10),
-	IPA_HW_2_CPU_OFFLOAD_UNKNOWN_RX_CHANNEL  =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_OFFLOAD, 11),
-	IPA_HW_2_CPU_OFFLOAD_RX_INVALID_FSM_TRANSITION  =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_OFFLOAD, 12),
-	IPA_HW_2_CPU_OFFLOAD_RX_FSM_TRANSITION_ERROR  =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_OFFLOAD, 13),
-	IPA_HW_2_CPU_OFFLOAD_RX_RING_OVERRUN_POSSIBILITY  =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_OFFLOAD, 14),
-};
-
-/**
- * struct IpaHw11adSetupCmdData_t  - 11ad setup channel command data
- * @dir: Direction RX/TX
- * @wifi_ch: 11ad peripheral pipe number
- * @gsi_ch: GSI Channel number
- * @reserved: 8 bytes padding
- * @wifi_hp_addr_lsb: Head/Tail pointer absolute address
- * @wifi_hp_addr_msb: Head/Tail pointer absolute address
- */
-struct IpaHw11adSetupCmdData_t {
-	u8 dir;
-	u8 wifi_ch;
-	u8 gsi_ch;
-	u8 reserved;
-	u32 wifi_hp_addr_lsb;
-	u32 wifi_hp_addr_msb;
-} __packed;
-
-
-/**
- * struct IpaHw11adCommonChCmdData_t - 11ad tear down channel command data
- * @gsi_ch: GSI Channel number
- * @reserved_0: padding
- * @reserved_1: padding
- */
-struct IpaHw11adCommonChCmdData_t {
-	u8 gsi_ch;
-	u8 reserved_0;
-	u16 reserved_1;
-} __packed;
-
-/**
- * struct IpaHw11adInitCmdData_t - 11ad peripheral init command data
- * @periph_baddr_lsb: Peripheral Base Address LSB (pa/IOVA)
- * @periph_baddr_msb: Peripheral Base Address MSB (pa/IOVA)
- */
-struct IpaHw11adInitCmdData_t {
-	u32 periph_baddr_lsb;
-	u32 periph_baddr_msb;
-} __packed;
-
-/**
- * struct IpaHw11adDeinitCmdData_t - 11ad peripheral deinit command data
- * @reserved: Reserved for future
- */
-struct IpaHw11adDeinitCmdData_t {
-	u32 reserved;
-};
-
-/**
- * struct IpaHwSetUpCmd  - Structure holding the parameters
- * for IPA_CPU_2_HW_CMD_OFFLOAD_CHANNEL_SET_UP
- *
- *
- */
-union IpaHwSetUpCmd {
-	struct Ipa3HwNtnSetUpCmdData_t NtnSetupCh_params;
-	struct IpaHw11adSetupCmdData_t	W11AdSetupCh_params;
-} __packed;
-
-struct IpaHwOffloadSetUpCmdData_t {
-	u8 protocol;
-	union IpaHwSetUpCmd SetupCh_params;
-} __packed;
-
-struct IpaHwOffloadSetUpCmdData_t_v4_0 {
-	u32 protocol;
-	union IpaHwSetUpCmd SetupCh_params;
-} __packed;
-
-/**
- * struct IpaHwCommonChCmd  - Structure holding the parameters
- * for IPA_CPU_2_HW_CMD_OFFLOAD_TEAR_DOWN
- *
- *
- */
-union IpaHwCommonChCmd {
-	union Ipa3HwNtnCommonChCmdData_t NtnCommonCh_params;
-	struct IpaHw11adCommonChCmdData_t W11AdCommonCh_params;
-} __packed;
-
-struct IpaHwOffloadCommonChCmdData_t {
-	u8 protocol;
-	union IpaHwCommonChCmd CommonCh_params;
-} __packed;
-
-struct IpaHwOffloadCommonChCmdData_t_v4_0 {
-	u32 protocol;
-	union IpaHwCommonChCmd CommonCh_params;
-} __packed;
-
-
-/**
- * union IpaHwPeripheralInitCmd - Structure holding the parameters
- * for IPA_CPU_2_HW_CMD_PERIPHERAL_INIT
- *
- */
-union IpaHwPeripheralInitCmd {
-	struct IpaHw11adInitCmdData_t W11AdInit_params;
-} __packed;
-
-struct IpaHwPeripheralInitCmdData_t {
-	u32 protocol;
-	union IpaHwPeripheralInitCmd Init_params;
-} __packed;
-
-/**
- * union IpaHwPeripheralDeinitCmd - Structure holding the parameters
- * for IPA_CPU_2_HW_CMD_PERIPHERAL_DEINIT
- *
- */
-union IpaHwPeripheralDeinitCmd {
-	struct IpaHw11adDeinitCmdData_t W11AdDeinit_params;
-} __packed;
-
-struct IpaHwPeripheralDeinitCmdData_t {
-	u32 protocol;
-	union IpaHwPeripheralDeinitCmd PeripheralDeinit_params;
-
-} __packed;
-
-#endif /* _IPA_UC_OFFLOAD_I_H_ */

ipa/ipa_v3/ipa_uc_wdi.c

@@ -1,3092 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2012-2019, The Linux Foundation. All rights reserved.
- */
-
-#include "ipa_i.h"
-#include <linux/dmapool.h>
-#include <linux/delay.h>
-#include <linux/mm.h>
-#include "ipa_qmi_service.h"
-
-#define IPA_HOLB_TMR_DIS 0x0
-
-#define IPA_HW_INTERFACE_WDI_VERSION 0x0001
-#define IPA_HW_WDI_RX_MBOX_START_INDEX 48
-#define IPA_HW_WDI_TX_MBOX_START_INDEX 50
-#define IPA_WDI_RING_ALIGNMENT 8
-
-#define IPA_GSI_EVT_RING_INT_MODT (32 * 1) /* 1ms under 32KHz clock */
-
-#define IPA_AGGR_PKT_LIMIT 1
-#define IPA_AGGR_HARD_BYTE_LIMIT 2 /*2 Kbytes Agger hard byte limit*/
-#define UPDATE_RI_MODERATION_THRESHOLD		8
-
-
-#define IPA_WDI_CONNECTED BIT(0)
-#define IPA_WDI_ENABLED BIT(1)
-#define IPA_WDI_RESUMED BIT(2)
-#define IPA_UC_POLL_SLEEP_USEC 100
-
-#define GSI_STOP_MAX_RETRY_CNT 10
-
-struct ipa_wdi_res {
-	struct ipa_wdi_buffer_info *res;
-	unsigned int nents;
-	bool valid;
-};
-
-static struct ipa_wdi_res wdi_res[IPA_WDI_MAX_RES];
-
-static void ipa3_uc_wdi_loaded_handler(void);
-
-/**
- * enum ipa_hw_2_cpu_wdi_events - Values that represent HW event to be sent to
- * CPU.
- * @IPA_HW_2_CPU_EVENT_WDI_ERROR : Event to specify that HW detected an error
- * in WDI
- */
-enum ipa_hw_2_cpu_wdi_events {
-	IPA_HW_2_CPU_EVENT_WDI_ERROR =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_WDI, 0),
-};
-
-/**
- * enum ipa_hw_wdi_channel_states - Values that represent WDI channel state
- * machine.
- * @IPA_HW_WDI_CHANNEL_STATE_INITED_DISABLED : Channel is initialized but
- * disabled
- * @IPA_HW_WDI_CHANNEL_STATE_ENABLED_SUSPEND : Channel is enabled but in
- * suspended state
- * @IPA_HW_WDI_CHANNEL_STATE_RUNNING : Channel is running. Entered after
- * SET_UP_COMMAND is processed successfully
- * @IPA_HW_WDI_CHANNEL_STATE_ERROR : Channel is in error state
- * @IPA_HW_WDI_CHANNEL_STATE_INVALID : Invalid state. Shall not be in use in
- * operational scenario
- *
- * These states apply to both Tx and Rx paths. These do not reflect the
- * sub-state the state machine may be in.
- */
-enum ipa_hw_wdi_channel_states {
-	IPA_HW_WDI_CHANNEL_STATE_INITED_DISABLED = 1,
-	IPA_HW_WDI_CHANNEL_STATE_ENABLED_SUSPEND = 2,
-	IPA_HW_WDI_CHANNEL_STATE_RUNNING         = 3,
-	IPA_HW_WDI_CHANNEL_STATE_ERROR           = 4,
-	IPA_HW_WDI_CHANNEL_STATE_INVALID         = 0xFF
-};
-
-/**
- * enum ipa3_cpu_2_hw_commands -  Values that represent the WDI commands from
- * CPU
- * @IPA_CPU_2_HW_CMD_WDI_TX_SET_UP : Command to set up WDI Tx Path
- * @IPA_CPU_2_HW_CMD_WDI_RX_SET_UP : Command to set up WDI Rx Path
- * @IPA_CPU_2_HW_CMD_WDI_RX_EXT_CFG : Provide extended config info for Rx path
- * @IPA_CPU_2_HW_CMD_WDI_CH_ENABLE : Command to enable a channel
- * @IPA_CPU_2_HW_CMD_WDI_CH_DISABLE : Command to disable a channel
- * @IPA_CPU_2_HW_CMD_WDI_CH_SUSPEND : Command to suspend a channel
- * @IPA_CPU_2_HW_CMD_WDI_CH_RESUME : Command to resume a channel
- * @IPA_CPU_2_HW_CMD_WDI_TEAR_DOWN : Command to tear down WDI Tx/ Rx Path
- */
-enum ipa_cpu_2_hw_wdi_commands {
-	IPA_CPU_2_HW_CMD_WDI_TX_SET_UP  =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_WDI, 0),
-	IPA_CPU_2_HW_CMD_WDI_RX_SET_UP  =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_WDI, 1),
-	IPA_CPU_2_HW_CMD_WDI_RX_EXT_CFG =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_WDI, 2),
-	IPA_CPU_2_HW_CMD_WDI_CH_ENABLE  =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_WDI, 3),
-	IPA_CPU_2_HW_CMD_WDI_CH_DISABLE =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_WDI, 4),
-	IPA_CPU_2_HW_CMD_WDI_CH_SUSPEND =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_WDI, 5),
-	IPA_CPU_2_HW_CMD_WDI_CH_RESUME  =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_WDI, 6),
-	IPA_CPU_2_HW_CMD_WDI_TEAR_DOWN  =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_WDI, 7),
-};
-
-/**
- * enum ipa_hw_2_cpu_cmd_resp_status -  Values that represent WDI related
- * command response status to be sent to CPU.
- */
-enum ipa_hw_2_cpu_cmd_resp_status {
-	IPA_HW_2_CPU_WDI_CMD_STATUS_SUCCESS            =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_WDI, 0),
-	IPA_HW_2_CPU_MAX_WDI_TX_CHANNELS               =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_WDI, 1),
-	IPA_HW_2_CPU_WDI_CE_RING_OVERRUN_POSSIBILITY   =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_WDI, 2),
-	IPA_HW_2_CPU_WDI_CE_RING_SET_UP_FAILURE        =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_WDI, 3),
-	IPA_HW_2_CPU_WDI_CE_RING_PARAMS_UNALIGNED      =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_WDI, 4),
-	IPA_HW_2_CPU_WDI_COMP_RING_OVERRUN_POSSIBILITY =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_WDI, 5),
-	IPA_HW_2_CPU_WDI_COMP_RING_SET_UP_FAILURE      =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_WDI, 6),
-	IPA_HW_2_CPU_WDI_COMP_RING_PARAMS_UNALIGNED    =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_WDI, 7),
-	IPA_HW_2_CPU_WDI_UNKNOWN_TX_CHANNEL            =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_WDI, 8),
-	IPA_HW_2_CPU_WDI_TX_INVALID_FSM_TRANSITION     =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_WDI, 9),
-	IPA_HW_2_CPU_WDI_TX_FSM_TRANSITION_ERROR       =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_WDI, 10),
-	IPA_HW_2_CPU_MAX_WDI_RX_CHANNELS               =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_WDI, 11),
-	IPA_HW_2_CPU_WDI_RX_RING_PARAMS_UNALIGNED      =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_WDI, 12),
-	IPA_HW_2_CPU_WDI_RX_RING_SET_UP_FAILURE        =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_WDI, 13),
-	IPA_HW_2_CPU_WDI_UNKNOWN_RX_CHANNEL            =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_WDI, 14),
-	IPA_HW_2_CPU_WDI_RX_INVALID_FSM_TRANSITION     =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_WDI, 15),
-	IPA_HW_2_CPU_WDI_RX_FSM_TRANSITION_ERROR       =
-		FEATURE_ENUM_VAL(IPA_HW_FEATURE_WDI, 16),
-};
-
-/**
- * enum ipa_hw_wdi_errors - WDI specific error types.
- * @IPA_HW_WDI_ERROR_NONE : No error persists
- * @IPA_HW_WDI_CHANNEL_ERROR : Error is specific to channel
- */
-enum ipa_hw_wdi_errors {
-	IPA_HW_WDI_ERROR_NONE    = 0,
-	IPA_HW_WDI_CHANNEL_ERROR = 1
-};
-
-/**
- * enum ipa_hw_wdi_ch_errors = List of WDI Channel error types. This is present
- * in the event param.
- * @IPA_HW_WDI_CH_ERR_NONE : No error persists
- * @IPA_HW_WDI_TX_COMP_RING_WP_UPDATE_FAIL : Write pointer update failed in Tx
- * Completion ring
- * @IPA_HW_WDI_TX_FSM_ERROR : Error in the state machine transition
- * @IPA_HW_WDI_TX_COMP_RE_FETCH_FAIL : Error while calculating num RE to bring
- * @IPA_HW_WDI_CH_ERR_RESERVED : Reserved - Not available for CPU to use
- */
-enum ipa_hw_wdi_ch_errors {
-	IPA_HW_WDI_CH_ERR_NONE                 = 0,
-	IPA_HW_WDI_TX_COMP_RING_WP_UPDATE_FAIL = 1,
-	IPA_HW_WDI_TX_FSM_ERROR                = 2,
-	IPA_HW_WDI_TX_COMP_RE_FETCH_FAIL       = 3,
-	IPA_HW_WDI_CH_ERR_RESERVED             = 0xFF
-};
-
-/**
- * struct IpaHwSharedMemWdiMapping_t  - Structure referring to the common and
- * WDI section of 128B shared memory located in offset zero of SW Partition in
- * IPA SRAM.
- *
- * The shared memory is used for communication between IPA HW and CPU.
- */
-struct IpaHwSharedMemWdiMapping_t {
-	struct IpaHwSharedMemCommonMapping_t common;
-	u32 reserved_2B_28;
-	u32 reserved_2F_2C;
-	u32 reserved_33_30;
-	u32 reserved_37_34;
-	u32 reserved_3B_38;
-	u32 reserved_3F_3C;
-	u16 interfaceVersionWdi;
-	u16 reserved_43_42;
-	u8  wdi_tx_ch_0_state;
-	u8  wdi_rx_ch_0_state;
-	u16 reserved_47_46;
-} __packed;
-
-/**
- * struct IpaHwWdiTxSetUpCmdData_t - Structure holding the parameters for
- * IPA_CPU_2_HW_CMD_WDI_TX_SET_UP command.
- * @comp_ring_base_pa : This is the physical address of the base of the Tx
- * completion ring
- * @comp_ring_size : This is the size of the Tx completion ring
- * @reserved_comp_ring : Reserved field for expansion of Completion ring params
- * @ce_ring_base_pa : This is the physical address of the base of the Copy
- * Engine Source Ring
- * @ce_ring_size : Copy Engine Ring size
- * @reserved_ce_ring : Reserved field for expansion of CE ring params
- * @ce_ring_doorbell_pa : This is the physical address of the doorbell that the
- * IPA uC has to write into to trigger the copy engine
- * @num_tx_buffers : Number of pkt buffers allocated. The size of the CE ring
- * and the Tx completion ring has to be atleast ( num_tx_buffers + 1)
- * @ipa_pipe_number : This is the IPA pipe number that has to be used for the
- * Tx path
- * @reserved : Reserved field
- *
- * Parameters are sent as pointer thus should be reside in address accessible
- * to HW
- */
-struct IpaHwWdiTxSetUpCmdData_t {
-	u32 comp_ring_base_pa;
-	u16 comp_ring_size;
-	u16 reserved_comp_ring;
-	u32 ce_ring_base_pa;
-	u16 ce_ring_size;
-	u16 reserved_ce_ring;
-	u32 ce_ring_doorbell_pa;
-	u16 num_tx_buffers;
-	u8  ipa_pipe_number;
-	u8  reserved;
-} __packed;
-
-struct IpaHwWdi2TxSetUpCmdData_t {
-	u32 comp_ring_base_pa;
-	u32 comp_ring_base_pa_hi;
-	u16 comp_ring_size;
-	u16 reserved_comp_ring;
-	u32 ce_ring_base_pa;
-	u32 ce_ring_base_pa_hi;
-	u16 ce_ring_size;
-	u16 reserved_ce_ring;
-	u32 ce_ring_doorbell_pa;
-	u32 ce_ring_doorbell_pa_hi;
-	u16 num_tx_buffers;
-	u8  ipa_pipe_number;
-	u8  reserved;
-} __packed;
-/**
- * struct IpaHwWdiRxSetUpCmdData_t -  Structure holding the parameters for
- * IPA_CPU_2_HW_CMD_WDI_RX_SET_UP command.
- * @rx_ring_base_pa : This is the physical address of the base of the Rx ring
- * (containing Rx buffers)
- * @rx_ring_size : This is the size of the Rx ring
- * @rx_ring_rp_pa : This is the physical address of the location through which
- * IPA uc is expected to communicate about the Read pointer into the Rx Ring
- * @ipa_pipe_number : This is the IPA pipe number that has to be used for the
- * Rx path
- *
- * Parameters are sent as pointer thus should be reside in address accessible
- * to HW
- */
-struct IpaHwWdiRxSetUpCmdData_t {
-	u32 rx_ring_base_pa;
-	u32 rx_ring_size;
-	u32 rx_ring_rp_pa;
-	u8  ipa_pipe_number;
-} __packed;
-
-struct IpaHwWdi2RxSetUpCmdData_t {
-	u32 rx_ring_base_pa;
-	u32 rx_ring_base_pa_hi;
-	u32 rx_ring_size;
-	u32 rx_ring_rp_pa;
-	u32 rx_ring_rp_pa_hi;
-	u32 rx_comp_ring_base_pa;
-	u32 rx_comp_ring_base_pa_hi;
-	u32 rx_comp_ring_size;
-	u32 rx_comp_ring_wp_pa;
-	u32 rx_comp_ring_wp_pa_hi;
-	u8  ipa_pipe_number;
-} __packed;
-/**
- * union IpaHwWdiRxExtCfgCmdData_t - Structure holding the parameters for
- * IPA_CPU_2_HW_CMD_WDI_RX_EXT_CFG command.
- * @ipa_pipe_number : The IPA pipe number for which this config is passed
- * @qmap_id : QMAP ID to be set in the metadata register
- * @reserved : Reserved
- *
- * The parameters are passed as immediate params in the shared memory
- */
-union IpaHwWdiRxExtCfgCmdData_t {
-	struct IpaHwWdiRxExtCfgCmdParams_t {
-		u32 ipa_pipe_number:8;
-		u32 qmap_id:8;
-		u32 reserved:16;
-	} __packed params;
-	u32 raw32b;
-} __packed;
-
-/**
- * union IpaHwWdiCommonChCmdData_t -  Structure holding the parameters for
- * IPA_CPU_2_HW_CMD_WDI_TEAR_DOWN,
- * IPA_CPU_2_HW_CMD_WDI_CH_ENABLE,
- * IPA_CPU_2_HW_CMD_WDI_CH_DISABLE,
- * IPA_CPU_2_HW_CMD_WDI_CH_SUSPEND,
- * IPA_CPU_2_HW_CMD_WDI_CH_RESUME command.
- * @ipa_pipe_number :  The IPA pipe number. This could be Tx or an Rx pipe
- * @reserved : Reserved
- *
- * The parameters are passed as immediate params in the shared memory
- */
-union IpaHwWdiCommonChCmdData_t {
-	struct IpaHwWdiCommonChCmdParams_t {
-		u32 ipa_pipe_number:8;
-		u32 reserved:24;
-	} __packed params;
-	u32 raw32b;
-} __packed;
-
-/**
- * union IpaHwWdiErrorEventData_t - parameters for IPA_HW_2_CPU_EVENT_WDI_ERROR
- * event.
- * @wdi_error_type : The IPA pipe number to be torn down. This could be Tx or
- * an Rx pipe
- * @reserved : Reserved
- * @ipa_pipe_number : IPA pipe number on which error has happened. Applicable
- * only if error type indicates channel error
- * @wdi_ch_err_type : Information about the channel error (if available)
- *
- * The parameters are passed as immediate params in the shared memory
- */
-union IpaHwWdiErrorEventData_t {
-	struct IpaHwWdiErrorEventParams_t {
-		u32 wdi_error_type:8;
-		u32 reserved:8;
-		u32 ipa_pipe_number:8;
-		u32 wdi_ch_err_type:8;
-	} __packed params;
-	u32 raw32b;
-} __packed;
-
-static void ipa3_uc_wdi_event_log_info_handler(
-struct IpaHwEventLogInfoData_t *uc_event_top_mmio)
-
-{
-	struct Ipa3HwEventInfoData_t *stats_ptr = &uc_event_top_mmio->statsInfo;
-
-	if ((uc_event_top_mmio->protocolMask &
-		(1 << IPA_HW_PROTOCOL_WDI)) == 0) {
-		IPAERR("WDI protocol missing 0x%x\n",
-			uc_event_top_mmio->protocolMask);
-		return;
-	}
-
-	if (stats_ptr->featureInfo[IPA_HW_PROTOCOL_WDI].params.size !=
-		sizeof(struct IpaHwStatsWDIInfoData_t)) {
-		IPAERR("wdi stats sz invalid exp=%zu is=%u\n",
-			sizeof(struct IpaHwStatsWDIInfoData_t),
-			stats_ptr->featureInfo[
-				IPA_HW_PROTOCOL_WDI].params.size);
-		return;
-	}
-
-	ipa3_ctx->uc_wdi_ctx.wdi_uc_stats_ofst =
-		stats_ptr->baseAddrOffset +
-		stats_ptr->featureInfo[IPA_HW_PROTOCOL_WDI].params.offset;
-	IPAERR("WDI stats ofst=0x%x\n", ipa3_ctx->uc_wdi_ctx.wdi_uc_stats_ofst);
-	if (ipa3_ctx->uc_wdi_ctx.wdi_uc_stats_ofst +
-		sizeof(struct IpaHwStatsWDIInfoData_t) >=
-		ipa3_ctx->ctrl->ipa_reg_base_ofst +
-		ipahal_get_reg_n_ofst(IPA_SW_AREA_RAM_DIRECT_ACCESS_n, 0) +
-		ipa3_ctx->smem_sz) {
-		IPAERR("uc_wdi_stats 0x%x outside SRAM\n",
-			ipa3_ctx->uc_wdi_ctx.wdi_uc_stats_ofst);
-		return;
-	}
-
-	ipa3_ctx->uc_wdi_ctx.wdi_uc_stats_mmio =
-		ioremap(ipa3_ctx->ipa_wrapper_base +
-		ipa3_ctx->uc_wdi_ctx.wdi_uc_stats_ofst,
-		sizeof(struct IpaHwStatsWDIInfoData_t));
-	if (!ipa3_ctx->uc_wdi_ctx.wdi_uc_stats_mmio) {
-		IPAERR("fail to ioremap uc wdi stats\n");
-		return;
-	}
-}
-
-static void ipa3_uc_wdi_event_handler(struct IpaHwSharedMemCommonMapping_t
-				     *uc_sram_mmio)
-
-{
-	union IpaHwWdiErrorEventData_t wdi_evt;
-	struct IpaHwSharedMemWdiMapping_t *wdi_sram_mmio_ext;
-
-	if (uc_sram_mmio->eventOp ==
-		IPA_HW_2_CPU_EVENT_WDI_ERROR) {
-		wdi_evt.raw32b = uc_sram_mmio->eventParams;
-		IPADBG("uC WDI evt errType=%u pipe=%d cherrType=%u\n",
-			wdi_evt.params.wdi_error_type,
-			wdi_evt.params.ipa_pipe_number,
-			wdi_evt.params.wdi_ch_err_type);
-		wdi_sram_mmio_ext =
-			(struct IpaHwSharedMemWdiMapping_t *)
-			uc_sram_mmio;
-		IPADBG("tx_ch_state=%u rx_ch_state=%u\n",
-			wdi_sram_mmio_ext->wdi_tx_ch_0_state,
-			wdi_sram_mmio_ext->wdi_rx_ch_0_state);
-	}
-}
-
-/**
- * ipa3_get_wdi_gsi_stats() - Query WDI gsi stats from uc
- * @stats:	[inout] stats blob from client populated by driver
- *
- * Returns:	0 on success, negative on failure
- *
- * @note Cannot be called from atomic context
- *
- */
-int ipa3_get_wdi_gsi_stats(struct ipa3_uc_dbg_ring_stats *stats)
-{
-	int i;
-
-	if (!ipa3_ctx->wdi2_ctx.dbg_stats.uc_dbg_stats_mmio) {
-		IPAERR("bad NULL parms for wdi_gsi_stats\n");
-		return -EINVAL;
-	}
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-	for (i = 0; i < MAX_WDI2_CHANNELS; i++) {
-		stats->ring[i].ringFull = ioread32(
-			ipa3_ctx->wdi2_ctx.dbg_stats.uc_dbg_stats_mmio
-			+ i * IPA3_UC_DEBUG_STATS_OFF +
-			IPA3_UC_DEBUG_STATS_RINGFULL_OFF);
-		stats->ring[i].ringEmpty = ioread32(
-			ipa3_ctx->wdi2_ctx.dbg_stats.uc_dbg_stats_mmio
-			+ i * IPA3_UC_DEBUG_STATS_OFF +
-			IPA3_UC_DEBUG_STATS_RINGEMPTY_OFF);
-		stats->ring[i].ringUsageHigh = ioread32(
-			ipa3_ctx->wdi2_ctx.dbg_stats.uc_dbg_stats_mmio
-			+ i * IPA3_UC_DEBUG_STATS_OFF +
-			IPA3_UC_DEBUG_STATS_RINGUSAGEHIGH_OFF);
-		stats->ring[i].ringUsageLow = ioread32(
-			ipa3_ctx->wdi2_ctx.dbg_stats.uc_dbg_stats_mmio
-			+ i * IPA3_UC_DEBUG_STATS_OFF +
-			IPA3_UC_DEBUG_STATS_RINGUSAGELOW_OFF);
-		stats->ring[i].RingUtilCount = ioread32(
-			ipa3_ctx->wdi2_ctx.dbg_stats.uc_dbg_stats_mmio
-			+ i * IPA3_UC_DEBUG_STATS_OFF +
-			IPA3_UC_DEBUG_STATS_RINGUTILCOUNT_OFF);
-	}
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-
-	return 0;
-}
-
-/**
- * ipa3_get_wdi_stats() - Query WDI statistics from uc
- * @stats:	[inout] stats blob from client populated by driver
- *
- * Returns:	0 on success, negative on failure
- *
- * @note Cannot be called from atomic context
- *
- */
-int ipa3_get_wdi_stats(struct IpaHwStatsWDIInfoData_t *stats)
-{
-#define TX_STATS(y) stats->tx_ch_stats.y = \
-	ipa3_ctx->uc_wdi_ctx.wdi_uc_stats_mmio->tx_ch_stats.y
-#define RX_STATS(y) stats->rx_ch_stats.y = \
-	ipa3_ctx->uc_wdi_ctx.wdi_uc_stats_mmio->rx_ch_stats.y
-
-	if (!stats || !ipa3_ctx->uc_wdi_ctx.wdi_uc_stats_mmio) {
-		IPAERR("bad parms stats=%pK wdi_stats=%pK\n",
-			stats,
-			ipa3_ctx->uc_wdi_ctx.wdi_uc_stats_mmio);
-		return -EINVAL;
-	}
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	TX_STATS(num_pkts_processed);
-	TX_STATS(copy_engine_doorbell_value);
-	TX_STATS(num_db_fired);
-	TX_STATS(tx_comp_ring_stats.ringFull);
-	TX_STATS(tx_comp_ring_stats.ringEmpty);
-	TX_STATS(tx_comp_ring_stats.ringUsageHigh);
-	TX_STATS(tx_comp_ring_stats.ringUsageLow);
-	TX_STATS(tx_comp_ring_stats.RingUtilCount);
-	TX_STATS(bam_stats.bamFifoFull);
-	TX_STATS(bam_stats.bamFifoEmpty);
-	TX_STATS(bam_stats.bamFifoUsageHigh);
-	TX_STATS(bam_stats.bamFifoUsageLow);
-	TX_STATS(bam_stats.bamUtilCount);
-	TX_STATS(num_db);
-	TX_STATS(num_unexpected_db);
-	TX_STATS(num_bam_int_handled);
-	TX_STATS(num_bam_int_in_non_running_state);
-	TX_STATS(num_qmb_int_handled);
-	TX_STATS(num_bam_int_handled_while_wait_for_bam);
-
-	RX_STATS(max_outstanding_pkts);
-	RX_STATS(num_pkts_processed);
-	RX_STATS(rx_ring_rp_value);
-	RX_STATS(rx_ind_ring_stats.ringFull);
-	RX_STATS(rx_ind_ring_stats.ringEmpty);
-	RX_STATS(rx_ind_ring_stats.ringUsageHigh);
-	RX_STATS(rx_ind_ring_stats.ringUsageLow);
-	RX_STATS(rx_ind_ring_stats.RingUtilCount);
-	RX_STATS(bam_stats.bamFifoFull);
-	RX_STATS(bam_stats.bamFifoEmpty);
-	RX_STATS(bam_stats.bamFifoUsageHigh);
-	RX_STATS(bam_stats.bamFifoUsageLow);
-	RX_STATS(bam_stats.bamUtilCount);
-	RX_STATS(num_bam_int_handled);
-	RX_STATS(num_db);
-	RX_STATS(num_unexpected_db);
-	RX_STATS(num_pkts_in_dis_uninit_state);
-	RX_STATS(num_ic_inj_vdev_change);
-	RX_STATS(num_ic_inj_fw_desc_change);
-	RX_STATS(num_qmb_int_handled);
-	RX_STATS(reserved1);
-	RX_STATS(reserved2);
-
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-
-	return 0;
-}
-
-int ipa3_wdi_init(void)
-{
-	struct ipa3_uc_hdlrs uc_wdi_cbs = { 0 };
-
-	uc_wdi_cbs.ipa_uc_event_hdlr = ipa3_uc_wdi_event_handler;
-	uc_wdi_cbs.ipa_uc_event_log_info_hdlr =
-		ipa3_uc_wdi_event_log_info_handler;
-	uc_wdi_cbs.ipa_uc_loaded_hdlr =
-		ipa3_uc_wdi_loaded_handler;
-
-	ipa3_uc_register_handlers(IPA_HW_FEATURE_WDI, &uc_wdi_cbs);
-
-	return 0;
-}
-
-static int ipa_create_ap_smmu_mapping_pa(phys_addr_t pa, size_t len,
-		bool device, unsigned long *iova)
-{
-	struct ipa_smmu_cb_ctx *cb = ipa3_get_smmu_ctx(IPA_SMMU_CB_AP);
-	unsigned long va = roundup(cb->next_addr, PAGE_SIZE);
-	int prot = IOMMU_READ | IOMMU_WRITE;
-	size_t true_len = roundup(len + pa - rounddown(pa, PAGE_SIZE),
-			PAGE_SIZE);
-	int ret;
-
-	if (!cb->valid) {
-		IPAERR("No SMMU CB setup\n");
-		return -EINVAL;
-	}
-
-	if (len > PAGE_SIZE)
-		va = roundup(cb->next_addr, len);
-
-	ret = ipa3_iommu_map(cb->iommu_domain, va, rounddown(pa, PAGE_SIZE),
-			true_len,
-			device ? (prot | IOMMU_MMIO) : prot);
-	if (ret) {
-		IPAERR("iommu map failed for pa=%pa len=%zu\n", &pa, true_len);
-		return -EINVAL;
-	}
-
-	ipa3_ctx->wdi_map_cnt++;
-	cb->next_addr = va + true_len;
-	*iova = va + pa - rounddown(pa, PAGE_SIZE);
-	return 0;
-}
-
-static int ipa_create_uc_smmu_mapping_pa(phys_addr_t pa, size_t len,
-		bool device, unsigned long *iova)
-{
-	struct ipa_smmu_cb_ctx *cb = ipa3_get_smmu_ctx(IPA_SMMU_CB_UC);
-	unsigned long va = roundup(cb->next_addr, PAGE_SIZE);
-	int prot = IOMMU_READ | IOMMU_WRITE;
-	size_t true_len = roundup(len + pa - rounddown(pa, PAGE_SIZE),
-			PAGE_SIZE);
-	int ret;
-
-	if (!cb->valid) {
-		IPAERR("No SMMU CB setup\n");
-		return -EINVAL;
-	}
-
-	ret = ipa3_iommu_map(cb->iommu_domain, va, rounddown(pa, PAGE_SIZE),
-			true_len,
-			device ? (prot | IOMMU_MMIO) : prot);
-	if (ret) {
-		IPAERR("iommu map failed for pa=%pa len=%zu\n", &pa, true_len);
-		return -EINVAL;
-	}
-
-	ipa3_ctx->wdi_map_cnt++;
-	cb->next_addr = va + true_len;
-	*iova = va + pa - rounddown(pa, PAGE_SIZE);
-	return 0;
-}
-
-static int ipa_create_ap_smmu_mapping_sgt(struct sg_table *sgt,
-		unsigned long *iova)
-{
-	struct ipa_smmu_cb_ctx *cb = ipa3_get_smmu_ctx(IPA_SMMU_CB_AP);
-	unsigned long va = roundup(cb->next_addr, PAGE_SIZE);
-	int prot = IOMMU_READ | IOMMU_WRITE;
-	int ret, i;
-	struct scatterlist *sg;
-	unsigned long start_iova = va;
-	phys_addr_t phys;
-	size_t len = 0;
-	int count = 0;
-
-	if (!cb->valid) {
-		IPAERR("No SMMU CB setup\n");
-		return -EINVAL;
-	}
-	if (!sgt) {
-		IPAERR("Bad parameters, scatter / gather list is NULL\n");
-		return -EINVAL;
-	}
-
-	for_each_sg(sgt->sgl, sg, sgt->nents, i) {
-		/* directly get sg_tbl PA from wlan-driver */
-		len += PAGE_ALIGN(sg->offset + sg->length);
-	}
-
-	if (len > PAGE_SIZE) {
-		va = roundup(cb->next_addr,
-				roundup_pow_of_two(len));
-		start_iova = va;
-	}
-
-	for_each_sg(sgt->sgl, sg, sgt->nents, i) {
-		/* directly get sg_tbl PA from wlan-driver */
-		phys = sg->dma_address;
-		len = PAGE_ALIGN(sg->offset + sg->length);
-
-		ret = ipa3_iommu_map(cb->iommu_domain, va, phys, len, prot);
-		if (ret) {
-			IPAERR("iommu map failed for pa=%pa len=%zu\n",
-					&phys, len);
-			goto bad_mapping;
-		}
-		va += len;
-		ipa3_ctx->wdi_map_cnt++;
-		count++;
-	}
-	cb->next_addr = va;
-	*iova = start_iova;
-
-	return 0;
-
-bad_mapping:
-	for_each_sg(sgt->sgl, sg, count, i)
-		iommu_unmap(cb->iommu_domain, sg_dma_address(sg),
-				sg_dma_len(sg));
-	return -EINVAL;
-}
-
-
-static int ipa_create_uc_smmu_mapping_sgt(struct sg_table *sgt,
-		unsigned long *iova)
-{
-	struct ipa_smmu_cb_ctx *cb = ipa3_get_smmu_ctx(IPA_SMMU_CB_UC);
-	unsigned long va = roundup(cb->next_addr, PAGE_SIZE);
-	int prot = IOMMU_READ | IOMMU_WRITE;
-	int ret;
-	int i;
-	struct scatterlist *sg;
-	unsigned long start_iova = va;
-	phys_addr_t phys;
-	size_t len;
-	int count = 0;
-
-	if (!cb->valid) {
-		IPAERR("No SMMU CB setup\n");
-		return -EINVAL;
-	}
-	if (!sgt) {
-		IPAERR("Bad parameters, scatter / gather list is NULL\n");
-		return -EINVAL;
-	}
-
-	for_each_sg(sgt->sgl, sg, sgt->nents, i) {
-		/* directly get sg_tbl PA from wlan-driver */
-		phys = sg->dma_address;
-		len = PAGE_ALIGN(sg->offset + sg->length);
-
-		ret = ipa3_iommu_map(cb->iommu_domain, va, phys, len, prot);
-		if (ret) {
-			IPAERR("iommu map failed for pa=%pa len=%zu\n",
-					&phys, len);
-			goto bad_mapping;
-		}
-		va += len;
-		ipa3_ctx->wdi_map_cnt++;
-		count++;
-	}
-	cb->next_addr = va;
-	*iova = start_iova;
-
-	return 0;
-
-bad_mapping:
-	for_each_sg(sgt->sgl, sg, count, i)
-		iommu_unmap(cb->iommu_domain, sg_dma_address(sg),
-				sg_dma_len(sg));
-	return -EINVAL;
-}
-
-static void ipa_release_ap_smmu_mappings(enum ipa_client_type client)
-{
-	struct ipa_smmu_cb_ctx *cb = ipa3_get_smmu_ctx(IPA_SMMU_CB_AP);
-	int i, j, start, end;
-
-	if (IPA_CLIENT_IS_CONS(client)) {
-		start = IPA_WDI_TX_RING_RES;
-		if (ipa3_ctx->ipa_wdi3_over_gsi)
-			end = IPA_WDI_TX_DB_RES;
-		else
-			end = IPA_WDI_CE_DB_RES;
-	} else {
-		start = IPA_WDI_RX_RING_RES;
-		if (ipa3_ctx->ipa_wdi2 ||
-			ipa3_ctx->ipa_wdi3_over_gsi)
-			end = IPA_WDI_RX_COMP_RING_WP_RES;
-		else
-			end = IPA_WDI_RX_RING_RP_RES;
-	}
-
-	for (i = start; i <= end; i++) {
-		if (wdi_res[i].valid) {
-			for (j = 0; j < wdi_res[i].nents; j++) {
-				iommu_unmap(cb->iommu_domain,
-					wdi_res[i].res[j].iova,
-					wdi_res[i].res[j].size);
-				ipa3_ctx->wdi_map_cnt--;
-			}
-			kfree(wdi_res[i].res);
-			wdi_res[i].valid = false;
-		}
-	}
-
-	if (ipa3_ctx->wdi_map_cnt == 0)
-		cb->next_addr = cb->va_end;
-}
-
-static void ipa_release_uc_smmu_mappings(enum ipa_client_type client)
-{
-	struct ipa_smmu_cb_ctx *cb = ipa3_get_smmu_ctx(IPA_SMMU_CB_UC);
-	int i;
-	int j;
-	int start;
-	int end;
-
-	if (IPA_CLIENT_IS_CONS(client)) {
-		start = IPA_WDI_TX_RING_RES;
-		end = IPA_WDI_CE_DB_RES;
-	} else {
-		start = IPA_WDI_RX_RING_RES;
-		if (ipa3_ctx->ipa_wdi2)
-			end = IPA_WDI_RX_COMP_RING_WP_RES;
-		else
-			end = IPA_WDI_RX_RING_RP_RES;
-	}
-
-	for (i = start; i <= end; i++) {
-		if (wdi_res[i].valid) {
-			for (j = 0; j < wdi_res[i].nents; j++) {
-				iommu_unmap(cb->iommu_domain,
-					wdi_res[i].res[j].iova,
-					wdi_res[i].res[j].size);
-				ipa3_ctx->wdi_map_cnt--;
-			}
-			kfree(wdi_res[i].res);
-			wdi_res[i].valid = false;
-		}
-	}
-
-	if (ipa3_ctx->wdi_map_cnt == 0)
-		cb->next_addr = cb->va_end;
-
-}
-
-static void ipa_save_uc_smmu_mapping_pa(int res_idx, phys_addr_t pa,
-		unsigned long iova, size_t len)
-{
-	IPADBG("--res_idx=%d pa=0x%pa iova=0x%lx sz=0x%zx\n", res_idx,
-		&pa, iova, len);
-	wdi_res[res_idx].res = kzalloc(sizeof(*wdi_res[res_idx].res),
-		GFP_KERNEL);
-	if (!wdi_res[res_idx].res) {
-		WARN_ON(1);
-		return;
-	}
-	wdi_res[res_idx].nents = 1;
-	wdi_res[res_idx].valid = true;
-	wdi_res[res_idx].res->pa = rounddown(pa, PAGE_SIZE);
-	wdi_res[res_idx].res->iova = rounddown(iova, PAGE_SIZE);
-	wdi_res[res_idx].res->size = roundup(len + pa - rounddown(pa,
-				PAGE_SIZE), PAGE_SIZE);
-	IPADBG("res_idx=%d pa=0x%pa iova=0x%lx sz=0x%zx\n", res_idx,
-			&wdi_res[res_idx].res->pa, wdi_res[res_idx].res->iova,
-			wdi_res[res_idx].res->size);
-}
-
-static void ipa_save_uc_smmu_mapping_sgt(int res_idx, struct sg_table *sgt,
-		unsigned long iova)
-{
-	int i;
-	struct scatterlist *sg;
-	unsigned long curr_iova = iova;
-
-	if (!sgt) {
-		IPAERR("Bad parameters, scatter / gather list is NULL\n");
-		return;
-	}
-
-	wdi_res[res_idx].res = kcalloc(sgt->nents,
-		sizeof(*wdi_res[res_idx].res),
-			GFP_KERNEL);
-	if (!wdi_res[res_idx].res) {
-		WARN_ON(1);
-		return;
-	}
-	wdi_res[res_idx].nents = sgt->nents;
-	wdi_res[res_idx].valid = true;
-	for_each_sg(sgt->sgl, sg, sgt->nents, i) {
-		/* directly get sg_tbl PA from wlan */
-		wdi_res[res_idx].res[i].pa = sg->dma_address;
-		wdi_res[res_idx].res[i].iova = curr_iova;
-		wdi_res[res_idx].res[i].size = PAGE_ALIGN(sg->offset +
-				sg->length);
-		IPADBG("res_idx=%d pa=0x%pa iova=0x%lx sz=0x%zx\n", res_idx,
-			&wdi_res[res_idx].res[i].pa,
-			wdi_res[res_idx].res[i].iova,
-			wdi_res[res_idx].res[i].size);
-		curr_iova += wdi_res[res_idx].res[i].size;
-	}
-}
-
-int ipa_create_uc_smmu_mapping(int res_idx, bool wlan_smmu_en,
-		phys_addr_t pa, struct sg_table *sgt, size_t len, bool device,
-		unsigned long *iova)
-{
-	/* support for SMMU on WLAN but no SMMU on IPA */
-	if (wlan_smmu_en && ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_UC]) {
-		IPAERR("Unsupported SMMU pairing\n");
-		return -EINVAL;
-	}
-
-	/* legacy: no SMMUs on either end */
-	if (!wlan_smmu_en && ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_UC]) {
-		*iova = pa;
-		return 0;
-	}
-
-	/* no SMMU on WLAN but SMMU on IPA */
-	if (!wlan_smmu_en && !ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_UC]) {
-		if (ipa_create_uc_smmu_mapping_pa(pa, len,
-			(res_idx == IPA_WDI_CE_DB_RES) ? true : false, iova)) {
-			IPAERR("Fail to create mapping res %d\n", res_idx);
-			return -EFAULT;
-		}
-		ipa_save_uc_smmu_mapping_pa(res_idx, pa, *iova, len);
-		return 0;
-	}
-
-	/* SMMU on WLAN and SMMU on IPA */
-	if (wlan_smmu_en && !ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_UC]) {
-		switch (res_idx) {
-		case IPA_WDI_RX_RING_RP_RES:
-		case IPA_WDI_RX_COMP_RING_WP_RES:
-		case IPA_WDI_CE_DB_RES:
-		case IPA_WDI_TX_DB_RES:
-			if (ipa_create_uc_smmu_mapping_pa(pa, len,
-				(res_idx == IPA_WDI_CE_DB_RES) ? true : false,
-				iova)) {
-				IPAERR("Fail to create mapping res %d\n",
-						res_idx);
-				return -EFAULT;
-			}
-			ipa_save_uc_smmu_mapping_pa(res_idx, pa, *iova, len);
-			break;
-		case IPA_WDI_RX_RING_RES:
-		case IPA_WDI_RX_COMP_RING_RES:
-		case IPA_WDI_TX_RING_RES:
-		case IPA_WDI_CE_RING_RES:
-			if (ipa_create_uc_smmu_mapping_sgt(sgt, iova)) {
-				IPAERR("Fail to create mapping res %d\n",
-						res_idx);
-				WARN_ON(1);
-				return -EFAULT;
-			}
-			ipa_save_uc_smmu_mapping_sgt(res_idx, sgt, *iova);
-			break;
-		default:
-			WARN_ON(1);
-		}
-	}
-
-	return 0;
-}
-
-void ipa3_release_wdi3_gsi_smmu_mappings(u8 dir)
-{
-	struct ipa_smmu_cb_ctx *cb = ipa3_get_smmu_ctx(IPA_SMMU_CB_AP);
-	int i, j, start, end;
-
-	if (dir == IPA_WDI3_TX_DIR) {
-		start = IPA_WDI_TX_RING_RES;
-		end = IPA_WDI_TX_DB_RES;
-	} else {
-		start = IPA_WDI_RX_RING_RES;
-		end = IPA_WDI_RX_COMP_RING_WP_RES;
-	}
-
-	for (i = start; i <= end; i++) {
-		if (wdi_res[i].valid) {
-			for (j = 0; j < wdi_res[i].nents; j++) {
-				iommu_unmap(cb->iommu_domain,
-					wdi_res[i].res[j].iova,
-					wdi_res[i].res[j].size);
-				ipa3_ctx->wdi_map_cnt--;
-			}
-			kfree(wdi_res[i].res);
-			wdi_res[i].valid = false;
-		}
-	}
-
-	if (ipa3_ctx->wdi_map_cnt == 0)
-		cb->next_addr = cb->va_end;
-}
-
-int ipa_create_gsi_smmu_mapping(int res_idx, bool wlan_smmu_en,
-		phys_addr_t pa, struct sg_table *sgt, size_t len, bool device,
-		unsigned long *iova)
-{
-	/* support for SMMU on WLAN but no SMMU on IPA */
-	if (wlan_smmu_en && ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_AP]) {
-		IPAERR("Unsupported SMMU pairing\n");
-		return -EINVAL;
-	}
-
-	/* legacy: no SMMUs on either end */
-	if (!wlan_smmu_en && ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_AP]) {
-		*iova = pa;
-		return 0;
-	}
-
-	/* no SMMU on WLAN but SMMU on IPA */
-	if (!wlan_smmu_en && !ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_AP]) {
-		if (ipa_create_ap_smmu_mapping_pa(pa, len,
-				(res_idx == IPA_WDI_CE_DB_RES) ? true : false,
-					iova)) {
-			IPAERR("Fail to create mapping res %d\n",
-					res_idx);
-			return -EFAULT;
-		}
-		ipa_save_uc_smmu_mapping_pa(res_idx, pa, *iova, len);
-		return 0;
-	}
-	/* SMMU on WLAN and SMMU on IPA */
-	if (wlan_smmu_en && !ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_AP]) {
-		switch (res_idx) {
-		case IPA_WDI_RX_RING_RP_RES:
-		case IPA_WDI_RX_COMP_RING_WP_RES:
-		case IPA_WDI_CE_DB_RES:
-		case IPA_WDI_TX_DB_RES:
-			if (ipa_create_ap_smmu_mapping_pa(pa, len,
-				(res_idx == IPA_WDI_CE_DB_RES) ? true : false,
-						iova)) {
-				IPAERR("Fail to create mapping res %d\n",
-						res_idx);
-				return -EFAULT;
-			}
-			ipa_save_uc_smmu_mapping_pa(res_idx, pa, *iova, len);
-			break;
-		case IPA_WDI_RX_RING_RES:
-		case IPA_WDI_RX_COMP_RING_RES:
-		case IPA_WDI_TX_RING_RES:
-		case IPA_WDI_CE_RING_RES:
-			if (ipa_create_ap_smmu_mapping_sgt(sgt, iova)) {
-				IPAERR("Fail to create mapping res %d\n",
-						res_idx);
-				return -EFAULT;
-			}
-			ipa_save_uc_smmu_mapping_sgt(res_idx, sgt, *iova);
-			break;
-		default:
-			WARN_ON(1);
-		}
-	}
-	return 0;
-}
-
-static void ipa_gsi_evt_ring_err_cb(struct gsi_evt_err_notify *notify)
-{
-	switch (notify->evt_id) {
-	case GSI_EVT_OUT_OF_BUFFERS_ERR:
-		IPAERR("Got GSI_EVT_OUT_OF_BUFFERS_ERR\n");
-		break;
-	case GSI_EVT_OUT_OF_RESOURCES_ERR:
-		IPAERR("Got GSI_EVT_OUT_OF_RESOURCES_ERR\n");
-		break;
-	case GSI_EVT_UNSUPPORTED_INTER_EE_OP_ERR:
-		IPAERR("Got GSI_EVT_UNSUPPORTED_INTER_EE_OP_ERR\n");
-		break;
-	case GSI_EVT_EVT_RING_EMPTY_ERR:
-		IPAERR("Got GSI_EVT_EVT_RING_EMPTY_ERR\n");
-		break;
-	default:
-		IPAERR("Unexpected err evt: %d\n", notify->evt_id);
-	}
-	ipa_assert();
-}
-
-static void ipa_gsi_chan_err_cb(struct gsi_chan_err_notify *notify)
-{
-	switch (notify->evt_id) {
-	case GSI_CHAN_INVALID_TRE_ERR:
-		IPAERR("Got GSI_CHAN_INVALID_TRE_ERR\n");
-		break;
-	case GSI_CHAN_NON_ALLOCATED_EVT_ACCESS_ERR:
-		IPAERR("Got GSI_CHAN_NON_ALLOCATED_EVT_ACCESS_ERR\n");
-		break;
-	case GSI_CHAN_OUT_OF_BUFFERS_ERR:
-		IPAERR("Got GSI_CHAN_OUT_OF_BUFFERS_ERR\n");
-		break;
-	case GSI_CHAN_OUT_OF_RESOURCES_ERR:
-		IPAERR("Got GSI_CHAN_OUT_OF_RESOURCES_ERR\n");
-		break;
-	case GSI_CHAN_UNSUPPORTED_INTER_EE_OP_ERR:
-		IPAERR("Got GSI_CHAN_UNSUPPORTED_INTER_EE_OP_ERR\n");
-		break;
-	case GSI_CHAN_HWO_1_ERR:
-		IPAERR("Got GSI_CHAN_HWO_1_ERR\n");
-		break;
-	default:
-		IPAERR("Unexpected err evt: %d\n", notify->evt_id);
-	}
-	ipa_assert();
-}
-static int ipa3_wdi2_gsi_alloc_evt_ring(
-			struct gsi_evt_ring_props *evt_ring_props,
-			enum ipa_client_type client,
-			unsigned long *evt_ring_hdl)
-{
-	union __packed gsi_evt_scratch evt_scratch;
-	int result = -EFAULT;
-
-	/* GSI EVENT RING allocation */
-	evt_ring_props->intf = GSI_EVT_CHTYPE_WDI2_EV;
-	evt_ring_props->intr = GSI_INTR_IRQ;
-
-	if (IPA_CLIENT_IS_PROD(client))
-		evt_ring_props->re_size = GSI_EVT_RING_RE_SIZE_8B;
-	else
-		evt_ring_props->re_size = GSI_EVT_RING_RE_SIZE_16B;
-
-	evt_ring_props->exclusive = true;
-	evt_ring_props->err_cb = ipa_gsi_evt_ring_err_cb;
-	evt_ring_props->user_data = NULL;
-	evt_ring_props->int_modt = IPA_GSI_EVT_RING_INT_MODT;
-	evt_ring_props->int_modc = 1;
-	IPADBG("GSI evt ring len: %d\n", evt_ring_props->ring_len);
-	IPADBG("client=%d moderation threshold cycles=%u cnt=%u\n",
-			client,
-			evt_ring_props->int_modt,
-			evt_ring_props->int_modc);
-
-
-	result = gsi_alloc_evt_ring(evt_ring_props,
-			ipa3_ctx->gsi_dev_hdl, evt_ring_hdl);
-	IPADBG("gsi_alloc_evt_ring result: %d\n", result);
-	if (result != GSI_STATUS_SUCCESS)
-		goto fail_alloc_evt_ring;
-
-	evt_scratch.wdi.update_ri_moderation_config =
-				UPDATE_RI_MODERATION_THRESHOLD;
-	evt_scratch.wdi.update_ri_mod_timer_running = 0;
-	evt_scratch.wdi.evt_comp_count = 0;
-	evt_scratch.wdi.last_update_ri = 0;
-	evt_scratch.wdi.resvd1 = 0;
-	evt_scratch.wdi.resvd2 = 0;
-	result = gsi_write_evt_ring_scratch(*evt_ring_hdl, evt_scratch);
-	if (result != GSI_STATUS_SUCCESS) {
-		IPAERR("Error writing WDI event ring scratch: %d\n", result);
-		gsi_dealloc_evt_ring(*evt_ring_hdl);
-		return -EFAULT;
-	}
-
-fail_alloc_evt_ring:
-	return result;
-
-}
-static int ipa3_wdi2_gsi_alloc_channel_ring(
-				struct gsi_chan_props *channel_props,
-				enum ipa_client_type client,
-				unsigned long *chan_hdl,
-				unsigned long evt_ring_hdl)
-{
-	int result = -EFAULT;
-	const struct ipa_gsi_ep_config *ep_cfg;
-
-	ep_cfg = ipa3_get_gsi_ep_info(client);
-	if (!ep_cfg) {
-		IPAERR("Failed getting GSI EP info for client=%d\n",
-				client);
-		return -EPERM;
-	}
-
-	if (IPA_CLIENT_IS_PROD(client)) {
-		IPAERR("Client is PROD\n");
-		channel_props->dir = GSI_CHAN_DIR_TO_GSI;
-		channel_props->re_size = GSI_CHAN_RE_SIZE_16B;
-	} else {
-		IPAERR("Client is CONS");
-		channel_props->dir = GSI_CHAN_DIR_FROM_GSI;
-		channel_props->re_size = GSI_CHAN_RE_SIZE_8B;
-	}
-
-	channel_props->prot = GSI_CHAN_PROT_WDI2;
-	channel_props->ch_id = ep_cfg->ipa_gsi_chan_num;
-	channel_props->evt_ring_hdl = evt_ring_hdl;
-
-	IPADBG("ch_id: %d\n", channel_props->ch_id);
-	IPADBG("evt_ring_hdl: %ld\n", channel_props->evt_ring_hdl);
-	IPADBG("re_size: %d\n", channel_props->re_size);
-	IPADBG("Config GSI xfer cb func");
-	IPADBG("GSI channel ring len: %d\n", channel_props->ring_len);
-	channel_props->xfer_cb = NULL;
-
-	IPADBG("channel ring  base vaddr = 0x%pa\n",
-			channel_props->ring_base_vaddr);
-
-	channel_props->use_db_eng = GSI_CHAN_DB_MODE;
-	channel_props->max_prefetch = GSI_ONE_PREFETCH_SEG;
-	channel_props->prefetch_mode = ep_cfg->prefetch_mode;
-	channel_props->low_weight = 1;
-	channel_props->err_cb = ipa_gsi_chan_err_cb;
-
-	IPADBG("Allocating GSI channel\n");
-	result =  gsi_alloc_channel(channel_props,
-			ipa3_ctx->gsi_dev_hdl,
-			chan_hdl);
-	if (result != GSI_STATUS_SUCCESS)
-		goto fail_alloc_channel;
-
-	IPADBG("gsi_chan_hdl: %ld\n", *chan_hdl);
-
-fail_alloc_channel:
-	return result;
-}
-
-
-int ipa3_connect_gsi_wdi_pipe(struct ipa_wdi_in_params *in,
-	struct ipa_wdi_out_params *out)
-{
-	u32 len;
-	int ipa_ep_idx, num_ring_ele;
-	int result = -EFAULT;
-	enum gsi_status gsi_res;
-	struct ipa3_ep_context *ep;
-	struct ipa_ep_cfg_ctrl ep_cfg_ctrl;
-	struct gsi_chan_props gsi_channel_props;
-	struct gsi_evt_ring_props gsi_evt_ring_props;
-	union __packed gsi_channel_scratch gsi_scratch;
-	phys_addr_t pa;
-	unsigned long va;
-	unsigned long wifi_rx_ri_addr = 0;
-	u32 gsi_db_reg_phs_addr_lsb;
-	u32 gsi_db_reg_phs_addr_msb;
-
-	ipa_ep_idx = ipa3_get_ep_mapping(in->sys.client);
-	if (ipa_ep_idx == -1) {
-		IPAERR("fail to alloc EP.\n");
-		goto fail;
-	}
-
-	ep = &ipa3_ctx->ep[ipa_ep_idx];
-
-	if (ep->valid) {
-		IPAERR("EP already allocated.\n");
-		goto fail;
-	}
-
-	IPA_ACTIVE_CLIENTS_INC_EP(in->sys.client);
-
-	memset(&ipa3_ctx->ep[ipa_ep_idx], 0, sizeof(struct ipa3_ep_context));
-	memset(&gsi_evt_ring_props, 0, sizeof(gsi_evt_ring_props));
-	memset(&gsi_channel_props, 0, sizeof(gsi_channel_props));
-	memset(&gsi_scratch, 0, sizeof(gsi_scratch));
-
-	IPADBG("client=%d ep=%d\n", in->sys.client, ipa_ep_idx);
-
-	if (IPA_CLIENT_IS_CONS(in->sys.client)) {
-		if (in->smmu_enabled) {
-			IPADBG("comp_ring_size=%d\n",
-				in->u.dl_smmu.comp_ring_size);
-			IPADBG("ce_ring_size=%d\n", in->u.dl_smmu.ce_ring_size);
-			IPADBG("ce_ring_doorbell_pa=0x%pa\n",
-					&in->u.dl_smmu.ce_door_bell_pa);
-			IPADBG("num_tx_buffers=%d\n",
-				in->u.dl_smmu.num_tx_buffers);
-		} else {
-			IPADBG("comp_ring_base_pa=0x%pa\n",
-					&in->u.dl.comp_ring_base_pa);
-			IPADBG("comp_ring_size=%d\n", in->u.dl.comp_ring_size);
-			IPADBG("ce_ring_base_pa=0x%pa\n",
-				&in->u.dl.ce_ring_base_pa);
-			IPADBG("ce_ring_size=%d\n", in->u.dl.ce_ring_size);
-			IPADBG("ce_ring_doorbell_pa=0x%pa\n",
-					&in->u.dl.ce_door_bell_pa);
-			IPADBG("num_tx_buffers=%d\n", in->u.dl.num_tx_buffers);
-		}
-	} else {
-		if (in->smmu_enabled) {
-			IPADBG("rx_ring_size=%d\n",
-				in->u.ul_smmu.rdy_ring_size);
-			IPADBG("rx_ring_rp_pa=0x%pa\n",
-				&in->u.ul_smmu.rdy_ring_rp_pa);
-			IPADBG("rx_comp_ring_size=%d\n",
-				in->u.ul_smmu.rdy_comp_ring_size);
-			IPADBG("rx_comp_ring_wp_pa=0x%pa\n",
-				&in->u.ul_smmu.rdy_comp_ring_wp_pa);
-			ipa3_ctx->wdi2_ctx.rdy_ring_rp_pa =
-				in->u.ul_smmu.rdy_ring_rp_pa;
-			ipa3_ctx->wdi2_ctx.rdy_ring_size =
-				in->u.ul_smmu.rdy_ring_size;
-			ipa3_ctx->wdi2_ctx.rdy_comp_ring_wp_pa =
-				in->u.ul_smmu.rdy_comp_ring_wp_pa;
-			ipa3_ctx->wdi2_ctx.rdy_comp_ring_size =
-				in->u.ul_smmu.rdy_comp_ring_size;
-		} else {
-			IPADBG("rx_ring_base_pa=0x%pa\n",
-				&in->u.ul.rdy_ring_base_pa);
-			IPADBG("rx_ring_size=%d\n",
-				in->u.ul.rdy_ring_size);
-			IPADBG("rx_ring_rp_pa=0x%pa\n",
-				&in->u.ul.rdy_ring_rp_pa);
-			IPADBG("rx_comp_ring_base_pa=0x%pa\n",
-				&in->u.ul.rdy_comp_ring_base_pa);
-			IPADBG("rx_comp_ring_size=%d\n",
-				in->u.ul.rdy_comp_ring_size);
-			IPADBG("rx_comp_ring_wp_pa=0x%pa\n",
-				&in->u.ul.rdy_comp_ring_wp_pa);
-			ipa3_ctx->wdi2_ctx.rdy_ring_base_pa =
-				in->u.ul.rdy_ring_base_pa;
-			ipa3_ctx->wdi2_ctx.rdy_ring_rp_pa =
-				in->u.ul.rdy_ring_rp_pa;
-			ipa3_ctx->wdi2_ctx.rdy_ring_size =
-				in->u.ul.rdy_ring_size;
-			ipa3_ctx->wdi2_ctx.rdy_comp_ring_base_pa =
-				in->u.ul.rdy_comp_ring_base_pa;
-			ipa3_ctx->wdi2_ctx.rdy_comp_ring_wp_pa =
-				in->u.ul.rdy_comp_ring_wp_pa;
-			ipa3_ctx->wdi2_ctx.rdy_comp_ring_size =
-				in->u.ul.rdy_comp_ring_size;
-		}
-	}
-	if (IPA_CLIENT_IS_CONS(in->sys.client)) {
-		len = in->smmu_enabled ? in->u.dl_smmu.comp_ring_size :
-			in->u.dl.comp_ring_size;
-		IPADBG("TX ring smmu_en=%d ring_size=%d %d\n",
-				in->smmu_enabled,
-				in->u.dl_smmu.comp_ring_size,
-				in->u.dl.comp_ring_size);
-		if (ipa_create_gsi_smmu_mapping(IPA_WDI_TX_RING_RES,
-					in->smmu_enabled,
-					in->u.dl.comp_ring_base_pa,
-					&in->u.dl_smmu.comp_ring,
-					len,
-					false,
-					&va)) {
-			IPAERR("fail to create gsi mapping TX ring.\n");
-			result = -ENOMEM;
-			goto gsi_timeout;
-		}
-		gsi_channel_props.ring_base_addr = va;
-		gsi_channel_props.ring_base_vaddr = NULL;
-		gsi_channel_props.ring_len = len;
-
-		len = in->smmu_enabled ? in->u.dl_smmu.ce_ring_size :
-			in->u.dl.ce_ring_size;
-		IPADBG("CE ring smmu_en=%d ring_size=%d %d\n",
-				in->smmu_enabled,
-				in->u.dl_smmu.ce_ring_size,
-				in->u.dl.ce_ring_size);
-
-		/* WA: wlan passed ce_ring sg_table PA directly */
-		if (ipa_create_gsi_smmu_mapping(IPA_WDI_CE_RING_RES,
-					in->smmu_enabled,
-					in->u.dl.ce_ring_base_pa,
-					&in->u.dl_smmu.ce_ring,
-					len,
-					false,
-					&va)) {
-			IPAERR("fail to create gsi mapping CE ring.\n");
-			result = -ENOMEM;
-			goto gsi_timeout;
-		}
-		gsi_evt_ring_props.ring_base_addr = va;
-		gsi_evt_ring_props.ring_base_vaddr = NULL;
-		gsi_evt_ring_props.ring_len = len;
-		pa = in->smmu_enabled ? in->u.dl_smmu.ce_door_bell_pa :
-			in->u.dl.ce_door_bell_pa;
-		if (ipa_create_gsi_smmu_mapping(IPA_WDI_CE_DB_RES,
-					in->smmu_enabled,
-					pa,
-					NULL,
-					4,
-					true,
-					&va)) {
-			IPAERR("fail to create gsi mapping CE DB.\n");
-			result = -ENOMEM;
-			goto gsi_timeout;
-		}
-		gsi_evt_ring_props.rp_update_addr = va;
-	} else {
-		len = in->smmu_enabled ? in->u.ul_smmu.rdy_ring_size :
-			in->u.ul.rdy_ring_size;
-		IPADBG("RX ring smmu_en=%d ring_size=%d %d\n",
-				in->smmu_enabled,
-				in->u.ul_smmu.rdy_ring_size,
-				in->u.ul.rdy_ring_size);
-		if (ipa_create_gsi_smmu_mapping(IPA_WDI_RX_RING_RES,
-					in->smmu_enabled,
-					in->u.ul.rdy_ring_base_pa,
-					&in->u.ul_smmu.rdy_ring,
-					len,
-					false,
-					&va)) {
-			IPAERR("fail to create gsi RX ring.\n");
-			result = -ENOMEM;
-			goto gsi_timeout;
-		}
-		gsi_channel_props.ring_base_addr = va;
-		gsi_channel_props.ring_base_vaddr =  NULL;
-		gsi_channel_props.ring_len = len;
-		pa = in->smmu_enabled ? in->u.ul_smmu.rdy_ring_rp_pa :
-			in->u.ul.rdy_ring_rp_pa;
-		if (ipa_create_gsi_smmu_mapping(IPA_WDI_RX_RING_RP_RES,
-					in->smmu_enabled,
-					pa,
-					NULL,
-					4,
-					false,
-					&wifi_rx_ri_addr)) {
-			IPAERR("fail to create gsi RX rng RP\n");
-			result = -ENOMEM;
-			goto gsi_timeout;
-		}
-		len = in->smmu_enabled ?
-			in->u.ul_smmu.rdy_comp_ring_size :
-			in->u.ul.rdy_comp_ring_size;
-		IPADBG("RX ring smmu_en=%d comp_ring_size=%d %d\n",
-				in->smmu_enabled,
-				in->u.ul_smmu.rdy_comp_ring_size,
-				in->u.ul.rdy_comp_ring_size);
-		if (ipa_create_gsi_smmu_mapping(
-					IPA_WDI_RX_COMP_RING_RES,
-					in->smmu_enabled,
-					in->u.ul.rdy_comp_ring_base_pa,
-					&in->u.ul_smmu.rdy_comp_ring,
-					len,
-					false,
-					&va)) {
-			IPAERR("fail to create gsi RX comp_ring.\n");
-			result = -ENOMEM;
-			goto gsi_timeout;
-		}
-		gsi_evt_ring_props.ring_base_addr = va;
-		gsi_evt_ring_props.ring_base_vaddr = NULL;
-		gsi_evt_ring_props.ring_len = len;
-		pa = in->smmu_enabled ?
-			in->u.ul_smmu.rdy_comp_ring_wp_pa :
-			in->u.ul.rdy_comp_ring_wp_pa;
-		if (ipa_create_gsi_smmu_mapping(
-					IPA_WDI_RX_COMP_RING_WP_RES,
-					in->smmu_enabled,
-					pa,
-					NULL,
-					4,
-					false,
-					&va)) {
-			IPAERR("fail to create gsi RX comp_rng WP\n");
-			result = -ENOMEM;
-			goto gsi_timeout;
-		}
-		gsi_evt_ring_props.rp_update_addr = va;
-	}
-
-	ep->valid = 1;
-	ep->client = in->sys.client;
-	ep->keep_ipa_awake = in->sys.keep_ipa_awake;
-	ep->skip_ep_cfg = in->sys.skip_ep_cfg;
-	ep->client_notify = in->sys.notify;
-	ep->priv = in->sys.priv;
-	if (IPA_CLIENT_IS_PROD(in->sys.client)) {
-		memset(&ep_cfg_ctrl, 0, sizeof(struct ipa_ep_cfg_ctrl));
-		ep_cfg_ctrl.ipa_ep_delay = true;
-		ipa3_cfg_ep_ctrl(ipa_ep_idx, &ep_cfg_ctrl);
-	}
-
-	if (IPA_CLIENT_IS_CONS(in->sys.client)) {
-		in->sys.ipa_ep_cfg.aggr.aggr_en = IPA_ENABLE_AGGR;
-		in->sys.ipa_ep_cfg.aggr.aggr = IPA_GENERIC;
-		in->sys.ipa_ep_cfg.aggr.aggr_pkt_limit = IPA_AGGR_PKT_LIMIT;
-		in->sys.ipa_ep_cfg.aggr.aggr_byte_limit =
-						IPA_AGGR_HARD_BYTE_LIMIT;
-		in->sys.ipa_ep_cfg.aggr.aggr_hard_byte_limit_en =
-						IPA_ENABLE_AGGR;
-	}
-	if (!ep->skip_ep_cfg) {
-		if (ipa3_cfg_ep(ipa_ep_idx, &in->sys.ipa_ep_cfg)) {
-			IPAERR("fail to configure EP.\n");
-			goto ipa_cfg_ep_fail;
-		}
-		IPADBG("ep configuration successful\n");
-	} else {
-		IPADBG("Skipping endpoint configuration.\n");
-	}
-	result = ipa3_wdi2_gsi_alloc_evt_ring(&gsi_evt_ring_props,
-				in->sys.client,
-				&ep->gsi_evt_ring_hdl);
-	if (result)
-		goto fail_alloc_evt_ring;
-	/*copy mem info */
-	ep->gsi_mem_info.evt_ring_len = gsi_evt_ring_props.ring_len;
-	ep->gsi_mem_info.evt_ring_base_addr = gsi_evt_ring_props.ring_base_addr;
-	ep->gsi_mem_info.evt_ring_base_vaddr =
-				gsi_evt_ring_props.ring_base_vaddr;
-	IPAERR("evt ring len: %d\n", ep->gsi_mem_info.evt_ring_len);
-	IPAERR("element size: %d\n", gsi_evt_ring_props.re_size);
-
-	result = ipa3_wdi2_gsi_alloc_channel_ring(&gsi_channel_props,
-					in->sys.client,
-				&ep->gsi_chan_hdl, ep->gsi_evt_ring_hdl);
-	if (result)
-		goto fail_alloc_channel;
-	ep->gsi_mem_info.chan_ring_len = gsi_channel_props.ring_len;
-	ep->gsi_mem_info.chan_ring_base_addr = gsi_channel_props.ring_base_addr;
-	ep->gsi_mem_info.chan_ring_base_vaddr =
-		gsi_channel_props.ring_base_vaddr;
-
-	num_ring_ele = ep->gsi_mem_info.evt_ring_len/gsi_evt_ring_props.re_size;
-	IPAERR("UPDATE_RI_MODERATION_THRESHOLD: %d\n", num_ring_ele);
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_7) {
-		if (IPA_CLIENT_IS_PROD(in->sys.client)) {
-			gsi_scratch.wdi.wifi_rx_ri_addr_low =
-				wifi_rx_ri_addr & 0xFFFFFFFF;
-			gsi_scratch.wdi.wifi_rx_ri_addr_high =
-				(wifi_rx_ri_addr & 0xFFFFF00000000) >> 32;
-			gsi_scratch.wdi.wdi_rx_vdev_id = 0xff;
-			gsi_scratch.wdi.wdi_rx_fw_desc = 0xff;
-			gsi_scratch.wdi.endp_metadatareg_offset =
-						ipahal_get_reg_mn_ofst(
-						IPA_ENDP_INIT_HDR_METADATA_n, 0,
-								ipa_ep_idx)/4;
-			gsi_scratch.wdi.qmap_id = 0;
-		}
-		gsi_scratch.wdi.update_ri_moderation_threshold =
-			min(UPDATE_RI_MODERATION_THRESHOLD, num_ring_ele);
-		gsi_scratch.wdi.update_ri_moderation_counter = 0;
-		gsi_scratch.wdi.wdi_rx_tre_proc_in_progress = 0;
-	} else {
-		if (IPA_CLIENT_IS_PROD(in->sys.client)) {
-			gsi_scratch.wdi2_new.wifi_rx_ri_addr_low =
-				wifi_rx_ri_addr & 0xFFFFFFFF;
-			gsi_scratch.wdi2_new.wifi_rx_ri_addr_high =
-				(wifi_rx_ri_addr & 0xFFFFF00000000) >> 32;
-			gsi_scratch.wdi2_new.wdi_rx_vdev_id = 0xff;
-			gsi_scratch.wdi2_new.wdi_rx_fw_desc = 0xff;
-			gsi_scratch.wdi2_new.endp_metadatareg_offset =
-						ipahal_get_reg_mn_ofst(
-						IPA_ENDP_INIT_HDR_METADATA_n, 0,
-								ipa_ep_idx)/4;
-			gsi_scratch.wdi2_new.qmap_id = 0;
-		}
-		gsi_scratch.wdi2_new.update_ri_moderation_threshold =
-			min(UPDATE_RI_MODERATION_THRESHOLD, num_ring_ele);
-		gsi_scratch.wdi2_new.update_ri_moderation_counter = 0;
-		gsi_scratch.wdi2_new.wdi_rx_tre_proc_in_progress = 0;
-	}
-
-	result = gsi_write_channel_scratch(ep->gsi_chan_hdl,
-			gsi_scratch);
-	if (result != GSI_STATUS_SUCCESS) {
-		IPAERR("gsi_write_channel_scratch failed %d\n",
-				result);
-		goto fail_write_channel_scratch;
-	}
-
-	/* for AP+STA stats update */
-	if (in->wdi_notify)
-		ipa3_ctx->uc_wdi_ctx.stats_notify = in->wdi_notify;
-	else
-		IPADBG("in->wdi_notify is null\n");
-
-	ipa3_enable_data_path(ipa_ep_idx);
-
-	if (!ep->skip_ep_cfg && IPA_CLIENT_IS_PROD(in->sys.client))
-		ipa3_install_dflt_flt_rules(ipa_ep_idx);
-
-	if (!ep->keep_ipa_awake)
-		IPA_ACTIVE_CLIENTS_DEC_EP(in->sys.client);
-
-	IPADBG("GSI connected.\n");
-	gsi_res = gsi_query_channel_db_addr(ep->gsi_chan_hdl,
-			&gsi_db_reg_phs_addr_lsb,
-			&gsi_db_reg_phs_addr_msb);
-	out->uc_door_bell_pa = gsi_db_reg_phs_addr_lsb;
-	IPADBG("GSI query result: %d\n", gsi_res);
-	IPADBG("GSI lsb addr: %d\n", gsi_db_reg_phs_addr_lsb);
-	IPADBG("GSI msb addr: %d\n", gsi_db_reg_phs_addr_msb);
-
-	ep->gsi_offload_state |= IPA_WDI_CONNECTED;
-	out->clnt_hdl = ipa_ep_idx;
-	return 0;
-
-fail_write_channel_scratch:
-	gsi_dealloc_channel(ep->gsi_chan_hdl);
-fail_alloc_channel:
-	if (ep->gsi_evt_ring_hdl != ~0) {
-		gsi_dealloc_evt_ring(ep->gsi_evt_ring_hdl);
-		ep->gsi_evt_ring_hdl = ~0;
-	}
-fail_alloc_evt_ring:
-ipa_cfg_ep_fail:
-	memset(&ipa3_ctx->ep[ipa_ep_idx], 0, sizeof(struct ipa3_ep_context));
-gsi_timeout:
-	ipa_release_ap_smmu_mappings(in->sys.client);
-	IPA_ACTIVE_CLIENTS_DEC_EP(in->sys.client);
-fail:
-	return result;
-}
-
-/**
- * ipa3_connect_wdi_pipe() - WDI client connect
- * @in:	[in] input parameters from client
- * @out: [out] output params to client
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_connect_wdi_pipe(struct ipa_wdi_in_params *in,
-		struct ipa_wdi_out_params *out)
-{
-	int ipa_ep_idx;
-	int result = -EFAULT;
-	struct ipa3_ep_context *ep;
-	struct ipa_mem_buffer cmd;
-	struct IpaHwWdiTxSetUpCmdData_t *tx;
-	struct IpaHwWdiRxSetUpCmdData_t *rx;
-	struct IpaHwWdi2TxSetUpCmdData_t *tx_2;
-	struct IpaHwWdi2RxSetUpCmdData_t *rx_2;
-
-	struct ipa_ep_cfg_ctrl ep_cfg_ctrl;
-	unsigned long va;
-	phys_addr_t pa;
-	u32 len;
-
-	if (in == NULL || out == NULL || in->sys.client >= IPA_CLIENT_MAX) {
-		IPAERR("bad parm. in=%pK out=%pK\n", in, out);
-		if (in)
-			IPAERR("client = %d\n", in->sys.client);
-		return -EINVAL;
-	}
-
-	if (!in->smmu_enabled) {
-		if (IPA_CLIENT_IS_CONS(in->sys.client)) {
-			if (in->u.dl.comp_ring_base_pa %
-				IPA_WDI_RING_ALIGNMENT ||
-				in->u.dl.ce_ring_base_pa %
-				IPA_WDI_RING_ALIGNMENT) {
-				IPAERR("alignment failure on TX\n");
-					return -EINVAL;
-			}
-		} else {
-			if (in->u.ul.rdy_ring_base_pa %
-				IPA_WDI_RING_ALIGNMENT) {
-				IPAERR("alignment failure on RX\n");
-				return -EINVAL;
-			}
-		}
-	}
-
-	if (ipa3_ctx->ipa_wdi2_over_gsi)
-		return ipa3_connect_gsi_wdi_pipe(in, out);
-
-	result = ipa3_uc_state_check();
-	if (result)
-		return result;
-
-	ipa_ep_idx = ipa3_get_ep_mapping(in->sys.client);
-	if (ipa_ep_idx == -1) {
-		IPAERR("fail to alloc EP.\n");
-		goto fail;
-	}
-
-	ep = &ipa3_ctx->ep[ipa_ep_idx];
-
-	if (ep->valid) {
-		IPAERR("EP already allocated.\n");
-		goto fail;
-	}
-
-	memset(&ipa3_ctx->ep[ipa_ep_idx], 0, sizeof(struct ipa3_ep_context));
-	IPA_ACTIVE_CLIENTS_INC_EP(in->sys.client);
-
-	IPADBG("client=%d ep=%d\n", in->sys.client, ipa_ep_idx);
-	if (IPA_CLIENT_IS_CONS(in->sys.client)) {
-		if (ipa3_ctx->ipa_wdi2)
-			cmd.size = sizeof(*tx_2);
-		else
-			cmd.size = sizeof(*tx);
-		if (in->smmu_enabled) {
-			IPADBG("comp_ring_size=%d\n",
-				in->u.dl_smmu.comp_ring_size);
-			IPADBG("ce_ring_size=%d\n", in->u.dl_smmu.ce_ring_size);
-			IPADBG("ce_ring_doorbell_pa=0x%pa\n",
-					&in->u.dl_smmu.ce_door_bell_pa);
-			IPADBG("num_tx_buffers=%d\n",
-				in->u.dl_smmu.num_tx_buffers);
-		} else {
-			IPADBG("comp_ring_base_pa=0x%pa\n",
-					&in->u.dl.comp_ring_base_pa);
-			IPADBG("comp_ring_size=%d\n", in->u.dl.comp_ring_size);
-			IPADBG("ce_ring_base_pa=0x%pa\n",
-				&in->u.dl.ce_ring_base_pa);
-			IPADBG("ce_ring_size=%d\n", in->u.dl.ce_ring_size);
-			IPADBG("ce_ring_doorbell_pa=0x%pa\n",
-					&in->u.dl.ce_door_bell_pa);
-			IPADBG("num_tx_buffers=%d\n", in->u.dl.num_tx_buffers);
-		}
-	} else {
-		if (ipa3_ctx->ipa_wdi2)
-			cmd.size = sizeof(*rx_2);
-		else
-			cmd.size = sizeof(*rx);
-		if (in->smmu_enabled) {
-			IPADBG("rx_ring_size=%d\n",
-				in->u.ul_smmu.rdy_ring_size);
-			IPADBG("rx_ring_rp_pa=0x%pa\n",
-				&in->u.ul_smmu.rdy_ring_rp_pa);
-			IPADBG("rx_comp_ring_size=%d\n",
-				in->u.ul_smmu.rdy_comp_ring_size);
-			IPADBG("rx_comp_ring_wp_pa=0x%pa\n",
-				&in->u.ul_smmu.rdy_comp_ring_wp_pa);
-			ipa3_ctx->uc_ctx.rdy_ring_rp_pa =
-				in->u.ul_smmu.rdy_ring_rp_pa;
-			ipa3_ctx->uc_ctx.rdy_ring_size =
-				in->u.ul_smmu.rdy_ring_size;
-			ipa3_ctx->uc_ctx.rdy_comp_ring_wp_pa =
-				in->u.ul_smmu.rdy_comp_ring_wp_pa;
-			ipa3_ctx->uc_ctx.rdy_comp_ring_size =
-				in->u.ul_smmu.rdy_comp_ring_size;
-		} else {
-			IPADBG("rx_ring_base_pa=0x%pa\n",
-				&in->u.ul.rdy_ring_base_pa);
-			IPADBG("rx_ring_size=%d\n",
-				in->u.ul.rdy_ring_size);
-			IPADBG("rx_ring_rp_pa=0x%pa\n",
-				&in->u.ul.rdy_ring_rp_pa);
-			IPADBG("rx_comp_ring_base_pa=0x%pa\n",
-				&in->u.ul.rdy_comp_ring_base_pa);
-			IPADBG("rx_comp_ring_size=%d\n",
-				in->u.ul.rdy_comp_ring_size);
-			IPADBG("rx_comp_ring_wp_pa=0x%pa\n",
-				&in->u.ul.rdy_comp_ring_wp_pa);
-			ipa3_ctx->uc_ctx.rdy_ring_base_pa =
-				in->u.ul.rdy_ring_base_pa;
-			ipa3_ctx->uc_ctx.rdy_ring_rp_pa =
-				in->u.ul.rdy_ring_rp_pa;
-			ipa3_ctx->uc_ctx.rdy_ring_size =
-				in->u.ul.rdy_ring_size;
-			ipa3_ctx->uc_ctx.rdy_comp_ring_base_pa =
-				in->u.ul.rdy_comp_ring_base_pa;
-			ipa3_ctx->uc_ctx.rdy_comp_ring_wp_pa =
-				in->u.ul.rdy_comp_ring_wp_pa;
-			ipa3_ctx->uc_ctx.rdy_comp_ring_size =
-				in->u.ul.rdy_comp_ring_size;
-		}
-	}
-
-	cmd.base = dma_alloc_coherent(ipa3_ctx->uc_pdev, cmd.size,
-			&cmd.phys_base, GFP_KERNEL);
-	if (cmd.base == NULL) {
-		IPAERR("fail to get DMA memory.\n");
-		result = -ENOMEM;
-		goto dma_alloc_fail;
-	}
-
-	if (IPA_CLIENT_IS_CONS(in->sys.client)) {
-		if (ipa3_ctx->ipa_wdi2) {
-			tx_2 = (struct IpaHwWdi2TxSetUpCmdData_t *)cmd.base;
-
-			len = in->smmu_enabled ? in->u.dl_smmu.comp_ring_size :
-				in->u.dl.comp_ring_size;
-			IPADBG("TX_2 ring smmu_en=%d ring_size=%d %d\n",
-				in->smmu_enabled,
-				in->u.dl_smmu.comp_ring_size,
-				in->u.dl.comp_ring_size);
-			if (ipa_create_uc_smmu_mapping(IPA_WDI_TX_RING_RES,
-					in->smmu_enabled,
-					in->u.dl.comp_ring_base_pa,
-					&in->u.dl_smmu.comp_ring,
-					len,
-					false,
-					&va)) {
-				IPAERR("fail to create uc mapping TX ring.\n");
-				result = -ENOMEM;
-				goto uc_timeout;
-			}
-			tx_2->comp_ring_base_pa_hi =
-				(u32) ((va & 0xFFFFFFFF00000000) >> 32);
-			tx_2->comp_ring_base_pa = (u32) (va & 0xFFFFFFFF);
-			tx_2->comp_ring_size = len;
-			IPADBG("TX_2 comp_ring_base_pa_hi=0x%08x :0x%08x\n",
-					tx_2->comp_ring_base_pa_hi,
-					tx_2->comp_ring_base_pa);
-
-			len = in->smmu_enabled ? in->u.dl_smmu.ce_ring_size :
-				in->u.dl.ce_ring_size;
-			IPADBG("TX_2 CE ring smmu_en=%d ring_size=%d %d\n",
-					in->smmu_enabled,
-					in->u.dl_smmu.ce_ring_size,
-					in->u.dl.ce_ring_size);
-			/* WA: wlan passed ce_ring sg_table PA directly */
-			if (ipa_create_uc_smmu_mapping(IPA_WDI_CE_RING_RES,
-						in->smmu_enabled,
-						in->u.dl.ce_ring_base_pa,
-						&in->u.dl_smmu.ce_ring,
-						len,
-						false,
-						&va)) {
-				IPAERR("fail to create uc mapping CE ring.\n");
-				result = -ENOMEM;
-				goto uc_timeout;
-			}
-			tx_2->ce_ring_base_pa_hi =
-				(u32) ((va & 0xFFFFFFFF00000000) >> 32);
-			tx_2->ce_ring_base_pa = (u32) (va & 0xFFFFFFFF);
-			tx_2->ce_ring_size = len;
-			IPADBG("TX_2 ce_ring_base_pa_hi=0x%08x :0x%08x\n",
-					tx_2->ce_ring_base_pa_hi,
-					tx_2->ce_ring_base_pa);
-
-			pa = in->smmu_enabled ? in->u.dl_smmu.ce_door_bell_pa :
-				in->u.dl.ce_door_bell_pa;
-			if (ipa_create_uc_smmu_mapping(IPA_WDI_CE_DB_RES,
-						in->smmu_enabled,
-						pa,
-						NULL,
-						4,
-						true,
-						&va)) {
-				IPAERR("fail to create uc mapping CE DB.\n");
-				result = -ENOMEM;
-				goto uc_timeout;
-			}
-			tx_2->ce_ring_doorbell_pa_hi =
-				(u32) ((va & 0xFFFFFFFF00000000) >> 32);
-			tx_2->ce_ring_doorbell_pa = (u32) (va & 0xFFFFFFFF);
-			IPADBG("TX_2 ce_ring_doorbell_pa_hi=0x%08x :0x%08x\n",
-					tx_2->ce_ring_doorbell_pa_hi,
-					tx_2->ce_ring_doorbell_pa);
-
-			tx_2->num_tx_buffers = in->smmu_enabled ?
-				in->u.dl_smmu.num_tx_buffers :
-				in->u.dl.num_tx_buffers;
-			tx_2->ipa_pipe_number = ipa_ep_idx;
-		} else {
-			tx = (struct IpaHwWdiTxSetUpCmdData_t *)cmd.base;
-
-			len = in->smmu_enabled ? in->u.dl_smmu.comp_ring_size :
-				in->u.dl.comp_ring_size;
-			IPADBG("TX ring smmu_en=%d ring_size=%d %d\n",
-					in->smmu_enabled,
-					in->u.dl_smmu.comp_ring_size,
-					in->u.dl.comp_ring_size);
-			if (ipa_create_uc_smmu_mapping(IPA_WDI_TX_RING_RES,
-						in->smmu_enabled,
-						in->u.dl.comp_ring_base_pa,
-						&in->u.dl_smmu.comp_ring,
-						len,
-						false,
-						&va)) {
-				IPAERR("fail to create uc mapping TX ring.\n");
-				result = -ENOMEM;
-				goto uc_timeout;
-			}
-			tx->comp_ring_base_pa = va;
-			tx->comp_ring_size = len;
-			len = in->smmu_enabled ? in->u.dl_smmu.ce_ring_size :
-				in->u.dl.ce_ring_size;
-			IPADBG("TX CE ring smmu_en=%d ring_size=%d %d 0x%lx\n",
-					in->smmu_enabled,
-					in->u.dl_smmu.ce_ring_size,
-					in->u.dl.ce_ring_size,
-					va);
-			if (ipa_create_uc_smmu_mapping(IPA_WDI_CE_RING_RES,
-						in->smmu_enabled,
-						in->u.dl.ce_ring_base_pa,
-						&in->u.dl_smmu.ce_ring,
-						len,
-						false,
-						&va)) {
-				IPAERR("fail to create uc mapping CE ring.\n");
-				result = -ENOMEM;
-				goto uc_timeout;
-			}
-			tx->ce_ring_base_pa = va;
-			tx->ce_ring_size = len;
-			pa = in->smmu_enabled ? in->u.dl_smmu.ce_door_bell_pa :
-				in->u.dl.ce_door_bell_pa;
-			if (ipa_create_uc_smmu_mapping(IPA_WDI_CE_DB_RES,
-						in->smmu_enabled,
-						pa,
-						NULL,
-						4,
-						true,
-						&va)) {
-				IPAERR("fail to create uc mapping CE DB.\n");
-				result = -ENOMEM;
-				goto uc_timeout;
-			}
-
-			IPADBG("CE doorbell pa: 0x%pa va:0x%lx\n", &pa, va);
-			IPADBG("Is wdi_over_pcie ? (%s)\n",
-				ipa3_ctx->wdi_over_pcie ? "Yes":"No");
-
-			if (ipa3_ctx->wdi_over_pcie)
-				tx->ce_ring_doorbell_pa = pa;
-			else
-				tx->ce_ring_doorbell_pa = va;
-
-			tx->num_tx_buffers = in->smmu_enabled ?
-				in->u.dl_smmu.num_tx_buffers :
-				in->u.dl.num_tx_buffers;
-			tx->ipa_pipe_number = ipa_ep_idx;
-		}
-		out->uc_door_bell_pa = ipa3_ctx->ipa_wrapper_base +
-				ipahal_get_reg_base() +
-				ipahal_get_reg_mn_ofst(IPA_UC_MAILBOX_m_n,
-				IPA_HW_WDI_TX_MBOX_START_INDEX/32,
-				IPA_HW_WDI_TX_MBOX_START_INDEX % 32);
-	} else {
-		if (ipa3_ctx->ipa_wdi2) {
-			rx_2 = (struct IpaHwWdi2RxSetUpCmdData_t *)cmd.base;
-
-			len = in->smmu_enabled ? in->u.ul_smmu.rdy_ring_size :
-				in->u.ul.rdy_ring_size;
-			IPADBG("RX_2 ring smmu_en=%d ring_size=%d %d\n",
-				in->smmu_enabled,
-				in->u.ul_smmu.rdy_ring_size,
-				in->u.ul.rdy_ring_size);
-			if (ipa_create_uc_smmu_mapping(IPA_WDI_RX_RING_RES,
-						in->smmu_enabled,
-						in->u.ul.rdy_ring_base_pa,
-						&in->u.ul_smmu.rdy_ring,
-						len,
-						false,
-						&va)) {
-				IPAERR("fail to create uc RX_2 ring.\n");
-				result = -ENOMEM;
-				goto uc_timeout;
-			}
-			rx_2->rx_ring_base_pa_hi =
-				(u32) ((va & 0xFFFFFFFF00000000) >> 32);
-			rx_2->rx_ring_base_pa = (u32) (va & 0xFFFFFFFF);
-			rx_2->rx_ring_size = len;
-			IPADBG("RX_2 rx_ring_base_pa_hi=0x%08x:0x%08x\n",
-					rx_2->rx_ring_base_pa_hi,
-					rx_2->rx_ring_base_pa);
-
-			pa = in->smmu_enabled ? in->u.ul_smmu.rdy_ring_rp_pa :
-				in->u.ul.rdy_ring_rp_pa;
-			if (ipa_create_uc_smmu_mapping(IPA_WDI_RX_RING_RP_RES,
-						in->smmu_enabled,
-						pa,
-						NULL,
-						4,
-						false,
-						&va)) {
-				IPAERR("fail to create uc RX_2 rng RP\n");
-				result = -ENOMEM;
-				goto uc_timeout;
-			}
-			rx_2->rx_ring_rp_pa_hi =
-				(u32) ((va & 0xFFFFFFFF00000000) >> 32);
-			rx_2->rx_ring_rp_pa = (u32) (va & 0xFFFFFFFF);
-			IPADBG("RX_2 rx_ring_rp_pa_hi=0x%08x :0x%08x\n",
-					rx_2->rx_ring_rp_pa_hi,
-					rx_2->rx_ring_rp_pa);
-			len = in->smmu_enabled ?
-				in->u.ul_smmu.rdy_comp_ring_size :
-				in->u.ul.rdy_comp_ring_size;
-			IPADBG("RX_2 ring smmu_en=%d comp_ring_size=%d %d\n",
-					in->smmu_enabled,
-					in->u.ul_smmu.rdy_comp_ring_size,
-					in->u.ul.rdy_comp_ring_size);
-			if (ipa_create_uc_smmu_mapping(IPA_WDI_RX_COMP_RING_RES,
-						in->smmu_enabled,
-						in->u.ul.rdy_comp_ring_base_pa,
-						&in->u.ul_smmu.rdy_comp_ring,
-						len,
-						false,
-						&va)) {
-				IPAERR("fail to create uc RX_2 comp_ring.\n");
-				result = -ENOMEM;
-				goto uc_timeout;
-			}
-			rx_2->rx_comp_ring_base_pa_hi =
-				(u32) ((va & 0xFFFFFFFF00000000) >> 32);
-			rx_2->rx_comp_ring_base_pa = (u32) (va & 0xFFFFFFFF);
-			rx_2->rx_comp_ring_size = len;
-			IPADBG("RX_2 rx_comp_ring_base_pa_hi=0x%08x:0x%08x\n",
-					rx_2->rx_comp_ring_base_pa_hi,
-					rx_2->rx_comp_ring_base_pa);
-
-			pa = in->smmu_enabled ?
-				in->u.ul_smmu.rdy_comp_ring_wp_pa :
-				in->u.ul.rdy_comp_ring_wp_pa;
-			if (ipa_create_uc_smmu_mapping(
-						IPA_WDI_RX_COMP_RING_WP_RES,
-						in->smmu_enabled,
-						pa,
-						NULL,
-						4,
-						false,
-						&va)) {
-				IPAERR("fail to create uc RX_2 comp_rng WP\n");
-				result = -ENOMEM;
-				goto uc_timeout;
-			}
-			rx_2->rx_comp_ring_wp_pa_hi =
-				(u32) ((va & 0xFFFFFFFF00000000) >> 32);
-			rx_2->rx_comp_ring_wp_pa = (u32) (va & 0xFFFFFFFF);
-			IPADBG("RX_2 rx_comp_ring_wp_pa_hi=0x%08x:0x%08x\n",
-					rx_2->rx_comp_ring_wp_pa_hi,
-					rx_2->rx_comp_ring_wp_pa);
-			rx_2->ipa_pipe_number = ipa_ep_idx;
-		} else {
-			rx = (struct IpaHwWdiRxSetUpCmdData_t *)cmd.base;
-
-			len = in->smmu_enabled ? in->u.ul_smmu.rdy_ring_size :
-				in->u.ul.rdy_ring_size;
-			IPADBG("RX ring smmu_en=%d ring_size=%d %d\n",
-					in->smmu_enabled,
-					in->u.ul_smmu.rdy_ring_size,
-					in->u.ul.rdy_ring_size);
-			if (ipa_create_uc_smmu_mapping(IPA_WDI_RX_RING_RES,
-						in->smmu_enabled,
-						in->u.ul.rdy_ring_base_pa,
-						&in->u.ul_smmu.rdy_ring,
-						len,
-						false,
-						&va)) {
-				IPAERR("fail to create uc mapping RX ring.\n");
-				result = -ENOMEM;
-				goto uc_timeout;
-			}
-			rx->rx_ring_base_pa = va;
-			rx->rx_ring_size = len;
-
-			pa = in->smmu_enabled ? in->u.ul_smmu.rdy_ring_rp_pa :
-				in->u.ul.rdy_ring_rp_pa;
-			if (ipa_create_uc_smmu_mapping(IPA_WDI_RX_RING_RP_RES,
-						in->smmu_enabled,
-						pa,
-						NULL,
-						4,
-						false,
-						&va)) {
-				IPAERR("fail to create uc mapping RX rng RP\n");
-				result = -ENOMEM;
-				goto uc_timeout;
-			}
-			rx->rx_ring_rp_pa = va;
-			rx->ipa_pipe_number = ipa_ep_idx;
-		}
-		out->uc_door_bell_pa = ipa3_ctx->ipa_wrapper_base +
-				ipahal_get_reg_base() +
-				ipahal_get_reg_mn_ofst(IPA_UC_MAILBOX_m_n,
-					IPA_HW_WDI_RX_MBOX_START_INDEX/32,
-					IPA_HW_WDI_RX_MBOX_START_INDEX % 32);
-	}
-
-	ep->valid = 1;
-	ep->client = in->sys.client;
-	ep->keep_ipa_awake = in->sys.keep_ipa_awake;
-	result = ipa3_disable_data_path(ipa_ep_idx);
-	if (result) {
-		IPAERR("disable data path failed res=%d clnt=%d.\n", result,
-			ipa_ep_idx);
-		goto uc_timeout;
-	}
-	if (IPA_CLIENT_IS_PROD(in->sys.client)) {
-		memset(&ep_cfg_ctrl, 0, sizeof(struct ipa_ep_cfg_ctrl));
-		ep_cfg_ctrl.ipa_ep_delay = true;
-		ipa3_cfg_ep_ctrl(ipa_ep_idx, &ep_cfg_ctrl);
-	}
-
-	result = ipa3_uc_send_cmd((u32)(cmd.phys_base),
-				IPA_CLIENT_IS_CONS(in->sys.client) ?
-				IPA_CPU_2_HW_CMD_WDI_TX_SET_UP :
-				IPA_CPU_2_HW_CMD_WDI_RX_SET_UP,
-				IPA_HW_2_CPU_WDI_CMD_STATUS_SUCCESS,
-				false, 10*HZ);
-
-	if (result) {
-		result = -EFAULT;
-		goto uc_timeout;
-	}
-
-	ep->skip_ep_cfg = in->sys.skip_ep_cfg;
-	ep->client_notify = in->sys.notify;
-	ep->priv = in->sys.priv;
-
-	/* for AP+STA stats update */
-	if (in->wdi_notify)
-		ipa3_ctx->uc_wdi_ctx.stats_notify = in->wdi_notify;
-	else
-		IPADBG("in->wdi_notify is null\n");
-
-	if (IPA_CLIENT_IS_CONS(in->sys.client)) {
-		in->sys.ipa_ep_cfg.aggr.aggr_en = IPA_ENABLE_AGGR;
-		in->sys.ipa_ep_cfg.aggr.aggr = IPA_GENERIC;
-		in->sys.ipa_ep_cfg.aggr.aggr_pkt_limit = IPA_AGGR_PKT_LIMIT;
-		in->sys.ipa_ep_cfg.aggr.aggr_byte_limit =
-						IPA_AGGR_HARD_BYTE_LIMIT;
-		in->sys.ipa_ep_cfg.aggr.aggr_hard_byte_limit_en =
-						IPA_ENABLE_AGGR;
-	}
-	if (!ep->skip_ep_cfg) {
-		if (ipa3_cfg_ep(ipa_ep_idx, &in->sys.ipa_ep_cfg)) {
-			IPAERR("fail to configure EP.\n");
-			goto ipa_cfg_ep_fail;
-		}
-		IPADBG("ep configuration successful\n");
-	} else {
-		IPADBG("Skipping endpoint configuration.\n");
-	}
-
-	ipa3_enable_data_path(ipa_ep_idx);
-
-	out->clnt_hdl = ipa_ep_idx;
-
-	if (!ep->skip_ep_cfg && IPA_CLIENT_IS_PROD(in->sys.client))
-		ipa3_install_dflt_flt_rules(ipa_ep_idx);
-
-	if (!ep->keep_ipa_awake)
-		IPA_ACTIVE_CLIENTS_DEC_EP(in->sys.client);
-
-	dma_free_coherent(ipa3_ctx->uc_pdev, cmd.size, cmd.base, cmd.phys_base);
-	ep->uc_offload_state |= IPA_WDI_CONNECTED;
-	IPADBG("client %d (ep: %d) connected\n", in->sys.client, ipa_ep_idx);
-
-	return 0;
-
-ipa_cfg_ep_fail:
-	memset(&ipa3_ctx->ep[ipa_ep_idx], 0, sizeof(struct ipa3_ep_context));
-uc_timeout:
-	ipa_release_uc_smmu_mappings(in->sys.client);
-	dma_free_coherent(ipa3_ctx->uc_pdev, cmd.size, cmd.base, cmd.phys_base);
-dma_alloc_fail:
-	IPA_ACTIVE_CLIENTS_DEC_EP(in->sys.client);
-fail:
-	return result;
-}
-
-int ipa3_disconnect_gsi_wdi_pipe(u32 clnt_hdl)
-{
-	int result = 0;
-	struct ipa3_ep_context *ep;
-
-	ep = &ipa3_ctx->ep[clnt_hdl];
-
-	if (ep->gsi_offload_state != IPA_WDI_CONNECTED) {
-		IPAERR("WDI channel bad state %d\n", ep->gsi_offload_state);
-		return -EFAULT;
-	}
-
-	if (!ep->keep_ipa_awake)
-		IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	ipa3_reset_gsi_channel(clnt_hdl);
-	ipa3_reset_gsi_event_ring(clnt_hdl);
-
-	if (!ep->keep_ipa_awake)
-		IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	result = ipa3_release_gsi_channel(clnt_hdl);
-	if (result) {
-		IPAERR("GSI dealloc channel failed %d\n",
-				result);
-		goto fail_dealloc_channel;
-	}
-	ipa_release_ap_smmu_mappings(clnt_hdl);
-
-	/* for AP+STA stats update */
-	if (ipa3_ctx->uc_wdi_ctx.stats_notify)
-		ipa3_ctx->uc_wdi_ctx.stats_notify = NULL;
-	else
-		IPADBG("uc_wdi_ctx.stats_notify already null\n");
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_5 &&
-		ipa3_ctx->ipa_hw_type != IPA_HW_v4_7)
-		ipa3_uc_debug_stats_dealloc(IPA_HW_PROTOCOL_WDI);
-	IPADBG("client (ep: %d) disconnected\n", clnt_hdl);
-
-fail_dealloc_channel:
-	return result;
-}
-
-/**
- * ipa3_disconnect_wdi_pipe() - WDI client disconnect
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_disconnect_wdi_pipe(u32 clnt_hdl)
-{
-	int result = 0;
-	struct ipa3_ep_context *ep;
-	union IpaHwWdiCommonChCmdData_t tear;
-
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-	    ipa3_ctx->ep[clnt_hdl].valid == 0) {
-		IPAERR("bad parm, %d\n", clnt_hdl);
-		return -EINVAL;
-	}
-
-	if (ipa3_ctx->ipa_wdi2_over_gsi)
-		return ipa3_disconnect_gsi_wdi_pipe(clnt_hdl);
-
-	result = ipa3_uc_state_check();
-	if (result)
-		return result;
-
-	IPADBG("ep=%d\n", clnt_hdl);
-
-	ep = &ipa3_ctx->ep[clnt_hdl];
-
-	if (ep->uc_offload_state != IPA_WDI_CONNECTED) {
-		IPAERR("WDI channel bad state %d\n", ep->uc_offload_state);
-		return -EFAULT;
-	}
-
-	if (!ep->keep_ipa_awake)
-		IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	tear.params.ipa_pipe_number = clnt_hdl;
-
-	result = ipa3_uc_send_cmd(tear.raw32b,
-				IPA_CPU_2_HW_CMD_WDI_TEAR_DOWN,
-				IPA_HW_2_CPU_WDI_CMD_STATUS_SUCCESS,
-				false, 10*HZ);
-
-	if (result) {
-		result = -EFAULT;
-		goto uc_timeout;
-	}
-
-	ipa3_delete_dflt_flt_rules(clnt_hdl);
-	ipa_release_uc_smmu_mappings(ep->client);
-
-	memset(&ipa3_ctx->ep[clnt_hdl], 0, sizeof(struct ipa3_ep_context));
-
-	IPADBG("client (ep: %d) disconnected\n", clnt_hdl);
-
-	/* for AP+STA stats update */
-	if (ipa3_ctx->uc_wdi_ctx.stats_notify)
-		ipa3_ctx->uc_wdi_ctx.stats_notify = NULL;
-	else
-		IPADBG("uc_wdi_ctx.stats_notify already null\n");
-
-uc_timeout:
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-	return result;
-}
-
-int ipa3_enable_gsi_wdi_pipe(u32 clnt_hdl)
-{
-	int result = 0;
-	struct ipa3_ep_context *ep;
-	struct ipa_ep_cfg_ctrl ep_cfg_ctrl;
-	int ipa_ep_idx;
-
-	IPADBG("ep=%d\n", clnt_hdl);
-
-	ep = &ipa3_ctx->ep[clnt_hdl];
-	if (ep->gsi_offload_state != IPA_WDI_CONNECTED) {
-		IPAERR("WDI channel bad state %d\n", ep->gsi_offload_state);
-		return -EFAULT;
-	}
-
-	ipa_ep_idx = ipa3_get_ep_mapping(ipa3_get_client_mapping(clnt_hdl));
-	if (ipa_ep_idx == -1) {
-		IPAERR("fail to alloc EP.\n");
-		return -EPERM;
-	}
-
-	IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	memset(&ep_cfg_ctrl, 0, sizeof(struct ipa_ep_cfg_ctrl));
-	ipa3_cfg_ep_ctrl(ipa_ep_idx, &ep_cfg_ctrl);
-
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-	ep->gsi_offload_state |= IPA_WDI_ENABLED;
-	IPADBG("client (ep: %d) enabled\n", clnt_hdl);
-
-	return result;
-}
-int ipa3_disable_gsi_wdi_pipe(u32 clnt_hdl)
-{
-	int result = 0;
-	struct ipa3_ep_context *ep;
-	struct ipa_ep_cfg_ctrl ep_cfg_ctrl;
-	u32 cons_hdl;
-
-	IPADBG("ep=%d\n", clnt_hdl);
-
-	ep = &ipa3_ctx->ep[clnt_hdl];
-
-	if (ep->gsi_offload_state != (IPA_WDI_CONNECTED | IPA_WDI_ENABLED)) {
-		IPAERR("WDI channel bad state %d\n", ep->gsi_offload_state);
-		return -EFAULT;
-	}
-	IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	result = ipa3_disable_data_path(clnt_hdl);
-	if (result) {
-		IPAERR("disable data path failed res=%d clnt=%d.\n", result,
-				clnt_hdl);
-		goto gsi_timeout;
-	}
-
-	/**
-	 * To avoid data stall during continuous SAP on/off before
-	 * setting delay to IPA Consumer pipe (Client Producer),
-	 * remove delay and enable holb on IPA Producer pipe
-	 */
-	if (IPA_CLIENT_IS_PROD(ep->client)) {
-		IPADBG("Stopping PROD channel - hdl=%d clnt=%d\n",
-			clnt_hdl, ep->client);
-		/* remove delay on wlan-prod pipe*/
-		memset(&ep_cfg_ctrl, 0, sizeof(struct ipa_ep_cfg_ctrl));
-		ipa3_cfg_ep_ctrl(clnt_hdl, &ep_cfg_ctrl);
-
-		cons_hdl = ipa3_get_ep_mapping(IPA_CLIENT_WLAN1_CONS);
-		if (cons_hdl == IPA_EP_NOT_ALLOCATED) {
-			IPAERR("Client %u is not mapped\n",
-				IPA_CLIENT_WLAN1_CONS);
-			goto gsi_timeout;
-		}
-		if (ipa3_ctx->ep[cons_hdl].valid == 1) {
-			result = ipa3_disable_data_path(cons_hdl);
-			if (result) {
-				IPAERR("disable data path failed\n");
-				IPAERR("res=%d clnt=%d\n",
-						result, cons_hdl);
-				goto gsi_timeout;
-			}
-		}
-		usleep_range(IPA_UC_POLL_SLEEP_USEC * IPA_UC_POLL_SLEEP_USEC,
-			IPA_UC_POLL_SLEEP_USEC * IPA_UC_POLL_SLEEP_USEC);
-
-	}
-
-	/* Set the delay after disabling IPA Producer pipe */
-	if (IPA_CLIENT_IS_PROD(ep->client)) {
-		memset(&ep_cfg_ctrl, 0, sizeof(struct ipa_ep_cfg_ctrl));
-		ep_cfg_ctrl.ipa_ep_delay = true;
-		ipa3_cfg_ep_ctrl(clnt_hdl, &ep_cfg_ctrl);
-	}
-	ep->gsi_offload_state &= ~IPA_WDI_ENABLED;
-	IPADBG("client (ep: %d) disabled\n", clnt_hdl);
-
-gsi_timeout:
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-	return result;
-}
-/**
- * ipa3_enable_wdi_pipe() - WDI client enable
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_enable_wdi_pipe(u32 clnt_hdl)
-{
-	int result = 0;
-	struct ipa3_ep_context *ep;
-	union IpaHwWdiCommonChCmdData_t enable;
-	struct ipa_ep_cfg_holb holb_cfg;
-
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-	    ipa3_ctx->ep[clnt_hdl].valid == 0) {
-		IPAERR("bad parm, %d\n", clnt_hdl);
-		return -EINVAL;
-	}
-
-	if (ipa3_ctx->ipa_wdi2_over_gsi)
-		return ipa3_enable_gsi_wdi_pipe(clnt_hdl);
-
-	result = ipa3_uc_state_check();
-	if (result)
-		return result;
-
-	IPADBG("ep=%d\n", clnt_hdl);
-
-	ep = &ipa3_ctx->ep[clnt_hdl];
-
-	if (ep->uc_offload_state != IPA_WDI_CONNECTED) {
-		IPAERR("WDI channel bad state %d\n", ep->uc_offload_state);
-		return -EFAULT;
-	}
-	IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-	enable.params.ipa_pipe_number = clnt_hdl;
-
-	result = ipa3_uc_send_cmd(enable.raw32b,
-		IPA_CPU_2_HW_CMD_WDI_CH_ENABLE,
-		IPA_HW_2_CPU_WDI_CMD_STATUS_SUCCESS,
-		false, 10*HZ);
-
-	if (result) {
-		result = -EFAULT;
-		goto uc_timeout;
-	}
-
-	if (IPA_CLIENT_IS_CONS(ep->client)) {
-		memset(&holb_cfg, 0, sizeof(holb_cfg));
-		holb_cfg.en = IPA_HOLB_TMR_DIS;
-		holb_cfg.tmr_val = 0;
-		result = ipa3_cfg_ep_holb(clnt_hdl, &holb_cfg);
-	}
-
-	ep->uc_offload_state |= IPA_WDI_ENABLED;
-	IPADBG("client (ep: %d) enabled\n", clnt_hdl);
-
-uc_timeout:
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-	return result;
-}
-
-/**
- * ipa3_disable_wdi_pipe() - WDI client disable
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_disable_wdi_pipe(u32 clnt_hdl)
-{
-	int result = 0;
-	struct ipa3_ep_context *ep;
-	union IpaHwWdiCommonChCmdData_t disable;
-	struct ipa_ep_cfg_ctrl ep_cfg_ctrl;
-	u32 cons_hdl;
-
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-	    ipa3_ctx->ep[clnt_hdl].valid == 0) {
-		IPAERR("bad parm, %d\n", clnt_hdl);
-		return -EINVAL;
-	}
-
-	if (ipa3_ctx->ipa_wdi2_over_gsi)
-		return ipa3_disable_gsi_wdi_pipe(clnt_hdl);
-
-	result = ipa3_uc_state_check();
-	if (result)
-		return result;
-
-	IPADBG("ep=%d\n", clnt_hdl);
-
-	ep = &ipa3_ctx->ep[clnt_hdl];
-
-	if (ep->uc_offload_state != (IPA_WDI_CONNECTED | IPA_WDI_ENABLED)) {
-		IPAERR("WDI channel bad state %d\n", ep->uc_offload_state);
-		return -EFAULT;
-	}
-	IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	result = ipa3_disable_data_path(clnt_hdl);
-	if (result) {
-		IPAERR("disable data path failed res=%d clnt=%d.\n", result,
-			clnt_hdl);
-		result = -EPERM;
-		goto uc_timeout;
-	}
-
-	/**
-	 * To avoid data stall during continuous SAP on/off before
-	 * setting delay to IPA Consumer pipe (Client Producer),
-	 * remove delay and enable holb on IPA Producer pipe
-	 */
-	if (IPA_CLIENT_IS_PROD(ep->client)) {
-		IPADBG("Stopping PROD channel - hdl=%d clnt=%d\n",
-			clnt_hdl, ep->client);
-		/* remove delay on wlan-prod pipe*/
-		memset(&ep_cfg_ctrl, 0, sizeof(struct ipa_ep_cfg_ctrl));
-		ipa3_cfg_ep_ctrl(clnt_hdl, &ep_cfg_ctrl);
-
-		cons_hdl = ipa3_get_ep_mapping(IPA_CLIENT_WLAN1_CONS);
-		if (cons_hdl == IPA_EP_NOT_ALLOCATED) {
-			IPAERR("Client %u is not mapped\n",
-				IPA_CLIENT_WLAN1_CONS);
-			goto uc_timeout;
-		}
-		if (ipa3_ctx->ep[cons_hdl].valid == 1) {
-			result = ipa3_disable_data_path(cons_hdl);
-			if (result) {
-				IPAERR("disable data path failed\n");
-				IPAERR("res=%d clnt=%d\n",
-					result, cons_hdl);
-				result = -EPERM;
-				goto uc_timeout;
-			}
-		}
-		usleep_range(IPA_UC_POLL_SLEEP_USEC * IPA_UC_POLL_SLEEP_USEC,
-			IPA_UC_POLL_SLEEP_USEC * IPA_UC_POLL_SLEEP_USEC);
-
-	}
-
-	disable.params.ipa_pipe_number = clnt_hdl;
-	result = ipa3_uc_send_cmd(disable.raw32b,
-		IPA_CPU_2_HW_CMD_WDI_CH_DISABLE,
-		IPA_HW_2_CPU_WDI_CMD_STATUS_SUCCESS,
-		false, 10*HZ);
-
-	if (result) {
-		result = -EFAULT;
-		goto uc_timeout;
-	}
-
-	/* Set the delay after disabling IPA Producer pipe */
-	if (IPA_CLIENT_IS_PROD(ep->client)) {
-		memset(&ep_cfg_ctrl, 0, sizeof(struct ipa_ep_cfg_ctrl));
-		ep_cfg_ctrl.ipa_ep_delay = true;
-		ipa3_cfg_ep_ctrl(clnt_hdl, &ep_cfg_ctrl);
-	}
-	ep->uc_offload_state &= ~IPA_WDI_ENABLED;
-	IPADBG("client (ep: %d) disabled\n", clnt_hdl);
-
-
-uc_timeout:
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-	return result;
-}
-
-int ipa3_resume_gsi_wdi_pipe(u32 clnt_hdl)
-{
-	int result = 0;
-	struct ipa3_ep_context *ep;
-	struct ipa_ep_cfg_ctrl ep_cfg_ctrl;
-	struct gsi_chan_info chan_info;
-	union __packed gsi_channel_scratch gsi_scratch;
-	struct IpaHwOffloadStatsAllocCmdData_t *pcmd_t = NULL;
-
-	IPADBG("ep=%d\n", clnt_hdl);
-	ep = &ipa3_ctx->ep[clnt_hdl];
-
-	if (ep->gsi_offload_state != (IPA_WDI_CONNECTED | IPA_WDI_ENABLED)) {
-		IPAERR("WDI channel bad state %d\n", ep->gsi_offload_state);
-		return -EFAULT;
-	}
-	IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	memset(&ep_cfg_ctrl, 0, sizeof(struct ipa_ep_cfg_ctrl));
-	result = ipa3_cfg_ep_ctrl(clnt_hdl, &ep_cfg_ctrl);
-	if (result)
-		IPAERR("client (ep: %d) fail un-susp/delay result=%d\n",
-				clnt_hdl, result);
-	else
-		IPADBG("client (ep: %d) un-susp/delay\n", clnt_hdl);
-
-	result =  gsi_start_channel(ep->gsi_chan_hdl);
-	if (result != GSI_STATUS_SUCCESS) {
-		IPAERR("gsi_start_channel failed %d\n", result);
-		ipa_assert();
-	}
-	pcmd_t = &ipa3_ctx->gsi_info[IPA_HW_PROTOCOL_WDI];
-	/* start uC gsi dbg stats monitor */
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_5 &&
-		ipa3_ctx->ipa_hw_type != IPA_HW_v4_7) {
-		if (IPA_CLIENT_IS_PROD(ep->client)) {
-			pcmd_t->ch_id_info[0].ch_id
-				= ep->gsi_chan_hdl;
-			pcmd_t->ch_id_info[0].dir
-				= DIR_PRODUCER;
-		} else {
-			pcmd_t->ch_id_info[1].ch_id
-				= ep->gsi_chan_hdl;
-			pcmd_t->ch_id_info[1].dir
-				= DIR_CONSUMER;
-		}
-		ipa3_uc_debug_stats_alloc(
-			ipa3_ctx->gsi_info[IPA_HW_PROTOCOL_WDI]);
-	}
-	gsi_query_channel_info(ep->gsi_chan_hdl, &chan_info);
-	gsi_read_channel_scratch(ep->gsi_chan_hdl, &gsi_scratch);
-	IPADBG("ch=%lu channel base = 0x%llx , event base 0x%llx\n",
-				ep->gsi_chan_hdl,
-				ep->gsi_mem_info.chan_ring_base_addr,
-				ep->gsi_mem_info.evt_ring_base_addr);
-	IPADBG("RP=0x%llx WP=0x%llx ev_valid=%d ERP=0x%llx EWP=0x%llx\n",
-			chan_info.rp, chan_info.wp, chan_info.evt_valid,
-			chan_info.evt_rp, chan_info.evt_wp);
-	IPADBG("Scratch 0 = %x Scratch 1 = %x Scratch 2 = %x Scratch 3 = %x\n",
-				gsi_scratch.data.word1, gsi_scratch.data.word2,
-				gsi_scratch.data.word3, gsi_scratch.data.word4);
-
-	ep->gsi_offload_state |= IPA_WDI_RESUMED;
-	IPADBG("exit\n");
-	return result;
-}
-
-/**
- * ipa3_resume_wdi_pipe() - WDI client resume
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_resume_wdi_pipe(u32 clnt_hdl)
-{
-	int result = 0;
-	struct ipa3_ep_context *ep;
-	union IpaHwWdiCommonChCmdData_t resume;
-	struct ipa_ep_cfg_ctrl ep_cfg_ctrl;
-
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-	    ipa3_ctx->ep[clnt_hdl].valid == 0) {
-		IPAERR("bad parm, %d\n", clnt_hdl);
-		return -EINVAL;
-	}
-
-	if (ipa3_ctx->ipa_wdi2_over_gsi)
-		return ipa3_resume_gsi_wdi_pipe(clnt_hdl);
-
-	result = ipa3_uc_state_check();
-	if (result)
-		return result;
-
-	IPADBG("ep=%d\n", clnt_hdl);
-
-	ep = &ipa3_ctx->ep[clnt_hdl];
-
-	if (ep->uc_offload_state != (IPA_WDI_CONNECTED | IPA_WDI_ENABLED)) {
-		IPAERR("WDI channel bad state %d\n", ep->uc_offload_state);
-		return -EFAULT;
-	}
-	IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-	resume.params.ipa_pipe_number = clnt_hdl;
-
-	result = ipa3_uc_send_cmd(resume.raw32b,
-		IPA_CPU_2_HW_CMD_WDI_CH_RESUME,
-		IPA_HW_2_CPU_WDI_CMD_STATUS_SUCCESS,
-		false, 10*HZ);
-
-	if (result) {
-		result = -EFAULT;
-		IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-		goto uc_timeout;
-	}
-
-	memset(&ep_cfg_ctrl, 0, sizeof(struct ipa_ep_cfg_ctrl));
-	result = ipa3_cfg_ep_ctrl(clnt_hdl, &ep_cfg_ctrl);
-	if (result)
-		IPAERR("client (ep: %d) fail un-susp/delay result=%d\n",
-				clnt_hdl, result);
-	else
-		IPADBG("client (ep: %d) un-susp/delay\n", clnt_hdl);
-
-	ep->uc_offload_state |= IPA_WDI_RESUMED;
-	IPADBG("client (ep: %d) resumed\n", clnt_hdl);
-
-uc_timeout:
-	return result;
-}
-
-int ipa3_suspend_gsi_wdi_pipe(u32 clnt_hdl)
-{
-	int ipa_ep_idx;
-	struct ipa3_ep_context *ep;
-	int res = 0;
-	u32 source_pipe_bitmask = 0;
-	bool disable_force_clear = false;
-	struct ipahal_ep_cfg_ctrl_scnd ep_ctrl_scnd = { 0 };
-	int retry_cnt = 0;
-	struct gsi_chan_info chan_info;
-	union __packed gsi_channel_scratch gsi_scratch;
-	struct IpaHwOffloadStatsAllocCmdData_t *pcmd_t = NULL;
-
-	ipa_ep_idx = ipa3_get_ep_mapping(ipa3_get_client_mapping(clnt_hdl));
-	if (ipa_ep_idx < 0) {
-		IPAERR("IPA client mapping failed\n");
-		return -EPERM;
-	}
-	ep = &ipa3_ctx->ep[ipa_ep_idx];
-
-	if (ep->gsi_offload_state != (IPA_WDI_CONNECTED | IPA_WDI_ENABLED |
-				IPA_WDI_RESUMED)) {
-		IPAERR("WDI channel bad state %d\n", ep->gsi_offload_state);
-		return -EFAULT;
-	}
-	if (ep->valid) {
-		IPADBG("suspended pipe %d\n", ipa_ep_idx);
-		source_pipe_bitmask = 1 <<
-			ipa3_get_ep_mapping(ep->client);
-		res = ipa3_enable_force_clear(clnt_hdl,
-				false, source_pipe_bitmask);
-		if (res) {
-			/*
-			 * assuming here modem SSR, AP can remove
-			 * the delay in this case
-			 */
-			IPAERR("failed to force clear %d\n", res);
-			IPAERR("remove delay from SCND reg\n");
-			ep_ctrl_scnd.endp_delay = false;
-			ipahal_write_reg_n_fields(
-					IPA_ENDP_INIT_CTRL_SCND_n, clnt_hdl,
-					&ep_ctrl_scnd);
-		} else {
-			disable_force_clear = true;
-		}
-retry_gsi_stop:
-		res = ipa3_stop_gsi_channel(ipa_ep_idx);
-		if (res != 0 && res != -GSI_STATUS_AGAIN &&
-				res != -GSI_STATUS_TIMED_OUT) {
-			IPAERR("failed to stop channel res = %d\n", res);
-			goto fail_stop_channel;
-		} else if (res == -GSI_STATUS_AGAIN) {
-			IPADBG("GSI stop channel failed retry cnt = %d\n",
-						retry_cnt);
-			retry_cnt++;
-			if (retry_cnt >= GSI_STOP_MAX_RETRY_CNT)
-				goto fail_stop_channel;
-			goto retry_gsi_stop;
-		} else {
-			IPADBG("GSI channel %ld STOP\n", ep->gsi_chan_hdl);
-		}
-		gsi_query_channel_info(ep->gsi_chan_hdl, &chan_info);
-		gsi_read_channel_scratch(ep->gsi_chan_hdl, &gsi_scratch);
-		IPADBG("ch=%lu channel base = 0x%llx , event base 0x%llx\n",
-				ep->gsi_chan_hdl,
-				ep->gsi_mem_info.chan_ring_base_addr,
-				ep->gsi_mem_info.evt_ring_base_addr);
-		IPADBG("RP=0x%llx WP=0x%llx ev_valid=%d ERP=0x%llx",
-				chan_info.rp, chan_info.wp,
-				chan_info.evt_valid, chan_info.evt_rp);
-		IPADBG("EWP=0x%llx\n", chan_info.evt_wp);
-		IPADBG("Scratch 0 = %x Scratch 1 = %x Scratch 2 = %x",
-				gsi_scratch.data.word1, gsi_scratch.data.word2,
-				gsi_scratch.data.word3);
-		IPADBG("Scratch 3 = %x\n", gsi_scratch.data.word4);
-	}
-	pcmd_t = &ipa3_ctx->gsi_info[IPA_HW_PROTOCOL_WDI];
-	/* stop uC gsi dbg stats monitor */
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_5 &&
-		ipa3_ctx->ipa_hw_type != IPA_HW_v4_7) {
-		if (IPA_CLIENT_IS_PROD(ep->client)) {
-			pcmd_t->ch_id_info[0].ch_id
-				= 0xff;
-			pcmd_t->ch_id_info[0].dir
-				= DIR_PRODUCER;
-		} else {
-			pcmd_t->ch_id_info[1].ch_id
-				= 0xff;
-			pcmd_t->ch_id_info[1].dir
-				= DIR_CONSUMER;
-		}
-		ipa3_uc_debug_stats_alloc(
-			ipa3_ctx->gsi_info[IPA_HW_PROTOCOL_WDI]);
-	}
-	if (disable_force_clear)
-		ipa3_disable_force_clear(clnt_hdl);
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-	ep->gsi_offload_state &= ~IPA_WDI_RESUMED;
-	return res;
-fail_stop_channel:
-	ipa_assert();
-	return res;
-}
-
-/**
- * ipa3_suspend_wdi_pipe() - WDI client suspend
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_suspend_wdi_pipe(u32 clnt_hdl)
-{
-	int result = 0;
-	struct ipa3_ep_context *ep;
-	union IpaHwWdiCommonChCmdData_t suspend;
-	struct ipa_ep_cfg_ctrl ep_cfg_ctrl;
-	u32 source_pipe_bitmask = 0;
-	bool disable_force_clear = false;
-	struct ipahal_ep_cfg_ctrl_scnd ep_ctrl_scnd = { 0 };
-
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-	    ipa3_ctx->ep[clnt_hdl].valid == 0) {
-		IPAERR("bad parm, %d\n", clnt_hdl);
-		return -EINVAL;
-	}
-
-	if (ipa3_ctx->ipa_wdi2_over_gsi)
-		return ipa3_suspend_gsi_wdi_pipe(clnt_hdl);
-
-	result = ipa3_uc_state_check();
-	if (result)
-		return result;
-
-	IPADBG("ep=%d\n", clnt_hdl);
-
-	ep = &ipa3_ctx->ep[clnt_hdl];
-
-	if (ep->uc_offload_state != (IPA_WDI_CONNECTED | IPA_WDI_ENABLED |
-				IPA_WDI_RESUMED)) {
-		IPAERR("WDI channel bad state %d\n", ep->uc_offload_state);
-		return -EFAULT;
-	}
-
-	suspend.params.ipa_pipe_number = clnt_hdl;
-
-	if (IPA_CLIENT_IS_PROD(ep->client)) {
-		/*
-		 * For WDI 2.0 need to ensure pipe will be empty before suspend
-		 * as IPA uC will fail to suspend the pipe otherwise.
-		 */
-		if (ipa3_ctx->ipa_wdi2) {
-			source_pipe_bitmask = 1 <<
-					ipa3_get_ep_mapping(ep->client);
-			result = ipa3_enable_force_clear(clnt_hdl,
-				false, source_pipe_bitmask);
-			if (result) {
-				/*
-				 * assuming here modem SSR, AP can remove
-				 * the delay in this case
-				 */
-				IPAERR("failed to force clear %d\n", result);
-				IPAERR("remove delay from SCND reg\n");
-				ep_ctrl_scnd.endp_delay = false;
-				ipahal_write_reg_n_fields(
-					IPA_ENDP_INIT_CTRL_SCND_n, clnt_hdl,
-					&ep_ctrl_scnd);
-			} else {
-				disable_force_clear = true;
-			}
-		}
-
-		IPADBG("Post suspend event first for IPA Producer\n");
-		IPADBG("Client: %d clnt_hdl: %d\n", ep->client, clnt_hdl);
-		result = ipa3_uc_send_cmd(suspend.raw32b,
-			IPA_CPU_2_HW_CMD_WDI_CH_SUSPEND,
-			IPA_HW_2_CPU_WDI_CMD_STATUS_SUCCESS,
-			false, 10*HZ);
-
-		if (result) {
-			result = -EFAULT;
-			goto uc_timeout;
-		}
-	}
-
-	memset(&ep_cfg_ctrl, 0, sizeof(struct ipa_ep_cfg_ctrl));
-	if (IPA_CLIENT_IS_CONS(ep->client)) {
-		if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_0) {
-			ep_cfg_ctrl.ipa_ep_suspend = true;
-			result = ipa3_cfg_ep_ctrl(clnt_hdl, &ep_cfg_ctrl);
-			if (result)
-				IPAERR("(ep: %d) failed to suspend result=%d\n",
-						clnt_hdl, result);
-			else
-				IPADBG("(ep: %d) suspended\n", clnt_hdl);
-		}
-	} else {
-		ep_cfg_ctrl.ipa_ep_delay = true;
-		result = ipa3_cfg_ep_ctrl(clnt_hdl, &ep_cfg_ctrl);
-		if (result)
-			IPAERR("client (ep: %d) failed to delay result=%d\n",
-					clnt_hdl, result);
-		else
-			IPADBG("client (ep: %d) delayed\n", clnt_hdl);
-	}
-
-	if (IPA_CLIENT_IS_CONS(ep->client)) {
-		result = ipa3_uc_send_cmd(suspend.raw32b,
-			IPA_CPU_2_HW_CMD_WDI_CH_SUSPEND,
-			IPA_HW_2_CPU_WDI_CMD_STATUS_SUCCESS,
-			false, 10*HZ);
-
-		if (result) {
-			result = -EFAULT;
-			goto uc_timeout;
-		}
-	}
-
-	if (disable_force_clear)
-		ipa3_disable_force_clear(clnt_hdl);
-
-	ipa3_ctx->tag_process_before_gating = true;
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-	ep->uc_offload_state &= ~IPA_WDI_RESUMED;
-	IPADBG("client (ep: %d) suspended\n", clnt_hdl);
-
-uc_timeout:
-	return result;
-}
-
-/**
- * ipa_broadcast_wdi_quota_reach_ind() - quota reach
- * @uint32_t fid: [in] input netdev ID
- * @uint64_t num_bytes: [in] used bytes
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa3_broadcast_wdi_quota_reach_ind(uint32_t fid,
-	uint64_t num_bytes)
-{
-	IPAERR("Quota reached indication on fid(%d) Mbytes(%lu)\n",
-			  fid, (unsigned long)num_bytes);
-	ipa3_broadcast_quota_reach_ind(0, IPA_UPSTEAM_WLAN);
-	return 0;
-}
-
-int ipa3_write_qmapid_gsi_wdi_pipe(u32 clnt_hdl, u8 qmap_id)
-{
-	int result = 0;
-	struct ipa3_ep_context *ep;
-	union __packed gsi_wdi_channel_scratch3_reg gsi_scratch3;
-	union __packed gsi_wdi2_channel_scratch2_reg gsi_scratch2;
-
-	ep = &ipa3_ctx->ep[clnt_hdl];
-	IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_7) {
-		memset(&gsi_scratch3, 0, sizeof(gsi_scratch3));
-		gsi_scratch3.wdi.qmap_id = qmap_id;
-		gsi_scratch3.wdi.endp_metadatareg_offset =
-			ipahal_get_reg_mn_ofst(
-				IPA_ENDP_INIT_HDR_METADATA_n, 0, clnt_hdl)/4;
-		result = gsi_write_channel_scratch3_reg(ep->gsi_chan_hdl,
-								gsi_scratch3);
-	} else {
-		memset(&gsi_scratch2, 0, sizeof(gsi_scratch2));
-		gsi_scratch2.wdi.qmap_id = qmap_id;
-		gsi_scratch2.wdi.endp_metadatareg_offset =
-			ipahal_get_reg_mn_ofst(
-				IPA_ENDP_INIT_HDR_METADATA_n, 0, clnt_hdl)/4;
-		result = gsi_write_channel_scratch2_reg(ep->gsi_chan_hdl,
-								gsi_scratch2);
-	}
-	if (result != GSI_STATUS_SUCCESS) {
-		IPAERR("gsi_write_channel_scratch failed %d\n",
-			result);
-		goto fail_write_channel_scratch;
-	}
-
-	IPADBG("client (ep: %d) qmap_id %d updated\n", clnt_hdl, qmap_id);
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-	return 0;
-fail_write_channel_scratch:
-	ipa_assert();
-	return result;
-}
-int ipa3_write_qmapid_wdi_pipe(u32 clnt_hdl, u8 qmap_id)
-{
-	int result = 0;
-	struct ipa3_ep_context *ep;
-	union IpaHwWdiRxExtCfgCmdData_t qmap;
-
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-	    ipa3_ctx->ep[clnt_hdl].valid == 0) {
-		IPAERR_RL("bad parm, %d\n", clnt_hdl);
-		return -EINVAL;
-	}
-	if (ipa3_ctx->ipa_wdi2_over_gsi)
-		return ipa3_write_qmapid_gsi_wdi_pipe(clnt_hdl, qmap_id);
-
-	result = ipa3_uc_state_check();
-	if (result)
-		return result;
-
-	IPADBG("ep=%d\n", clnt_hdl);
-
-	ep = &ipa3_ctx->ep[clnt_hdl];
-
-	if (!(ep->uc_offload_state & IPA_WDI_CONNECTED)) {
-		IPAERR_RL("WDI channel bad state %d\n", ep->uc_offload_state);
-		return -EFAULT;
-	}
-	IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-	qmap.params.ipa_pipe_number = clnt_hdl;
-	qmap.params.qmap_id = qmap_id;
-
-	result = ipa3_uc_send_cmd(qmap.raw32b,
-		IPA_CPU_2_HW_CMD_WDI_RX_EXT_CFG,
-		IPA_HW_2_CPU_WDI_CMD_STATUS_SUCCESS,
-		false, 10*HZ);
-
-	if (result) {
-		result = -EFAULT;
-		goto uc_timeout;
-	}
-
-	IPADBG("client (ep: %d) qmap_id %d updated\n", clnt_hdl, qmap_id);
-
-uc_timeout:
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-	return result;
-}
-
-/**
- * ipa3_uc_reg_rdyCB() - To register uC
- * ready CB if uC not ready
- * @inout:	[in/out] input/output parameters
- * from/to client
- *
- * Returns:	0 on success, negative on failure
- *
- */
-int ipa3_uc_reg_rdyCB(
-	struct ipa_wdi_uc_ready_params *inout)
-{
-	int result = 0;
-
-	if (inout == NULL) {
-		IPAERR("bad parm. inout=%pK ", inout);
-		return -EINVAL;
-	}
-
-	result = ipa3_uc_state_check();
-	if (result) {
-		inout->is_uC_ready = false;
-		ipa3_ctx->uc_wdi_ctx.uc_ready_cb = inout->notify;
-		ipa3_ctx->uc_wdi_ctx.priv = inout->priv;
-	} else {
-		inout->is_uC_ready = true;
-	}
-
-	return 0;
-}
-
-/**
- * ipa3_uc_dereg_rdyCB() - To de-register uC ready CB
- *
- * Returns:	0 on success, negative on failure
- *
- */
-int ipa3_uc_dereg_rdyCB(void)
-{
-	ipa3_ctx->uc_wdi_ctx.uc_ready_cb = NULL;
-	ipa3_ctx->uc_wdi_ctx.priv = NULL;
-
-	return 0;
-}
-
-
-/**
- * ipa3_uc_wdi_get_dbpa() - To retrieve
- * doorbell physical address of wlan pipes
- * @param:  [in/out] input/output parameters
- *          from/to client
- *
- * Returns:	0 on success, negative on failure
- *
- */
-int ipa3_uc_wdi_get_dbpa(
-	struct ipa_wdi_db_params *param)
-{
-	if (param == NULL || param->client >= IPA_CLIENT_MAX) {
-		IPAERR("bad parm. param=%pK ", param);
-		if (param)
-			IPAERR("client = %d\n", param->client);
-		return -EINVAL;
-	}
-
-	if (IPA_CLIENT_IS_CONS(param->client)) {
-		param->uc_door_bell_pa = ipa3_ctx->ipa_wrapper_base +
-				ipahal_get_reg_base() +
-				ipahal_get_reg_mn_ofst(IPA_UC_MAILBOX_m_n,
-					IPA_HW_WDI_TX_MBOX_START_INDEX/32,
-					IPA_HW_WDI_TX_MBOX_START_INDEX % 32);
-	} else {
-		param->uc_door_bell_pa = ipa3_ctx->ipa_wrapper_base +
-				ipahal_get_reg_base() +
-				ipahal_get_reg_mn_ofst(IPA_UC_MAILBOX_m_n,
-					IPA_HW_WDI_RX_MBOX_START_INDEX/32,
-					IPA_HW_WDI_RX_MBOX_START_INDEX % 32);
-	}
-
-	return 0;
-}
-
-static void ipa3_uc_wdi_loaded_handler(void)
-{
-	if (!ipa3_ctx) {
-		IPAERR("IPA ctx is null\n");
-		return;
-	}
-
-	if (ipa3_ctx->uc_wdi_ctx.uc_ready_cb) {
-		ipa3_ctx->uc_wdi_ctx.uc_ready_cb(
-			ipa3_ctx->uc_wdi_ctx.priv);
-
-		ipa3_ctx->uc_wdi_ctx.uc_ready_cb =
-			NULL;
-		ipa3_ctx->uc_wdi_ctx.priv = NULL;
-	}
-}
-
-int ipa3_create_wdi_mapping(u32 num_buffers, struct ipa_wdi_buffer_info *info)
-{
-	struct ipa_smmu_cb_ctx *cb = ipa3_get_smmu_ctx(IPA_SMMU_CB_WLAN);
-	int i;
-	int ret = 0;
-	int prot = IOMMU_READ | IOMMU_WRITE;
-
-	if (!info) {
-		IPAERR("info = %pK\n", info);
-		return -EINVAL;
-	}
-
-	if (!cb->valid) {
-		IPAERR("No SMMU CB setup\n");
-		return -EINVAL;
-	}
-
-	if (ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_WLAN]) {
-		IPAERR("IPA SMMU not enabled\n");
-		return -EINVAL;
-	}
-
-	for (i = 0; i < num_buffers; i++) {
-		IPADBG("i=%d pa=0x%pa iova=0x%lx sz=0x%zx\n", i,
-			&info[i].pa, info[i].iova, info[i].size);
-		info[i].result = ipa3_iommu_map(cb->iommu_domain,
-			rounddown(info[i].iova, PAGE_SIZE),
-			rounddown(info[i].pa, PAGE_SIZE),
-			roundup(info[i].size + info[i].pa -
-				rounddown(info[i].pa, PAGE_SIZE), PAGE_SIZE),
-			prot);
-	}
-
-	return ret;
-}
-
-int ipa3_release_wdi_mapping(u32 num_buffers, struct ipa_wdi_buffer_info *info)
-{
-	struct ipa_smmu_cb_ctx *cb = ipa3_get_smmu_ctx(IPA_SMMU_CB_WLAN);
-	int i;
-	int ret = 0;
-
-	if (!info) {
-		IPAERR("info = %pK\n", info);
-		return -EINVAL;
-	}
-
-	if (!cb->valid) {
-		IPAERR("No SMMU CB setup\n");
-		return -EINVAL;
-	}
-
-	for (i = 0; i < num_buffers; i++) {
-		IPADBG("i=%d pa=0x%pa iova=0x%lx sz=0x%zx\n", i,
-			&info[i].pa, info[i].iova, info[i].size);
-		info[i].result = iommu_unmap(cb->iommu_domain,
-			rounddown(info[i].iova, PAGE_SIZE),
-			roundup(info[i].size + info[i].pa -
-				rounddown(info[i].pa, PAGE_SIZE), PAGE_SIZE));
-	}
-
-	return ret;
-}

ipa/ipa_v3/ipa_utils.c

@@ -1,8428 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2012-2019, The Linux Foundation. All rights reserved.
- */
-
-#include <net/ip.h>
-#include <linux/genalloc.h>	/* gen_pool_alloc() */
-#include <linux/io.h>
-#include <linux/ratelimit.h>
-#include <linux/msm-bus.h>
-#include <linux/msm-bus-board.h>
-#include <linux/msm_gsi.h>
-#include <linux/elf.h>
-#include "ipa_i.h"
-#include "ipahal/ipahal.h"
-#include "ipahal/ipahal_fltrt.h"
-#include "ipahal/ipahal_hw_stats.h"
-#include "../ipa_rm_i.h"
-
-/*
- * The following for adding code (ie. for EMULATION) not found on x86.
- */
-#if defined(CONFIG_IPA_EMULATION)
-# include "ipa_emulation_stubs.h"
-#endif
-
-#define IPA_V3_0_CLK_RATE_SVS2 (37.5 * 1000 * 1000UL)
-#define IPA_V3_0_CLK_RATE_SVS (75 * 1000 * 1000UL)
-#define IPA_V3_0_CLK_RATE_NOMINAL (150 * 1000 * 1000UL)
-#define IPA_V3_0_CLK_RATE_TURBO (200 * 1000 * 1000UL)
-
-#define IPA_V3_5_CLK_RATE_SVS2 (100 * 1000 * 1000UL)
-#define IPA_V3_5_CLK_RATE_SVS (200 * 1000 * 1000UL)
-#define IPA_V3_5_CLK_RATE_NOMINAL (400 * 1000 * 1000UL)
-#define IPA_V3_5_CLK_RATE_TURBO (42640 * 10 * 1000UL)
-
-#define IPA_V4_0_CLK_RATE_SVS2 (60 * 1000 * 1000UL)
-#define IPA_V4_0_CLK_RATE_SVS (125 * 1000 * 1000UL)
-#define IPA_V4_0_CLK_RATE_NOMINAL (220 * 1000 * 1000UL)
-#define IPA_V4_0_CLK_RATE_TURBO (250 * 1000 * 1000UL)
-
-#define IPA_V3_0_MAX_HOLB_TMR_VAL (4294967296 - 1)
-
-#define IPA_V3_0_BW_THRESHOLD_TURBO_MBPS (1000)
-#define IPA_V3_0_BW_THRESHOLD_NOMINAL_MBPS (600)
-#define IPA_V3_0_BW_THRESHOLD_SVS_MBPS (310)
-
-#define IPA_ENDP_INIT_HDR_METADATA_n_MUX_ID_BMASK 0xFF0000
-#define IPA_ENDP_INIT_HDR_METADATA_n_MUX_ID_SHFT 0x10
-
-/* Max pipes + ICs for TAG process */
-#define IPA_TAG_MAX_DESC (IPA3_MAX_NUM_PIPES + 6)
-
-#define IPA_TAG_SLEEP_MIN_USEC (1000)
-#define IPA_TAG_SLEEP_MAX_USEC (2000)
-#define IPA_FORCE_CLOSE_TAG_PROCESS_TIMEOUT (10 * HZ)
-#define IPA_BCR_REG_VAL_v3_0 (0x00000001)
-#define IPA_BCR_REG_VAL_v3_5 (0x0000003B)
-#define IPA_BCR_REG_VAL_v4_0 (0x00000039)
-#define IPA_BCR_REG_VAL_v4_2 (0x00000000)
-#define IPA_AGGR_GRAN_MIN (1)
-#define IPA_AGGR_GRAN_MAX (32)
-#define IPA_EOT_COAL_GRAN_MIN (1)
-#define IPA_EOT_COAL_GRAN_MAX (16)
-
-#define IPA_FILT_ROUT_HASH_REG_VAL_v4_2 (0x00000000)
-#define IPA_DMA_TASK_FOR_GSI_TIMEOUT_MSEC (15)
-
-#define IPA_AGGR_BYTE_LIMIT (\
-		IPA_ENDP_INIT_AGGR_N_AGGR_BYTE_LIMIT_BMSK >> \
-		IPA_ENDP_INIT_AGGR_N_AGGR_BYTE_LIMIT_SHFT)
-#define IPA_AGGR_PKT_LIMIT (\
-		IPA_ENDP_INIT_AGGR_n_AGGR_PKT_LIMIT_BMSK >> \
-		IPA_ENDP_INIT_AGGR_n_AGGR_PKT_LIMIT_SHFT)
-
-/* In IPAv3 only endpoints 0-3 can be configured to deaggregation */
-#define IPA_EP_SUPPORTS_DEAGGR(idx) ((idx) >= 0 && (idx) <= 3)
-
-#define IPA_TAG_TIMER_TIMESTAMP_SHFT (14) /* ~0.8msec */
-#define IPA_NAT_TIMER_TIMESTAMP_SHFT (24) /* ~0.8sec */
-
-/*
- * Units of time per a specific granularity
- * The limitation based on H/W HOLB/AGGR time limit field width
- */
-#define IPA_TIMER_SCALED_TIME_LIMIT 31
-
-/* HPS, DPS sequencers Types*/
-
-/* DMA Only */
-#define IPA_DPS_HPS_SEQ_TYPE_DMA_ONLY  0x00000000
-/* DMA + decipher */
-#define IPA_DPS_HPS_SEQ_TYPE_DMA_DEC 0x00000011
-/* Packet Processing + no decipher + uCP (for Ethernet Bridging) */
-#define IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_NO_DEC_UCP 0x00000002
-/* Packet Processing + decipher + uCP */
-#define IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_DEC_UCP 0x00000013
-/* Packet Processing + no decipher + no uCP */
-#define IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_NO_DEC_NO_UCP 0x00000006
-/* Packet Processing + decipher + no uCP */
-#define IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_DEC_NO_UCP 0x00000017
-/* 2 Packet Processing pass + no decipher + uCP */
-#define IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP 0x00000004
-/* 2 Packet Processing pass + decipher + uCP */
-#define IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_DEC_UCP 0x00000015
-/* 2 Packet Processing pass + no decipher + uCP + HPS REP DMA Parser. */
-#define IPA_DPS_HPS_REP_SEQ_TYPE_2PKT_PROC_PASS_NO_DEC_UCP_DMAP 0x00000804
-/* Packet Processing + no decipher + no uCP + HPS REP DMA Parser.*/
-#define IPA_DPS_HPS_REP_SEQ_TYPE_PKT_PROCESS_NO_DEC_NO_UCP_DMAP 0x00000806
-/* COMP/DECOMP */
-#define IPA_DPS_HPS_SEQ_TYPE_DMA_COMP_DECOMP 0x00000020
-/* 2 Packet Processing + no decipher + 2 uCP */
-#define IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_2ND_UCP 0x0000000a
-/* 2 Packet Processing + decipher + 2 uCP */
-#define IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_DEC_2ND_UCP 0x0000001b
-/* 3 Packet Processing + no decipher + 2 uCP */
-#define IPA_DPS_HPS_SEQ_TYPE_3RD_PKT_PROCESS_PASS_NO_DEC_2ND_UCP 0x0000000c
-/* 3 Packet Processing + decipher + 2 uCP */
-#define IPA_DPS_HPS_SEQ_TYPE_3RD_PKT_PROCESS_PASS_DEC_2ND_UCP 0x0000001d
-/* 2 Packet Processing + no decipher + 2 uCP + HPS REP DMA Parser */
-#define IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_2ND_UCP_DMAP 0x0000080a
-/* 3 Packet Processing + no decipher + 2 uCP + HPS REP DMA Parser */
-#define IPA_DPS_HPS_SEQ_TYPE_3RD_PKT_PROCESS_PASS_NO_DEC_2ND_UCP_DMAP 0x0000080c
-/* Invalid sequencer type */
-#define IPA_DPS_HPS_SEQ_TYPE_INVALID 0xFFFFFFFF
-
-#define IPA_DPS_HPS_SEQ_TYPE_IS_DMA(seq_type) \
-	(seq_type == IPA_DPS_HPS_SEQ_TYPE_DMA_ONLY || \
-	seq_type == IPA_DPS_HPS_SEQ_TYPE_DMA_DEC || \
-	seq_type == IPA_DPS_HPS_SEQ_TYPE_DMA_COMP_DECOMP)
-
-
-/* Resource Group index*/
-#define IPA_v3_0_GROUP_UL		(0)
-#define IPA_v3_0_GROUP_DL		(1)
-#define IPA_v3_0_GROUP_DPL		IPA_v3_0_GROUP_DL
-#define IPA_v3_0_GROUP_DIAG		(2)
-#define IPA_v3_0_GROUP_DMA		(3)
-#define IPA_v3_0_GROUP_IMM_CMD		IPA_v3_0_GROUP_UL
-#define IPA_v3_0_GROUP_Q6ZIP		(4)
-#define IPA_v3_0_GROUP_Q6ZIP_GENERAL	IPA_v3_0_GROUP_Q6ZIP
-#define IPA_v3_0_GROUP_UC_RX_Q		(5)
-#define IPA_v3_0_GROUP_Q6ZIP_ENGINE	IPA_v3_0_GROUP_UC_RX_Q
-#define IPA_v3_0_GROUP_MAX		(6)
-
-#define IPA_v3_5_GROUP_LWA_DL		(0) /* currently not used */
-#define IPA_v3_5_MHI_GROUP_PCIE	IPA_v3_5_GROUP_LWA_DL
-#define IPA_v3_5_GROUP_UL_DL		(1)
-#define IPA_v3_5_MHI_GROUP_DDR		IPA_v3_5_GROUP_UL_DL
-#define IPA_v3_5_MHI_GROUP_DMA		(2)
-#define IPA_v3_5_GROUP_UC_RX_Q		(3) /* currently not used */
-#define IPA_v3_5_SRC_GROUP_MAX		(4)
-#define IPA_v3_5_DST_GROUP_MAX		(3)
-
-#define IPA_v4_0_GROUP_LWA_DL		(0)
-#define IPA_v4_0_MHI_GROUP_PCIE		(0)
-#define IPA_v4_0_ETHERNET		(0)
-#define IPA_v4_0_GROUP_UL_DL		(1)
-#define IPA_v4_0_MHI_GROUP_DDR		(1)
-#define IPA_v4_0_MHI_GROUP_DMA		(2)
-#define IPA_v4_0_GROUP_UC_RX_Q		(3)
-#define IPA_v4_0_SRC_GROUP_MAX		(4)
-#define IPA_v4_0_DST_GROUP_MAX		(4)
-
-#define IPA_v4_2_GROUP_UL_DL		(0)
-#define IPA_v4_2_SRC_GROUP_MAX		(1)
-#define IPA_v4_2_DST_GROUP_MAX		(1)
-
-#define IPA_v4_5_MHI_GROUP_PCIE		(0)
-#define IPA_v4_5_GROUP_UL_DL		(1)
-#define IPA_v4_5_MHI_GROUP_DDR		(1)
-#define IPA_v4_5_MHI_GROUP_DMA		(2)
-#define IPA_v4_5_MHI_GROUP_QDSS		(3)
-#define IPA_v4_5_GROUP_UC_RX_Q		(4)
-#define IPA_v4_5_SRC_GROUP_MAX		(5)
-#define IPA_v4_5_DST_GROUP_MAX		(5)
-
-#define IPA_v4_7_GROUP_UL_DL		(0)
-#define IPA_v4_7_SRC_GROUP_MAX		(1)
-#define IPA_v4_7_DST_GROUP_MAX		(1)
-
-#define IPA_GROUP_MAX IPA_v3_0_GROUP_MAX
-
-enum ipa_rsrc_grp_type_src {
-	IPA_v3_0_RSRC_GRP_TYPE_SRC_PKT_CONTEXTS,
-	IPA_v3_0_RSRC_GRP_TYPE_SRC_HDR_SECTORS,
-	IPA_v3_0_RSRC_GRP_TYPE_SRC_HDRI1_BUFFER,
-	IPA_v3_0_RSRC_GRP_TYPE_SRC_DESCRIPTOR_LISTS,
-	IPA_v3_0_RSRC_GRP_TYPE_SRC_DESCRIPTOR_BUFF,
-	IPA_v3_0_RSRC_GRP_TYPE_SRC_HDRI2_BUFFERS,
-	IPA_v3_0_RSRC_GRP_TYPE_SRC_HPS_DMARS,
-	IPA_v3_0_RSRC_GRP_TYPE_SRC_ACK_ENTRIES,
-	IPA_v3_0_RSRC_GRP_TYPE_SRC_MAX,
-
-	IPA_v3_5_RSRC_GRP_TYPE_SRC_PKT_CONTEXTS = 0,
-	IPA_v3_5_RSRC_GRP_TYPE_SRC_DESCRIPTOR_LISTS,
-	IPA_v3_5_RSRC_GRP_TYPE_SRC_DESCRIPTOR_BUFF,
-	IPA_v3_5_RSRC_GRP_TYPE_SRC_HPS_DMARS,
-	IPA_v3_5_RSRC_GRP_TYPE_SRC_ACK_ENTRIES,
-	IPA_v3_5_RSRC_GRP_TYPE_SRC_MAX,
-
-	IPA_v4_0_RSRC_GRP_TYPE_SRC_PKT_CONTEXTS = 0,
-	IPA_v4_0_RSRC_GRP_TYPE_SRC_DESCRIPTOR_LISTS,
-	IPA_v4_0_RSRC_GRP_TYPE_SRC_DESCRIPTOR_BUFF,
-	IPA_v4_0_RSRC_GRP_TYPE_SRC_HPS_DMARS,
-	IPA_v4_0_RSRC_GRP_TYPE_SRC_ACK_ENTRIES,
-	IPA_v4_0_RSRC_GRP_TYPE_SRC_MAX
-};
-
-#define IPA_RSRC_GRP_TYPE_SRC_MAX IPA_v3_0_RSRC_GRP_TYPE_SRC_MAX
-
-enum ipa_rsrc_grp_type_dst {
-	IPA_v3_0_RSRC_GRP_TYPE_DST_DATA_SECTORS,
-	IPA_v3_0_RSRC_GRP_TYPE_DST_DATA_SECTOR_LISTS,
-	IPA_v3_0_RSRC_GRP_TYPE_DST_DPS_DMARS,
-	IPA_v3_0_RSRC_GRP_TYPE_DST_MAX,
-
-	IPA_v3_5_RSRC_GRP_TYPE_DST_DATA_SECTORS = 0,
-	IPA_v3_5_RSRC_GRP_TYPE_DST_DPS_DMARS,
-	IPA_v3_5_RSRC_GRP_TYPE_DST_MAX,
-
-	IPA_v4_0_RSRC_GRP_TYPE_DST_DATA_SECTORS = 0,
-	IPA_v4_0_RSRC_GRP_TYPE_DST_DPS_DMARS,
-	IPA_v4_0_RSRC_GRP_TYPE_DST_MAX,
-};
-#define IPA_RSRC_GRP_TYPE_DST_MAX IPA_v3_0_RSRC_GRP_TYPE_DST_MAX
-
-enum ipa_rsrc_grp_type_rx {
-	IPA_RSRC_GRP_TYPE_RX_HPS_CMDQ,
-	IPA_RSRC_GRP_TYPE_RX_MAX
-};
-
-enum ipa_rsrc_grp_rx_hps_weight_config {
-	IPA_RSRC_GRP_TYPE_RX_HPS_WEIGHT_CONFIG,
-	IPA_RSRC_GRP_TYPE_RX_HPS_WEIGHT_MAX
-};
-
-struct rsrc_min_max {
-	u32 min;
-	u32 max;
-};
-
-enum ipa_ver {
-	IPA_3_0,
-	IPA_3_5,
-	IPA_3_5_MHI,
-	IPA_3_5_1,
-	IPA_4_0,
-	IPA_4_0_MHI,
-	IPA_4_1,
-	IPA_4_1_APQ,
-	IPA_4_2,
-	IPA_4_5,
-	IPA_4_5_MHI,
-	IPA_4_5_APQ,
-	IPA_4_7,
-	IPA_VER_MAX,
-};
-
-
-static const struct rsrc_min_max ipa3_rsrc_src_grp_config
-	[IPA_VER_MAX][IPA_RSRC_GRP_TYPE_SRC_MAX][IPA_GROUP_MAX] = {
-	[IPA_3_0] = {
-		/* UL	DL	DIAG	DMA	Not Used	uC Rx */
-		[IPA_v3_0_RSRC_GRP_TYPE_SRC_PKT_CONTEXTS] = {
-		{3, 255}, {3, 255}, {1, 255}, {1, 255}, {1, 255}, {2, 255} },
-		[IPA_v3_0_RSRC_GRP_TYPE_SRC_HDR_SECTORS] = {
-		{0, 255}, {0, 255}, {0, 255}, {0, 255}, {0, 255}, {0, 255} },
-		[IPA_v3_0_RSRC_GRP_TYPE_SRC_HDRI1_BUFFER] = {
-		{0, 255}, {0, 255}, {0, 255}, {0, 255}, {0, 255}, {0, 255} },
-		[IPA_v3_0_RSRC_GRP_TYPE_SRC_DESCRIPTOR_LISTS] = {
-		{14, 14}, {16, 16}, {5, 5}, {5, 5},  {0, 0}, {8, 8} },
-		[IPA_v3_0_RSRC_GRP_TYPE_SRC_DESCRIPTOR_BUFF] = {
-		{19, 19}, {26, 26}, {3, 3}, {7, 7}, {0, 0}, {8, 8} },
-		[IPA_v3_0_RSRC_GRP_TYPE_SRC_HDRI2_BUFFERS] = {
-		{0, 255}, {0, 255}, {0, 255}, {0, 255}, {0, 255}, {0, 255} },
-		[IPA_v3_0_RSRC_GRP_TYPE_SRC_HPS_DMARS] = {
-		{0, 255}, {0, 255}, {0, 255}, {0, 255}, {0, 255}, {0, 255} },
-		[IPA_v3_0_RSRC_GRP_TYPE_SRC_ACK_ENTRIES] = {
-		{14, 14}, {16, 16}, {5, 5}, {5, 5}, {0, 0}, {8, 8} },
-	},
-	[IPA_3_5] = {
-		/* LWA_DL  UL_DL    unused  UC_RX_Q, other are invalid */
-		[IPA_v3_5_RSRC_GRP_TYPE_SRC_PKT_CONTEXTS] = {
-		{0, 0}, {1, 255}, {0, 0}, {1, 255}, {0, 0}, {0, 0} },
-		[IPA_v3_5_RSRC_GRP_TYPE_SRC_DESCRIPTOR_LISTS] = {
-		{0, 0}, {10, 10}, {0, 0}, {8, 8}, {0, 0}, {0, 0} },
-		[IPA_v3_5_RSRC_GRP_TYPE_SRC_DESCRIPTOR_BUFF] = {
-		{0, 0}, {14, 14}, {0, 0}, {8, 8}, {0, 0}, {0, 0} },
-		[IPA_v3_5_RSRC_GRP_TYPE_SRC_HPS_DMARS] = {
-		{0, 255}, {0, 255}, {0, 255}, {0, 255},  {0, 0}, {0, 0} },
-		[IPA_v3_5_RSRC_GRP_TYPE_SRC_ACK_ENTRIES] = {
-		{0, 0}, {20, 20}, {0, 0}, {14, 14}, {0, 0}, {0, 0} },
-	},
-	[IPA_3_5_MHI] = {
-		/* PCIE  DDR     DMA  unused, other are invalid */
-		[IPA_v3_5_RSRC_GRP_TYPE_SRC_PKT_CONTEXTS] = {
-		{4, 4}, {5, 5}, {1, 1}, {0, 0}, {0, 0}, {0, 0} },
-		[IPA_v3_5_RSRC_GRP_TYPE_SRC_DESCRIPTOR_LISTS] = {
-		{10, 10}, {10, 10}, {8, 8}, {0, 0}, {0, 0}, {0, 0} },
-		[IPA_v3_5_RSRC_GRP_TYPE_SRC_DESCRIPTOR_BUFF] = {
-		{12, 12}, {12, 12}, {8, 8}, {0, 0}, {0, 0}, {0, 0} },
-		[IPA_v3_5_RSRC_GRP_TYPE_SRC_HPS_DMARS] = {
-		{0, 255}, {0, 255}, {0, 255}, {0, 255},  {0, 0}, {0, 0} },
-		[IPA_v3_5_RSRC_GRP_TYPE_SRC_ACK_ENTRIES] = {
-		{14, 14}, {14, 14}, {14, 14}, {0, 0}, {0, 0}, {0, 0} },
-	},
-	[IPA_3_5_1] = {
-		/* LWA_DL  UL_DL    unused  UC_RX_Q, other are invalid */
-		[IPA_v3_5_RSRC_GRP_TYPE_SRC_PKT_CONTEXTS] = {
-		{1, 255}, {1, 255}, {0, 0}, {1, 255}, {0, 0}, {0, 0} },
-		[IPA_v3_5_RSRC_GRP_TYPE_SRC_DESCRIPTOR_LISTS] = {
-		{10, 10}, {10, 10}, {0, 0}, {8, 8}, {0, 0}, {0, 0} },
-		[IPA_v3_5_RSRC_GRP_TYPE_SRC_DESCRIPTOR_BUFF] = {
-		{12, 12}, {14, 14}, {0, 0}, {8, 8}, {0, 0}, {0, 0} },
-		[IPA_v3_5_RSRC_GRP_TYPE_SRC_HPS_DMARS] = {
-		{0, 255}, {0, 255}, {0, 255}, {0, 255},  {0, 0}, {0, 0} },
-		[IPA_v3_5_RSRC_GRP_TYPE_SRC_ACK_ENTRIES] = {
-		{14, 14}, {20, 20}, {0, 0}, {14, 14}, {0, 0}, {0, 0} },
-	},
-	[IPA_4_0] = {
-		/* LWA_DL  UL_DL    unused  UC_RX_Q, other are invalid */
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_PKT_CONTEXTS] = {
-		{1, 255}, {1, 255}, {0, 0}, {1, 255}, {0, 0}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_DESCRIPTOR_LISTS] = {
-		{10, 10}, {10, 10}, {0, 0}, {8, 8}, {0, 0}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_DESCRIPTOR_BUFF] = {
-		{12, 12}, {14, 14}, {0, 0}, {8, 8}, {0, 0}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_HPS_DMARS] = {
-		{0, 255}, {0, 255}, {0, 255}, {0, 255},  {0, 0}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_ACK_ENTRIES] = {
-		{14, 14}, {20, 20}, {0, 0}, {14, 14}, {0, 0}, {0, 0} },
-	},
-	[IPA_4_0_MHI] = {
-		/* PCIE  DDR     DMA  unused, other are invalid */
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_PKT_CONTEXTS] = {
-		{4, 4}, {5, 5}, {1, 1}, {0, 0}, {0, 0}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_DESCRIPTOR_LISTS] = {
-		{10, 10}, {10, 10}, {8, 8}, {0, 0}, {0, 0}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_DESCRIPTOR_BUFF] = {
-		{12, 12}, {12, 12}, {8, 8}, {0, 0}, {0, 0}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_HPS_DMARS] = {
-		{0, 255}, {0, 255}, {0, 255}, {0, 255},  {0, 0}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_ACK_ENTRIES] = {
-		{14, 14}, {14, 14}, {14, 14}, {0, 0}, {0, 0}, {0, 0} },
-	},
-	[IPA_4_1] = {
-		/* LWA_DL  UL_DL    unused  UC_RX_Q, other are invalid */
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_PKT_CONTEXTS] = {
-		{1, 63}, {1, 63}, {0, 0}, {1, 63}, {0, 0}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_DESCRIPTOR_LISTS] = {
-		{10, 10}, {10, 10}, {0, 0}, {8, 8}, {0, 0}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_DESCRIPTOR_BUFF] = {
-		{12, 12}, {14, 14}, {0, 0}, {8, 8}, {0, 0}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_HPS_DMARS] = {
-		{0, 63}, {0, 63}, {0, 63}, {0, 63},  {0, 0}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_ACK_ENTRIES] = {
-		{14, 14}, {20, 20}, {0, 0}, {14, 14}, {0, 0}, {0, 0} },
-	},
-	[IPA_4_2] = {
-		/* UL_DL   other are invalid */
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_PKT_CONTEXTS] = {
-		{3, 63}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_DESCRIPTOR_LISTS] = {
-		{3, 3}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_DESCRIPTOR_BUFF] = {
-		{10, 10}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_HPS_DMARS] = {
-		{1, 1}, {0, 0}, {0, 0},  {0, 0}, {0, 0}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_ACK_ENTRIES] = {
-		{5, 5}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0} },
-	},
-	[IPA_4_5] = {
-		/* unused  UL_DL  unused  unused  UC_RX_Q N/A */
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_PKT_CONTEXTS] = {
-		{0, 0}, {1, 11}, {0, 0}, {0, 0}, {1, 63}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_DESCRIPTOR_LISTS] = {
-		{0, 0}, {14, 14}, {0, 0}, {0, 0}, {3, 3}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_DESCRIPTOR_BUFF] = {
-		{0, 0}, {18, 18}, {0, 0}, {0, 0}, {8, 8}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_HPS_DMARS] = {
-		{0, 63}, {0, 63}, {0, 63}, {0, 63},  {0, 63}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_ACK_ENTRIES] = {
-		{0, 0}, {24, 24}, {0, 0}, {0, 0}, {8, 8}, {0, 0} },
-	},
-	[IPA_4_5_MHI] = {
-		/* PCIE  DDR  DMA  QDSS  unused  N/A */
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_PKT_CONTEXTS] = {
-		{3, 8}, {4, 11}, {1, 1}, {1, 1}, {0, 0}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_DESCRIPTOR_LISTS] = {
-		{9, 9}, {12, 12}, {2, 2}, {2, 2}, {0, 0}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_DESCRIPTOR_BUFF] = {
-		{9, 9}, {14, 14}, {4, 4}, {4, 4}, {0, 0}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_HPS_DMARS] = {
-		{0, 63}, {0, 63}, {0, 63}, {0, 63},  {0, 63}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_ACK_ENTRIES] = {
-		{22, 22}, {16, 16}, {6, 6}, {2, 2}, {0, 0}, {0, 0} },
-	},
-	[IPA_4_5_APQ] = {
-		/* unused  UL_DL  unused  unused  UC_RX_Q N/A */
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_PKT_CONTEXTS] = {
-		{0, 0}, {1, 11}, {0, 0}, {0, 0}, {1, 63}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_DESCRIPTOR_LISTS] = {
-		{0, 0}, {14, 14}, {0, 0}, {0, 0}, {3, 3}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_DESCRIPTOR_BUFF] = {
-		{0, 0}, {18, 18}, {0, 0}, {0, 0}, {8, 8}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_HPS_DMARS] = {
-		{0, 63}, {0, 63}, {0, 63}, {0, 63},  {0, 63}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_ACK_ENTRIES] = {
-		{0, 0}, {24, 24}, {0, 0}, {0, 0}, {8, 8}, {0, 0} },
-	},
-	[IPA_4_7] = {
-		/* UL_DL   other are invalid */
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_PKT_CONTEXTS] = {
-		{8, 8}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_DESCRIPTOR_LISTS] = {
-		{8, 8}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_DESCRIPTOR_BUFF] = {
-		{18, 18}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_HPS_DMARS] = {
-		{2, 2}, {0, 0}, {0, 0},  {0, 0}, {0, 0}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_SRC_ACK_ENTRIES] = {
-		{15, 15}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0} },
-	},
-
-};
-
-static const struct rsrc_min_max ipa3_rsrc_dst_grp_config
-	[IPA_VER_MAX][IPA_RSRC_GRP_TYPE_DST_MAX][IPA_GROUP_MAX] = {
-	[IPA_3_0] = {
-		/* UL	DL/DPL	DIAG	DMA  Q6zip_gen Q6zip_eng */
-		[IPA_v3_0_RSRC_GRP_TYPE_DST_DATA_SECTORS] = {
-		{2, 2}, {3, 3}, {0, 0}, {2, 2}, {3, 3}, {3, 3} },
-		[IPA_v3_0_RSRC_GRP_TYPE_DST_DATA_SECTOR_LISTS] = {
-		{0, 255}, {0, 255}, {0, 255}, {0, 255}, {0, 255}, {0, 255} },
-		[IPA_v3_0_RSRC_GRP_TYPE_DST_DPS_DMARS] = {
-		{1, 1}, {1, 1}, {1, 1}, {1, 1}, {1, 1}, {0, 0} },
-	},
-	[IPA_3_5] = {
-		/* unused UL/DL/DPL unused N/A    N/A     N/A */
-		[IPA_v3_5_RSRC_GRP_TYPE_DST_DATA_SECTORS] = {
-		{4, 4}, {4, 4}, {3, 3}, {0, 0}, {0, 0}, {0, 0} },
-		[IPA_v3_5_RSRC_GRP_TYPE_DST_DPS_DMARS] = {
-		{2, 255}, {1, 255}, {1, 2}, {0, 0}, {0, 0}, {0, 0} },
-	},
-	[IPA_3_5_MHI] = {
-		/* PCIE  DDR     DMA     N/A     N/A     N/A */
-		[IPA_v3_5_RSRC_GRP_TYPE_DST_DATA_SECTORS] = {
-		{4, 4}, {4, 4}, {3, 3}, {0, 0}, {0, 0}, {0, 0} },
-		[IPA_v3_5_RSRC_GRP_TYPE_DST_DPS_DMARS] = {
-		{2, 255}, {1, 255}, {1, 2}, {0, 0}, {0, 0}, {0, 0} },
-	},
-	[IPA_3_5_1] = {
-		/* LWA_DL UL/DL/DPL unused N/A   N/A     N/A */
-		[IPA_v3_5_RSRC_GRP_TYPE_DST_DATA_SECTORS] = {
-		{4, 4}, {4, 4}, {3, 3}, {0, 0}, {0, 0}, {0, 0} },
-		[IPA_v3_5_RSRC_GRP_TYPE_DST_DPS_DMARS] = {
-		{2, 255}, {1, 255}, {1, 2}, {0, 0}, {0, 0}, {0, 0} },
-	},
-	[IPA_4_0] = {
-		/* LWA_DL UL/DL/DPL uC, other are invalid */
-		[IPA_v4_0_RSRC_GRP_TYPE_DST_DATA_SECTORS] = {
-		{4, 4}, {4, 4}, {3, 3}, {2, 2}, {0, 0}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_DST_DPS_DMARS] = {
-		{2, 255}, {1, 255}, {1, 2}, {0, 2}, {0, 0}, {0, 0} },
-	},
-	[IPA_4_0_MHI] = {
-		/* LWA_DL UL/DL/DPL uC, other are invalid */
-		[IPA_v4_0_RSRC_GRP_TYPE_DST_DATA_SECTORS] = {
-		{4, 4}, {4, 4}, {3, 3}, {2, 2}, {0, 0}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_DST_DPS_DMARS] = {
-		{2, 255}, {1, 255}, {1, 2}, {0, 2}, {0, 0}, {0, 0} },
-	},
-	[IPA_4_1] = {
-		/* LWA_DL UL/DL/DPL uC, other are invalid */
-		[IPA_v4_0_RSRC_GRP_TYPE_DST_DATA_SECTORS] = {
-		{4, 4}, {4, 4}, {3, 3}, {2, 2}, {0, 0}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_DST_DPS_DMARS] = {
-		{2, 63}, {1, 63}, {1, 2}, {0, 2}, {0, 0}, {0, 0} },
-	},
-	[IPA_4_2] = {
-		/* UL/DL/DPL, other are invalid */
-		[IPA_v4_0_RSRC_GRP_TYPE_DST_DATA_SECTORS] = {
-		{3, 3}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_DST_DPS_DMARS] = {
-		{1, 63}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0} },
-	},
-	[IPA_4_5] = {
-		/* unused  UL/DL/DPL unused  unused  uC  N/A */
-		[IPA_v4_0_RSRC_GRP_TYPE_DST_DATA_SECTORS] = {
-		{0, 0}, {16, 16}, {2, 2}, {2, 2}, {0, 0}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_DST_DPS_DMARS] = {
-		{0, 0}, {2, 63}, {1, 2}, {1, 2}, {0, 2}, {0, 0} },
-	},
-	[IPA_4_5_MHI] = {
-		/* PCIE/DPL  DDR  DMA/CV2X  QDSS  uC  N/A */
-		[IPA_v4_0_RSRC_GRP_TYPE_DST_DATA_SECTORS] = {
-		{16, 16}, {5, 5}, {2, 2}, {2, 2}, {0, 0}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_DST_DPS_DMARS] = {
-		{2, 63}, {1, 63}, {1, 2}, {1, 2}, {0, 2}, {0, 0} },
-	},
-	[IPA_4_5_APQ] = {
-		/* unused  UL/DL/DPL unused  unused  uC  N/A */
-		[IPA_v4_0_RSRC_GRP_TYPE_DST_DATA_SECTORS] = {
-		{0, 0}, {16, 16}, {2, 2}, {2, 2}, {0, 0}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_DST_DPS_DMARS] = {
-		{0, 0}, {2, 63}, {1, 2}, {1, 2}, {0, 2}, {0, 0} },
-	},
-	[IPA_4_7] = {
-		/* UL/DL/DPL, other are invalid */
-		[IPA_v4_0_RSRC_GRP_TYPE_DST_DATA_SECTORS] = {
-		{7, 7}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0} },
-		[IPA_v4_0_RSRC_GRP_TYPE_DST_DPS_DMARS] = {
-		{2, 2}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0} },
-	},
-
-};
-
-static const struct rsrc_min_max ipa3_rsrc_rx_grp_config
-	[IPA_VER_MAX][IPA_RSRC_GRP_TYPE_RX_MAX][IPA_GROUP_MAX] = {
-	[IPA_3_0] = {
-		/* UL	DL	DIAG	DMA	unused	uC Rx */
-		[IPA_RSRC_GRP_TYPE_RX_HPS_CMDQ] = {
-		{16, 16}, {24, 24}, {8, 8}, {8, 8}, {0, 0}, {8, 8} },
-	},
-	[IPA_3_5] = {
-		/* unused UL_DL	unused UC_RX_Q   N/A     N/A */
-		[IPA_RSRC_GRP_TYPE_RX_HPS_CMDQ] = {
-		{0, 0}, {7, 7}, {0, 0}, {2, 2}, {0, 0}, {0, 0} },
-	},
-	[IPA_3_5_MHI] = {
-		/* PCIE   DDR	 DMA   unused   N/A     N/A */
-		[IPA_RSRC_GRP_TYPE_RX_HPS_CMDQ] = {
-		{3, 3}, {7, 7}, {2, 2}, {0, 0}, {0, 0}, {0, 0} },
-	},
-	[IPA_3_5_1] = {
-		/* LWA_DL UL_DL	unused   UC_RX_Q N/A     N/A */
-		[IPA_RSRC_GRP_TYPE_RX_HPS_CMDQ] = {
-		{3, 3}, {7, 7}, {0, 0}, {2, 2}, {0, 0}, {0, 0} },
-	},
-	[IPA_4_0] = {
-		/* LWA_DL UL_DL	unused UC_RX_Q, other are invalid */
-		[IPA_RSRC_GRP_TYPE_RX_HPS_CMDQ] = {
-		{3, 3}, {7, 7}, {0, 0}, {2, 2}, {0, 0}, {0, 0} },
-	},
-	[IPA_4_0_MHI] = {
-		/* PCIE   DDR	  DMA     unused   N/A     N/A */
-		[IPA_RSRC_GRP_TYPE_RX_HPS_CMDQ] = {
-		{3, 3}, {7, 7}, {2, 2}, {0, 0}, {0, 0}, {0, 0} },
-	},
-	[IPA_4_1] = {
-		/* LWA_DL UL_DL	unused UC_RX_Q, other are invalid */
-		[IPA_RSRC_GRP_TYPE_RX_HPS_CMDQ] = {
-		{3, 3}, {7, 7}, {0, 0}, {2, 2}, {0, 0}, {0, 0} },
-	},
-	[IPA_4_2] = {
-		/* UL_DL, other are invalid */
-		[IPA_RSRC_GRP_TYPE_RX_HPS_CMDQ] = {
-		{4, 4}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0} },
-	},
-	[IPA_4_5] = {
-		/* unused  UL_DL  unused unused  UC_RX_Q  N/A */
-		[IPA_RSRC_GRP_TYPE_RX_HPS_CMDQ] = {
-		{0, 0}, {3, 3}, {0, 0}, {0, 0}, {0, 0}, {0, 0} },
-	},
-	[IPA_4_5_MHI] = {
-		/* PCIE  DDR  DMA  QDSS  unused  N/A */
-		[IPA_RSRC_GRP_TYPE_RX_HPS_CMDQ] = {
-		{3, 3}, {3, 3}, {3, 3}, {3, 3}, {0, 0}, {0, 0} },
-	},
-	[IPA_4_5_APQ] = {
-		/* unused  UL_DL  unused unused  UC_RX_Q  N/A */
-		[IPA_RSRC_GRP_TYPE_RX_HPS_CMDQ] = {
-		{0, 0}, {3, 3}, {0, 0}, {0, 0}, {0, 0}, {0, 0} },
-	},
-	[IPA_4_7] = {
-		/* unused  UL_DL  unused unused  UC_RX_Q  N/A */
-		[IPA_RSRC_GRP_TYPE_RX_HPS_CMDQ] = {
-		{3, 3}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0} },
-	},
-
-};
-
-static const u32 ipa3_rsrc_rx_grp_hps_weight_config
-	[IPA_VER_MAX][IPA_RSRC_GRP_TYPE_RX_HPS_WEIGHT_MAX][IPA_GROUP_MAX] = {
-	[IPA_3_0] = {
-		/* UL	DL	DIAG	DMA	unused	uC Rx */
-		[IPA_RSRC_GRP_TYPE_RX_HPS_WEIGHT_CONFIG] = { 0, 0, 0, 0, 0, 0 },
-	},
-	[IPA_3_5] = {
-		/* unused UL_DL	unused UC_RX_Q   N/A     N/A */
-		[IPA_RSRC_GRP_TYPE_RX_HPS_WEIGHT_CONFIG] = { 1, 1, 1, 1, 0, 0 },
-	},
-	[IPA_3_5_MHI] = {
-		/* PCIE   DDR	     DMA       unused   N/A        N/A */
-		[IPA_RSRC_GRP_TYPE_RX_HPS_WEIGHT_CONFIG] = { 3, 5, 1, 1, 0, 0 },
-	},
-	[IPA_3_5_1] = {
-		/* LWA_DL UL_DL	unused   UC_RX_Q N/A     N/A */
-		[IPA_RSRC_GRP_TYPE_RX_HPS_WEIGHT_CONFIG] = { 1, 1, 1, 1, 0, 0 },
-	},
-	[IPA_4_0] = {
-		/* LWA_DL UL_DL	unused UC_RX_Q N/A */
-		[IPA_RSRC_GRP_TYPE_RX_HPS_WEIGHT_CONFIG] = { 1, 1, 1, 1, 0, 0 },
-	},
-	[IPA_4_0_MHI] = {
-		/* PCIE   DDR	     DMA       unused   N/A        N/A */
-		[IPA_RSRC_GRP_TYPE_RX_HPS_WEIGHT_CONFIG] = { 3, 5, 1, 1, 0, 0 },
-	},
-	[IPA_4_1] = {
-		/* LWA_DL UL_DL	unused UC_RX_Q, other are invalid */
-		[IPA_RSRC_GRP_TYPE_RX_HPS_WEIGHT_CONFIG] = { 1, 1, 1, 1, 0, 0 },
-	},
-};
-
-enum ipa_ees {
-	IPA_EE_AP = 0,
-	IPA_EE_Q6 = 1,
-	IPA_EE_UC = 2,
-};
-
-enum ipa_qmb_instance_type {
-	IPA_QMB_INSTANCE_DDR = 0,
-	IPA_QMB_INSTANCE_PCIE = 1,
-	IPA_QMB_INSTANCE_MAX
-};
-
-#define QMB_MASTER_SELECT_DDR IPA_QMB_INSTANCE_DDR
-#define QMB_MASTER_SELECT_PCIE IPA_QMB_INSTANCE_PCIE
-
-struct ipa_qmb_outstanding {
-	u16 ot_reads;
-	u16 ot_writes;
-};
-
-static const struct ipa_qmb_outstanding ipa3_qmb_outstanding
-		[IPA_VER_MAX][IPA_QMB_INSTANCE_MAX] = {
-	[IPA_3_0][IPA_QMB_INSTANCE_DDR]	= {8, 8},
-	[IPA_3_0][IPA_QMB_INSTANCE_PCIE]	= {8, 2},
-	[IPA_3_5][IPA_QMB_INSTANCE_DDR]	= {8, 8},
-	[IPA_3_5][IPA_QMB_INSTANCE_PCIE]	= {12, 4},
-	[IPA_3_5_MHI][IPA_QMB_INSTANCE_DDR]	= {8, 8},
-	[IPA_3_5_MHI][IPA_QMB_INSTANCE_PCIE]	= {12, 4},
-	[IPA_3_5_1][IPA_QMB_INSTANCE_DDR]	= {8, 8},
-	[IPA_3_5_1][IPA_QMB_INSTANCE_PCIE]	= {12, 4},
-	[IPA_4_0][IPA_QMB_INSTANCE_DDR]	= {12, 8},
-	[IPA_4_0][IPA_QMB_INSTANCE_PCIE]	= {12, 4},
-	[IPA_4_0_MHI][IPA_QMB_INSTANCE_DDR]	= {12, 8},
-	[IPA_4_0_MHI][IPA_QMB_INSTANCE_PCIE]	= {12, 4},
-	[IPA_4_1][IPA_QMB_INSTANCE_DDR]	= {12, 8},
-	[IPA_4_1][IPA_QMB_INSTANCE_PCIE]	= {12, 4},
-	[IPA_4_2][IPA_QMB_INSTANCE_DDR]	= {12, 8},
-	[IPA_4_5][IPA_QMB_INSTANCE_DDR]	= {16, 8},
-	[IPA_4_5][IPA_QMB_INSTANCE_PCIE]	= {12, 8},
-	[IPA_4_5_MHI][IPA_QMB_INSTANCE_DDR]	= {16, 8},
-	[IPA_4_5_MHI][IPA_QMB_INSTANCE_PCIE]	= {12, 8},
-	[IPA_4_5_APQ][IPA_QMB_INSTANCE_DDR]	= {16, 8},
-	[IPA_4_5_APQ][IPA_QMB_INSTANCE_PCIE]	= {12, 8},
-	[IPA_4_7][IPA_QMB_INSTANCE_DDR]	= {13, 12},
-};
-
-struct ipa_ep_configuration {
-	bool valid;
-	int group_num;
-	bool support_flt;
-	int sequencer_type;
-	u8 qmb_master_sel;
-	struct ipa_gsi_ep_config ipa_gsi_ep_info;
-};
-
-/* clients not included in the list below are considered as invalid */
-static const struct ipa_ep_configuration ipa3_ep_mapping
-					[IPA_VER_MAX][IPA_CLIENT_MAX] = {
-	[IPA_3_0][IPA_CLIENT_WLAN1_PROD]          = {
-			true, IPA_v3_0_GROUP_UL, true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 10, 1, 8, 16, IPA_EE_UC } },
-	[IPA_3_0][IPA_CLIENT_USB_PROD]            = {
-			true, IPA_v3_0_GROUP_UL, true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 1, 3, 8, 16, IPA_EE_AP } },
-	[IPA_3_0][IPA_CLIENT_APPS_LAN_PROD] = {
-			true, IPA_v3_0_GROUP_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 14, 11, 8, 16, IPA_EE_AP } },
-	[IPA_3_0][IPA_CLIENT_APPS_WAN_PROD] = {
-			true, IPA_v3_0_GROUP_UL, true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 3, 5, 16, 32, IPA_EE_AP } },
-	[IPA_3_0][IPA_CLIENT_APPS_CMD_PROD]	  = {
-			true, IPA_v3_0_GROUP_IMM_CMD, false,
-			IPA_DPS_HPS_SEQ_TYPE_DMA_ONLY,
-			QMB_MASTER_SELECT_DDR,
-			{ 22, 6, 18, 28, IPA_EE_AP } },
-	[IPA_3_0][IPA_CLIENT_ODU_PROD]            = {
-			true, IPA_v3_0_GROUP_UL, true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 12, 9, 8, 16, IPA_EE_AP } },
-	[IPA_3_0][IPA_CLIENT_MHI_PROD]            = {
-			true, IPA_v3_0_GROUP_UL, true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_PCIE,
-			{ 0, 0, 8, 16, IPA_EE_AP } },
-	[IPA_3_0][IPA_CLIENT_Q6_LAN_PROD]         = {
-			true, IPA_v3_0_GROUP_UL, false,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 9, 4, 8, 12, IPA_EE_Q6 } },
-	[IPA_3_0][IPA_CLIENT_Q6_WAN_PROD]         = {
-			true, IPA_v3_0_GROUP_DL, true,
-			IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 5, 0, 16, 32, IPA_EE_Q6 } },
-	[IPA_3_0][IPA_CLIENT_Q6_CMD_PROD] = {
-			true, IPA_v3_0_GROUP_IMM_CMD, false,
-			IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 6, 1, 18, 28, IPA_EE_Q6 } },
-	[IPA_3_0][IPA_CLIENT_Q6_DECOMP_PROD]      = {
-			true, IPA_v3_0_GROUP_Q6ZIP,
-			false, IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 7, 2, 0, 0, IPA_EE_Q6 } },
-	[IPA_3_0][IPA_CLIENT_Q6_DECOMP2_PROD]     = {
-			true, IPA_v3_0_GROUP_Q6ZIP,
-			false, IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 8, 3, 0, 0, IPA_EE_Q6 } },
-	[IPA_3_0][IPA_CLIENT_MEMCPY_DMA_SYNC_PROD] = {
-			true, IPA_v3_0_GROUP_DMA, false,
-			IPA_DPS_HPS_SEQ_TYPE_DMA_ONLY,
-			QMB_MASTER_SELECT_PCIE,
-			{ 12, 9, 8, 16, IPA_EE_AP } },
-	[IPA_3_0][IPA_CLIENT_MEMCPY_DMA_ASYNC_PROD] = {
-			true, IPA_v3_0_GROUP_DMA, false,
-			IPA_DPS_HPS_SEQ_TYPE_DMA_ONLY,
-			QMB_MASTER_SELECT_PCIE,
-			{ 13, 10, 8, 16, IPA_EE_AP } },
-	[IPA_3_0][IPA_CLIENT_ETHERNET_PROD]          = {
-			true, IPA_v3_0_GROUP_UL, true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{2, 0, 8, 16, IPA_EE_UC} },
-	/* Only for test purpose */
-	[IPA_3_0][IPA_CLIENT_TEST_PROD]           = {
-			true, IPA_v3_0_GROUP_UL, true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 1, 3, 8, 16, IPA_EE_AP } },
-	[IPA_3_0][IPA_CLIENT_TEST1_PROD]          = {
-			true, IPA_v3_0_GROUP_UL, true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 1, 3, 8, 16, IPA_EE_AP } },
-	[IPA_3_0][IPA_CLIENT_TEST2_PROD]          = {
-			true, IPA_v3_0_GROUP_UL, true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 3, 5, 16, 32, IPA_EE_AP } },
-	[IPA_3_0][IPA_CLIENT_TEST3_PROD]          = {
-			true, IPA_v3_0_GROUP_UL, true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 12, 9, 8, 16, IPA_EE_AP } },
-	[IPA_3_0][IPA_CLIENT_TEST4_PROD]          = {
-			true, IPA_v3_0_GROUP_UL, true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 13, 10, 8, 16, IPA_EE_AP } },
-
-	[IPA_3_0][IPA_CLIENT_WLAN1_CONS]          = {
-			true, IPA_v3_0_GROUP_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 25, 4, 8, 8, IPA_EE_UC } },
-	[IPA_3_0][IPA_CLIENT_WLAN2_CONS]          = {
-			true, IPA_v3_0_GROUP_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 27, 4, 8, 8, IPA_EE_AP } },
-	[IPA_3_0][IPA_CLIENT_WLAN3_CONS]          = {
-			true, IPA_v3_0_GROUP_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 28, 13, 8, 8, IPA_EE_AP } },
-	[IPA_3_0][IPA_CLIENT_WLAN4_CONS]          = {
-			true, IPA_v3_0_GROUP_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 29, 14, 8, 8, IPA_EE_AP } },
-	[IPA_3_0][IPA_CLIENT_USB_CONS]            = {
-			true, IPA_v3_0_GROUP_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 26, 12, 8, 8, IPA_EE_AP } },
-	[IPA_3_0][IPA_CLIENT_USB_DPL_CONS]        = {
-			true, IPA_v3_0_GROUP_DPL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 17, 2, 8, 12, IPA_EE_AP } },
-	[IPA_3_0][IPA_CLIENT_APPS_LAN_CONS]       = {
-			true, IPA_v3_0_GROUP_UL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 15, 7, 8, 12, IPA_EE_AP } },
-	[IPA_3_0][IPA_CLIENT_APPS_WAN_CONS]       = {
-			true, IPA_v3_0_GROUP_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 16, 8, 8, 12, IPA_EE_AP } },
-	[IPA_3_0][IPA_CLIENT_ODU_EMB_CONS]        = {
-			true, IPA_v3_0_GROUP_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 23, 1, 8, 8, IPA_EE_AP } },
-	[IPA_3_0][IPA_CLIENT_MHI_CONS]            = {
-			true, IPA_v3_0_GROUP_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_PCIE,
-			{ 23, 1, 8, 8, IPA_EE_AP } },
-	[IPA_3_0][IPA_CLIENT_Q6_LAN_CONS]         = {
-			true, IPA_v3_0_GROUP_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 19, 6, 8, 12, IPA_EE_Q6 } },
-	[IPA_3_0][IPA_CLIENT_Q6_WAN_CONS]         = {
-			true, IPA_v3_0_GROUP_UL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 18, 5, 8, 12, IPA_EE_Q6 } },
-	[IPA_3_0][IPA_CLIENT_Q6_DUN_CONS]         = {
-			true, IPA_v3_0_GROUP_DIAG, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 30, 7, 4, 4, IPA_EE_Q6 } },
-	[IPA_3_0][IPA_CLIENT_Q6_DECOMP_CONS] = {
-			true, IPA_v3_0_GROUP_Q6ZIP, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 21, 8, 4, 4, IPA_EE_Q6 } },
-	[IPA_3_0][IPA_CLIENT_Q6_DECOMP2_CONS] = {
-			true, IPA_v3_0_GROUP_Q6ZIP, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 4, 9, 4, 4, IPA_EE_Q6 } },
-	[IPA_3_0][IPA_CLIENT_MEMCPY_DMA_SYNC_CONS] = {
-			true, IPA_v3_0_GROUP_DMA, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_PCIE,
-			{ 28, 13, 8, 8, IPA_EE_AP } },
-	[IPA_3_0][IPA_CLIENT_MEMCPY_DMA_ASYNC_CONS] = {
-			true, IPA_v3_0_GROUP_DMA, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_PCIE,
-			{ 29, 14, 8, 8, IPA_EE_AP } },
-	[IPA_3_0][IPA_CLIENT_ETHERNET_CONS]          = {
-			true, IPA_v3_0_GROUP_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{24, 3, 8, 8, IPA_EE_UC} },
-	/* Only for test purpose */
-	[IPA_3_0][IPA_CLIENT_TEST_CONS]           = {
-			true, IPA_v3_0_GROUP_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 26, 12, 8, 8, IPA_EE_AP } },
-	[IPA_3_0][IPA_CLIENT_TEST1_CONS]          = {
-			true, IPA_v3_0_GROUP_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 26, 12, 8, 8, IPA_EE_AP } },
-	[IPA_3_0][IPA_CLIENT_TEST2_CONS]          = {
-			true, IPA_v3_0_GROUP_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 27, 4, 8, 8, IPA_EE_AP } },
-	[IPA_3_0][IPA_CLIENT_TEST3_CONS]          = {
-			true, IPA_v3_0_GROUP_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 28, 13, 8, 8, IPA_EE_AP } },
-	[IPA_3_0][IPA_CLIENT_TEST4_CONS]          = {
-			true, IPA_v3_0_GROUP_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 29, 14, 8, 8, IPA_EE_AP } },
-	/* Dummy consumer (pipe 31) is used in L2TP rt rule */
-	[IPA_3_0][IPA_CLIENT_DUMMY_CONS]          = {
-			true, IPA_v3_0_GROUP_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 31, 31, 8, 8, IPA_EE_AP } },
-
-	/* IPA_3_5 */
-	[IPA_3_5][IPA_CLIENT_WLAN1_PROD]          = {
-			true, IPA_v3_5_GROUP_UL_DL, true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 6, 1, 8, 16, IPA_EE_UC } },
-	[IPA_3_5][IPA_CLIENT_USB_PROD]            = {
-			true, IPA_v3_5_GROUP_UL_DL, true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 0, 7, 8, 16, IPA_EE_AP } },
-	[IPA_3_5][IPA_CLIENT_APPS_LAN_PROD]   = {
-			true, IPA_v3_5_GROUP_UL_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 8, 9, 8, 16, IPA_EE_AP } },
-	[IPA_3_5][IPA_CLIENT_APPS_WAN_PROD] = {
-			true, IPA_v3_5_GROUP_UL_DL, true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 2, 3, 16, 32, IPA_EE_AP } },
-	[IPA_3_5][IPA_CLIENT_APPS_CMD_PROD]	  = {
-			true, IPA_v3_5_GROUP_UL_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_DMA_ONLY,
-			QMB_MASTER_SELECT_DDR,
-			{ 5, 4, 20, 23, IPA_EE_AP } },
-	[IPA_3_5][IPA_CLIENT_ODU_PROD]            = {
-			true, IPA_v3_5_GROUP_UL_DL, true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 1, 0, 8, 16, IPA_EE_UC } },
-	[IPA_3_5][IPA_CLIENT_Q6_LAN_PROD]         = {
-			true, IPA_v3_5_GROUP_UL_DL, true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 3, 0, 16, 32, IPA_EE_Q6 } },
-	[IPA_3_5][IPA_CLIENT_Q6_CMD_PROD]	  = {
-			true, IPA_v3_5_GROUP_UL_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 4, 1, 20, 23, IPA_EE_Q6 } },
-	/* Only for test purpose */
-	[IPA_3_5][IPA_CLIENT_TEST_PROD]           = {
-			true, IPA_v3_5_GROUP_UL_DL, true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{0, 7, 8, 16, IPA_EE_AP } },
-	[IPA_3_5][IPA_CLIENT_TEST1_PROD]          = {
-			true, IPA_v3_5_GROUP_UL_DL, true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{0, 7, 8, 16, IPA_EE_AP } },
-	[IPA_3_5][IPA_CLIENT_TEST2_PROD]          = {
-			true, IPA_v3_5_GROUP_UL_DL, true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 1, 0, 8, 16, IPA_EE_AP } },
-	[IPA_3_5][IPA_CLIENT_TEST3_PROD]          = {
-			true, IPA_v3_5_GROUP_UL_DL, true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{7, 8, 8, 16, IPA_EE_AP } },
-	[IPA_3_5][IPA_CLIENT_TEST4_PROD]          = {
-			true, IPA_v3_5_GROUP_UL_DL, true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 8, 9, 8, 16, IPA_EE_AP } },
-
-	[IPA_3_5][IPA_CLIENT_WLAN1_CONS]          = {
-			true, IPA_v3_5_GROUP_UL_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 16, 3, 8, 8, IPA_EE_UC } },
-	[IPA_3_5][IPA_CLIENT_WLAN2_CONS]          = {
-			true, IPA_v3_5_GROUP_UL_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 18, 12, 8, 8, IPA_EE_AP } },
-	[IPA_3_5][IPA_CLIENT_WLAN3_CONS]          = {
-			true, IPA_v3_5_GROUP_UL_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 19, 13, 8, 8, IPA_EE_AP } },
-	[IPA_3_5][IPA_CLIENT_USB_CONS]            = {
-			true, IPA_v3_5_GROUP_UL_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_PCIE,
-			{ 17, 11, 8, 8, IPA_EE_AP } },
-	[IPA_3_5][IPA_CLIENT_USB_DPL_CONS]        = {
-			true, IPA_v3_5_GROUP_UL_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 14, 10, 4, 6, IPA_EE_AP } },
-	[IPA_3_5][IPA_CLIENT_APPS_LAN_CONS]       = {
-			true, IPA_v3_5_GROUP_UL_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 9, 5, 8, 12, IPA_EE_AP } },
-	[IPA_3_5][IPA_CLIENT_APPS_WAN_CONS]       = {
-			true, IPA_v3_5_GROUP_UL_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 10, 6, 8, 12, IPA_EE_AP } },
-	[IPA_3_5][IPA_CLIENT_ODU_EMB_CONS]        = {
-			true, IPA_v3_5_GROUP_UL_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 15, 1, 8, 8, IPA_EE_AP } },
-	[IPA_3_5][IPA_CLIENT_Q6_LAN_CONS]         = {
-			true, IPA_v3_5_GROUP_UL_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 13, 3, 8, 12, IPA_EE_Q6 } },
-	[IPA_3_5][IPA_CLIENT_Q6_WAN_CONS]         = {
-			true, IPA_v3_5_GROUP_UL_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 12, 2, 8, 12, IPA_EE_Q6 } },
-	/* Only for test purpose */
-	/* MBIM aggregation test pipes should have the same QMB as USB_CONS */
-	[IPA_3_5][IPA_CLIENT_TEST_CONS]           = {
-			true, IPA_v3_5_GROUP_UL_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_PCIE,
-			{ 15, 1, 8, 8, IPA_EE_AP } },
-	[IPA_3_5][IPA_CLIENT_TEST1_CONS]           = {
-			true, IPA_v3_5_GROUP_UL_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 15, 1, 8, 8, IPA_EE_AP } },
-	[IPA_3_5][IPA_CLIENT_TEST2_CONS]          = {
-			true, IPA_v3_5_GROUP_UL_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_PCIE,
-			{ 17, 11, 8, 8, IPA_EE_AP } },
-	[IPA_3_5][IPA_CLIENT_TEST3_CONS]          = {
-			true, IPA_v3_5_GROUP_UL_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 18, 12, 8, 8, IPA_EE_AP } },
-	[IPA_3_5][IPA_CLIENT_TEST4_CONS]          = {
-			true, IPA_v3_5_GROUP_UL_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_PCIE,
-			{ 19, 13, 8, 8, IPA_EE_AP } },
-	/* Dummy consumer (pipe 31) is used in L2TP rt rule */
-	[IPA_3_5][IPA_CLIENT_DUMMY_CONS]          = {
-			true, IPA_v3_5_GROUP_UL_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_PCIE,
-			{ 31, 31, 8, 8, IPA_EE_AP } },
-
-	/* IPA_3_5_MHI */
-	[IPA_3_5_MHI][IPA_CLIENT_USB_PROD]            = {
-			false, IPA_EP_NOT_ALLOCATED, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ -1, -1, -1, -1, -1 } },
-	[IPA_3_5_MHI][IPA_CLIENT_APPS_WAN_PROD]   = {
-			true, IPA_v3_5_MHI_GROUP_DDR, true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 2, 3, 16, 32, IPA_EE_AP } },
-	[IPA_3_5_MHI][IPA_CLIENT_APPS_CMD_PROD]	  = {
-			true, IPA_v3_5_MHI_GROUP_DDR, false,
-			IPA_DPS_HPS_SEQ_TYPE_DMA_ONLY,
-			QMB_MASTER_SELECT_DDR,
-			{ 5, 4, 20, 23, IPA_EE_AP } },
-	[IPA_3_5_MHI][IPA_CLIENT_MHI_PROD]            = {
-			true, IPA_v3_5_MHI_GROUP_PCIE, true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_PCIE,
-			{ 1, 0, 8, 16, IPA_EE_AP } },
-	[IPA_3_5_MHI][IPA_CLIENT_Q6_LAN_PROD]         = {
-			true, IPA_v3_5_MHI_GROUP_DDR, true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 3, 0, 16, 32, IPA_EE_Q6 } },
-	[IPA_3_5_MHI][IPA_CLIENT_Q6_WAN_PROD]         = {
-			true, IPA_v3_5_MHI_GROUP_DDR, true,
-			IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 6, 4, 10, 30, IPA_EE_Q6 } },
-	[IPA_3_5_MHI][IPA_CLIENT_Q6_CMD_PROD]	  = {
-			true, IPA_v3_5_MHI_GROUP_PCIE, false,
-			IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 4, 1, 20, 23, IPA_EE_Q6 } },
-	[IPA_3_5_MHI][IPA_CLIENT_MEMCPY_DMA_SYNC_PROD] = {
-			true, IPA_v3_5_MHI_GROUP_DMA, false,
-			IPA_DPS_HPS_SEQ_TYPE_DMA_ONLY,
-			QMB_MASTER_SELECT_DDR,
-			{ 7, 8, 8, 16, IPA_EE_AP } },
-	[IPA_3_5_MHI][IPA_CLIENT_MEMCPY_DMA_ASYNC_PROD] = {
-			true, IPA_v3_5_MHI_GROUP_DMA, false,
-			IPA_DPS_HPS_SEQ_TYPE_DMA_ONLY,
-			QMB_MASTER_SELECT_DDR,
-			{ 8, 9, 8, 16, IPA_EE_AP } },
-	/* Only for test purpose */
-	[IPA_3_5_MHI][IPA_CLIENT_TEST_PROD]           = {
-			true, IPA_v3_5_MHI_GROUP_DDR, true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{0, 7, 8, 16, IPA_EE_AP } },
-	[IPA_3_5_MHI][IPA_CLIENT_TEST1_PROD]          = {
-			0, IPA_v3_5_MHI_GROUP_DDR, true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{0, 7, 8, 16, IPA_EE_AP } },
-	[IPA_3_5_MHI][IPA_CLIENT_TEST2_PROD]          = {
-			true, IPA_v3_5_MHI_GROUP_PCIE, true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_PCIE,
-			{ 1, 0, 8, 16, IPA_EE_AP } },
-	[IPA_3_5_MHI][IPA_CLIENT_TEST3_PROD]          = {
-			true, IPA_v3_5_MHI_GROUP_DMA, true,
-			IPA_DPS_HPS_SEQ_TYPE_DMA_ONLY,
-			QMB_MASTER_SELECT_DDR,
-			{ 7, 8, 8, 16, IPA_EE_AP } },
-	[IPA_3_5_MHI][IPA_CLIENT_TEST4_PROD]          = {
-			true, IPA_v3_5_MHI_GROUP_DMA, true,
-			IPA_DPS_HPS_SEQ_TYPE_DMA_ONLY,
-			QMB_MASTER_SELECT_DDR,
-			{ 8, 9, 8, 16, IPA_EE_AP } },
-
-	[IPA_3_5_MHI][IPA_CLIENT_WLAN1_CONS]          = {
-			true, IPA_v3_5_MHI_GROUP_DDR, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 16, 3, 8, 8, IPA_EE_UC } },
-	[IPA_3_5_MHI][IPA_CLIENT_USB_CONS]            = {
-			false, IPA_EP_NOT_ALLOCATED, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ -1, -1, -1, -1, -1 } },
-	[IPA_3_5_MHI][IPA_CLIENT_USB_DPL_CONS]        = {
-			false, IPA_EP_NOT_ALLOCATED, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ -1, -1, -1, -1, -1 } },
-	[IPA_3_5_MHI][IPA_CLIENT_APPS_LAN_CONS]       = {
-			true, IPA_v3_5_MHI_GROUP_DDR, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 9, 5, 8, 12, IPA_EE_AP } },
-	[IPA_3_5_MHI][IPA_CLIENT_APPS_WAN_CONS]       = {
-			true, IPA_v3_5_MHI_GROUP_DDR, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 10, 6, 8, 12, IPA_EE_AP } },
-	[IPA_3_5_MHI][IPA_CLIENT_MHI_CONS]            = {
-			true, IPA_v3_5_MHI_GROUP_PCIE, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_PCIE,
-			{ 15, 1, 8, 8, IPA_EE_AP } },
-	[IPA_3_5_MHI][IPA_CLIENT_Q6_LAN_CONS]         = {
-			true, IPA_v3_5_MHI_GROUP_DDR, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 13, 3, 8, 12, IPA_EE_Q6 } },
-	[IPA_3_5_MHI][IPA_CLIENT_Q6_WAN_CONS]         = {
-			true, IPA_v3_5_MHI_GROUP_DDR, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 12, 2, 8, 12, IPA_EE_Q6 } },
-	[IPA_3_5_MHI][IPA_CLIENT_MEMCPY_DMA_SYNC_CONS] = {
-			true, IPA_v3_5_MHI_GROUP_DMA, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_PCIE,
-			{ 18, 12, 8, 8, IPA_EE_AP } },
-	[IPA_3_5_MHI][IPA_CLIENT_MEMCPY_DMA_ASYNC_CONS] = {
-			true, IPA_v3_5_MHI_GROUP_DMA, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_PCIE,
-			{ 19, 13, 8, 8, IPA_EE_AP } },
-	/* Only for test purpose */
-	[IPA_3_5_MHI][IPA_CLIENT_TEST_CONS]           = {
-			true, IPA_v3_5_MHI_GROUP_PCIE, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_PCIE,
-			{ 15, 1, 8, 8, IPA_EE_AP } },
-	[IPA_3_5_MHI][IPA_CLIENT_TEST1_CONS]           = {
-			true, IPA_v3_5_MHI_GROUP_PCIE, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_PCIE,
-			{ 15, 1, 8, 8, IPA_EE_AP } },
-	[IPA_3_5_MHI][IPA_CLIENT_TEST2_CONS]          = {
-			true, IPA_v3_5_MHI_GROUP_DDR, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 17, 11, 8, 8, IPA_EE_AP } },
-	[IPA_3_5_MHI][IPA_CLIENT_TEST3_CONS]          = {
-			true, IPA_v3_5_MHI_GROUP_DMA, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_PCIE,
-			{ 18, 12, 8, 8, IPA_EE_AP } },
-	[IPA_3_5_MHI][IPA_CLIENT_TEST4_CONS]          = {
-			true, IPA_v3_5_MHI_GROUP_DMA, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_PCIE,
-			{ 19, 13, 8, 8, IPA_EE_AP } },
-	/* Dummy consumer (pipe 31) is used in L2TP rt rule */
-	[IPA_3_5_MHI][IPA_CLIENT_DUMMY_CONS]          = {
-			true, IPA_v3_5_MHI_GROUP_DMA, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_PCIE,
-			{ 31, 31, 8, 8, IPA_EE_AP } },
-
-	/* IPA_3_5_1 */
-	[IPA_3_5_1][IPA_CLIENT_WLAN1_PROD]          = {
-			true, IPA_v3_5_GROUP_UL_DL, true,
-			IPA_DPS_HPS_REP_SEQ_TYPE_2PKT_PROC_PASS_NO_DEC_UCP_DMAP,
-			QMB_MASTER_SELECT_DDR,
-			{ 7, 1, 8, 16, IPA_EE_UC } },
-	[IPA_3_5_1][IPA_CLIENT_USB_PROD]            = {
-			true, IPA_v3_5_GROUP_UL_DL, true,
-			IPA_DPS_HPS_REP_SEQ_TYPE_2PKT_PROC_PASS_NO_DEC_UCP_DMAP,
-			QMB_MASTER_SELECT_DDR,
-			{ 0, 0, 8, 16, IPA_EE_AP } },
-	[IPA_3_5_1][IPA_CLIENT_APPS_LAN_PROD] = {
-			true, IPA_v3_5_GROUP_UL_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 8, 7, 8, 16, IPA_EE_AP } },
-	[IPA_3_5_1][IPA_CLIENT_APPS_WAN_PROD] = {
-			true, IPA_v3_5_GROUP_UL_DL, true,
-			IPA_DPS_HPS_REP_SEQ_TYPE_2PKT_PROC_PASS_NO_DEC_UCP_DMAP,
-			QMB_MASTER_SELECT_DDR,
-			{ 2, 3, 16, 32, IPA_EE_AP } },
-	[IPA_3_5_1][IPA_CLIENT_APPS_CMD_PROD]		= {
-			true, IPA_v3_5_GROUP_UL_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_DMA_ONLY,
-			QMB_MASTER_SELECT_DDR,
-			{ 5, 4, 20, 23, IPA_EE_AP } },
-	[IPA_3_5_1][IPA_CLIENT_Q6_LAN_PROD]         = {
-			true, IPA_v3_5_GROUP_UL_DL, true,
-			IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 3, 0, 16, 32, IPA_EE_Q6 } },
-	[IPA_3_5_1][IPA_CLIENT_Q6_WAN_PROD]         = {
-			true, IPA_v3_5_GROUP_UL_DL, true,
-			IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 6, 4, 12, 30, IPA_EE_Q6 } },
-	[IPA_3_5_1][IPA_CLIENT_Q6_CMD_PROD]	    = {
-			true, IPA_v3_5_GROUP_UL_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 4, 1, 20, 23, IPA_EE_Q6 } },
-	/* Only for test purpose */
-	[IPA_3_5_1][IPA_CLIENT_TEST_PROD]           = {
-			true, IPA_v3_5_GROUP_UL_DL, true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 0, 0, 8, 16, IPA_EE_AP } },
-	[IPA_3_5_1][IPA_CLIENT_TEST1_PROD]          = {
-			true, IPA_v3_5_GROUP_UL_DL, true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 0, 0, 8, 16, IPA_EE_AP } },
-	[IPA_3_5_1][IPA_CLIENT_TEST2_PROD]          = {
-			true, IPA_v3_5_GROUP_UL_DL, true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 2, 3, 16, 32, IPA_EE_AP } },
-	[IPA_3_5_1][IPA_CLIENT_TEST3_PROD]          = {
-			true, IPA_v3_5_GROUP_UL_DL, true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 4, 1, 20, 23, IPA_EE_Q6 } },
-	[IPA_3_5_1][IPA_CLIENT_TEST4_PROD]          = {
-			true, IPA_v3_5_GROUP_UL_DL, true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 1, 0, 8, 16, IPA_EE_UC } },
-
-	[IPA_3_5_1][IPA_CLIENT_WLAN1_CONS]          = {
-			true, IPA_v3_5_GROUP_UL_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 16, 11, 8, 8, IPA_EE_UC } },
-	[IPA_3_5_1][IPA_CLIENT_WLAN2_CONS]          =  {
-			true, IPA_v3_5_GROUP_UL_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 18, 9, 8, 8, IPA_EE_AP } },
-	[IPA_3_5_1][IPA_CLIENT_WLAN3_CONS]          =  {
-			true, IPA_v3_5_GROUP_UL_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 19, 10, 8, 8, IPA_EE_AP } },
-	[IPA_3_5_1][IPA_CLIENT_USB_CONS]            = {
-			true, IPA_v3_5_GROUP_UL_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 17, 8, 8, 8, IPA_EE_AP } },
-	[IPA_3_5_1][IPA_CLIENT_USB_DPL_CONS]        = {
-			true, IPA_v3_5_GROUP_UL_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 11, 2, 4, 6, IPA_EE_AP } },
-	[IPA_3_5_1][IPA_CLIENT_APPS_LAN_CONS]       = {
-			true, IPA_v3_5_GROUP_UL_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 9, 5, 8, 12, IPA_EE_AP } },
-	[IPA_3_5_1][IPA_CLIENT_APPS_WAN_CONS]       = {
-			true, IPA_v3_5_GROUP_UL_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 10, 6, 8, 12, IPA_EE_AP } },
-	[IPA_3_5_1][IPA_CLIENT_Q6_LAN_CONS]         = {
-			true, IPA_v3_5_GROUP_UL_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 13, 3, 8, 12, IPA_EE_Q6 } },
-	[IPA_3_5_1][IPA_CLIENT_Q6_WAN_CONS]         = {
-			true, IPA_v3_5_GROUP_UL_DL, false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 12, 2, 8, 12, IPA_EE_Q6 } },
-	/* Only for test purpose */
-	[IPA_3_5_1][IPA_CLIENT_TEST_CONS]           = {
-			true, IPA_v3_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 17, 8, 8, 8, IPA_EE_AP } },
-	[IPA_3_5_1][IPA_CLIENT_TEST1_CONS]          = {
-			true, IPA_v3_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 17, 8, 8, 8, IPA_EE_AP } },
-	[IPA_3_5_1][IPA_CLIENT_TEST2_CONS]          = {
-			true, IPA_v3_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 18, 9, 8, 8, IPA_EE_AP } },
-	[IPA_3_5_1][IPA_CLIENT_TEST3_CONS]          = {
-			true, IPA_v3_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 19, 10, 8, 8, IPA_EE_AP } },
-	[IPA_3_5_1][IPA_CLIENT_TEST4_CONS]          = {
-			true, IPA_v3_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 11, 2, 4, 6, IPA_EE_AP } },
-	/* Dummy consumer (pipe 31) is used in L2TP rt rule */
-	[IPA_3_5_1][IPA_CLIENT_DUMMY_CONS]          = {
-			true, IPA_v3_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 31, 31, 8, 8, IPA_EE_AP } },
-
-	/* IPA_4_0 */
-	[IPA_4_0][IPA_CLIENT_WLAN1_PROD]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_REP_SEQ_TYPE_2PKT_PROC_PASS_NO_DEC_UCP_DMAP,
-			QMB_MASTER_SELECT_DDR,
-			{ 6, 2, 8, 16, IPA_EE_UC } },
-	[IPA_4_0][IPA_CLIENT_USB_PROD]            = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_REP_SEQ_TYPE_2PKT_PROC_PASS_NO_DEC_UCP_DMAP,
-			QMB_MASTER_SELECT_DDR,
-			{ 0, 8, 8, 16, IPA_EE_AP } },
-	[IPA_4_0][IPA_CLIENT_APPS_LAN_PROD]   = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 8, 10, 8, 16, IPA_EE_AP } },
-	[IPA_4_0][IPA_CLIENT_APPS_WAN_PROD] = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_REP_SEQ_TYPE_2PKT_PROC_PASS_NO_DEC_UCP_DMAP,
-			QMB_MASTER_SELECT_DDR,
-			{ 2, 3, 16, 32, IPA_EE_AP } },
-	[IPA_4_0][IPA_CLIENT_APPS_CMD_PROD]	  = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_DMA_ONLY,
-			QMB_MASTER_SELECT_DDR,
-			{ 5, 4, 20, 24, IPA_EE_AP } },
-	[IPA_4_0][IPA_CLIENT_ODU_PROD]            = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_REP_SEQ_TYPE_2PKT_PROC_PASS_NO_DEC_UCP_DMAP,
-			QMB_MASTER_SELECT_DDR,
-			{ 1, 0, 8, 16, IPA_EE_AP } },
-	[IPA_4_0][IPA_CLIENT_ETHERNET_PROD]	  = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_REP_SEQ_TYPE_2PKT_PROC_PASS_NO_DEC_UCP_DMAP,
-			QMB_MASTER_SELECT_DDR,
-			{ 9, 0, 8, 16, IPA_EE_UC } },
-	[IPA_4_0][IPA_CLIENT_Q6_WAN_PROD]         = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 3, 0, 16, 32, IPA_EE_Q6 } },
-	[IPA_4_0][IPA_CLIENT_Q6_CMD_PROD]	  = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 4, 1, 20, 24, IPA_EE_Q6 } },
-	/* Only for test purpose */
-	[IPA_4_0][IPA_CLIENT_TEST_PROD]           = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{0, 8, 8, 16, IPA_EE_AP } },
-	[IPA_4_0][IPA_CLIENT_TEST1_PROD]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{0, 8, 8, 16, IPA_EE_AP } },
-	[IPA_4_0][IPA_CLIENT_TEST2_PROD]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 1, 0, 8, 16, IPA_EE_AP } },
-	[IPA_4_0][IPA_CLIENT_TEST3_PROD]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 7, 9, 8, 16, IPA_EE_AP } },
-	[IPA_4_0][IPA_CLIENT_TEST4_PROD]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{8, 10, 8, 16, IPA_EE_AP } },
-
-
-	[IPA_4_0][IPA_CLIENT_WLAN1_CONS]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 18, 3, 6, 9, IPA_EE_UC } },
-	[IPA_4_0][IPA_CLIENT_WLAN2_CONS]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 20, 13, 9, 9, IPA_EE_AP } },
-	[IPA_4_0][IPA_CLIENT_WLAN3_CONS]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 21, 14, 9, 9, IPA_EE_AP } },
-	[IPA_4_0][IPA_CLIENT_USB_CONS]            = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 19, 12, 9, 9, IPA_EE_AP } },
-	[IPA_4_0][IPA_CLIENT_USB_DPL_CONS]        = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 15, 7, 5, 5, IPA_EE_AP } },
-	[IPA_4_0][IPA_CLIENT_APPS_LAN_CONS]       = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 10, 5, 9, 9, IPA_EE_AP } },
-	[IPA_4_0][IPA_CLIENT_APPS_WAN_CONS]       = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 11, 6, 9, 9, IPA_EE_AP } },
-	[IPA_4_0][IPA_CLIENT_ODU_EMB_CONS]        = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 17, 1, 17, 17, IPA_EE_AP } },
-	[IPA_4_0][IPA_CLIENT_ETHERNET_CONS]	  = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 22, 1, 17, 17, IPA_EE_UC } },
-	[IPA_4_0][IPA_CLIENT_Q6_LAN_CONS]         = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 14, 4, 9, 9, IPA_EE_Q6 } },
-	[IPA_4_0][IPA_CLIENT_Q6_WAN_CONS]         = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 13, 3, 9, 9, IPA_EE_Q6 } },
-	[IPA_4_0][IPA_CLIENT_Q6_LTE_WIFI_AGGR_CONS] = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 16, 5, 9, 9, IPA_EE_Q6 } },
-	/* Only for test purpose */
-	/* MBIM aggregation test pipes should have the same QMB as USB_CONS */
-	[IPA_4_0][IPA_CLIENT_TEST_CONS]           = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 11, 6, 9, 9, IPA_EE_AP } },
-	[IPA_4_0][IPA_CLIENT_TEST1_CONS]           = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 11, 6, 9, 9, IPA_EE_AP } },
-	[IPA_4_0][IPA_CLIENT_TEST2_CONS]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 12, 2, 5, 5, IPA_EE_AP } },
-	[IPA_4_0][IPA_CLIENT_TEST3_CONS]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 19, 12, 9, 9, IPA_EE_AP } },
-	[IPA_4_0][IPA_CLIENT_TEST4_CONS]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 21, 14, 9, 9, IPA_EE_AP } },
-	/* Dummy consumer (pipe 31) is used in L2TP rt rule */
-	[IPA_4_0][IPA_CLIENT_DUMMY_CONS]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 31, 31, 8, 8, IPA_EE_AP } },
-
-	/* IPA_4_0_MHI */
-	[IPA_4_0_MHI][IPA_CLIENT_APPS_WAN_PROD]   = {
-			true, IPA_v4_0_MHI_GROUP_DDR,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 2, 3, 16, 32, IPA_EE_AP } },
-	[IPA_4_0_MHI][IPA_CLIENT_APPS_CMD_PROD]	  = {
-			true, IPA_v4_0_MHI_GROUP_DDR,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_DMA_ONLY,
-			QMB_MASTER_SELECT_DDR,
-			{ 5, 4, 20, 24, IPA_EE_AP } },
-	[IPA_4_0_MHI][IPA_CLIENT_MHI_PROD]            = {
-			true, IPA_v4_0_MHI_GROUP_PCIE,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_PCIE,
-			{ 1, 0, 8, 16, IPA_EE_AP } },
-	[IPA_4_0_MHI][IPA_CLIENT_Q6_WAN_PROD]         = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 3, 0, 16, 32, IPA_EE_Q6 } },
-	[IPA_4_0_MHI][IPA_CLIENT_Q6_CMD_PROD]	  = {
-			true, IPA_v4_0_MHI_GROUP_PCIE,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 4, 1, 20, 24, IPA_EE_Q6 } },
-	[IPA_4_0_MHI][IPA_CLIENT_MEMCPY_DMA_SYNC_PROD] = {
-			true, IPA_v4_0_MHI_GROUP_DMA,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_DMA_ONLY,
-			QMB_MASTER_SELECT_DDR,
-			{ 7, 9, 8, 16, IPA_EE_AP } },
-	[IPA_4_0_MHI][IPA_CLIENT_MEMCPY_DMA_ASYNC_PROD] = {
-			true, IPA_v4_0_MHI_GROUP_DMA,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_DMA_ONLY,
-			QMB_MASTER_SELECT_DDR,
-			{ 8, 10, 8, 16, IPA_EE_AP } },
-	/* Only for test purpose */
-	[IPA_4_0_MHI][IPA_CLIENT_TEST_PROD]           = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{0, 8, 8, 16, IPA_EE_AP } },
-	[IPA_4_0][IPA_CLIENT_TEST1_PROD]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{0, 8, 8, 16, IPA_EE_AP } },
-	[IPA_4_0_MHI][IPA_CLIENT_TEST2_PROD]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 1, 0, 8, 16, IPA_EE_AP } },
-	[IPA_4_0_MHI][IPA_CLIENT_TEST3_PROD]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 7, 9, 8, 16, IPA_EE_AP } },
-	[IPA_4_0_MHI][IPA_CLIENT_TEST4_PROD]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 8, 10, 8, 16, IPA_EE_AP } },
-	[IPA_4_0_MHI][IPA_CLIENT_APPS_LAN_CONS]       = {
-			true, IPA_v4_0_MHI_GROUP_DDR,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 10, 5, 9, 9, IPA_EE_AP } },
-	[IPA_4_0_MHI][IPA_CLIENT_APPS_WAN_CONS]       = {
-			true, IPA_v4_0_MHI_GROUP_DDR,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 11, 6, 9, 9, IPA_EE_AP } },
-	[IPA_4_0_MHI][IPA_CLIENT_MHI_CONS]            = {
-			true, IPA_v4_0_MHI_GROUP_PCIE,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_PCIE,
-			{ 17, 1, 17, 17, IPA_EE_AP } },
-	[IPA_4_0_MHI][IPA_CLIENT_Q6_LAN_CONS]         = {
-			true, IPA_v4_0_MHI_GROUP_DDR,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 14, 4, 9, 9, IPA_EE_Q6 } },
-	[IPA_4_0_MHI][IPA_CLIENT_Q6_WAN_CONS]         = {
-			true, IPA_v4_0_MHI_GROUP_DDR,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 13, 3, 9, 9, IPA_EE_Q6 } },
-	[IPA_4_0_MHI][IPA_CLIENT_MEMCPY_DMA_SYNC_CONS] = {
-			true, IPA_v4_0_MHI_GROUP_DMA,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_PCIE,
-			{ 20, 13, 9, 9, IPA_EE_AP } },
-	[IPA_4_0_MHI][IPA_CLIENT_MEMCPY_DMA_ASYNC_CONS] = {
-			true, IPA_v4_0_MHI_GROUP_DMA,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_PCIE,
-			{ 21, 14, 9, 9, IPA_EE_AP } },
-	[IPA_4_0_MHI][IPA_CLIENT_Q6_LTE_WIFI_AGGR_CONS] = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 16, 5, 9, 9, IPA_EE_Q6 } },
-	[IPA_4_0_MHI][IPA_CLIENT_USB_DPL_CONS]        = {
-			true, IPA_v4_0_MHI_GROUP_DDR,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 15, 7, 5, 5, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY } },
-	[IPA_4_0_MHI][IPA_CLIENT_MHI_DPL_CONS]        = {
-			true, IPA_v4_0_MHI_GROUP_PCIE,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_PCIE,
-			{ 12, 2, 5, 5, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY } },
-	/* Only for test purpose */
-	[IPA_4_0_MHI][IPA_CLIENT_TEST_CONS]           = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_PCIE,
-			{ 11, 6, 9, 9, IPA_EE_AP } },
-	[IPA_4_0_MHI][IPA_CLIENT_TEST1_CONS]           = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_PCIE,
-			{ 11, 6, 9, 9, IPA_EE_AP } },
-	[IPA_4_0_MHI][IPA_CLIENT_TEST2_CONS]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 12, 2, 5, 5, IPA_EE_AP } },
-	[IPA_4_0_MHI][IPA_CLIENT_TEST3_CONS]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_PCIE,
-			{ 19, 12, 9, 9, IPA_EE_AP } },
-	[IPA_4_0_MHI][IPA_CLIENT_TEST4_CONS]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_PCIE,
-			{ 21, 14, 9, 9, IPA_EE_AP } },
-	/* Dummy consumer (pipe 31) is used in L2TP rt rule */
-	[IPA_4_0_MHI][IPA_CLIENT_DUMMY_CONS]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 31, 31, 8, 8, IPA_EE_AP } },
-	/* IPA_4_1 */
-	[IPA_4_1][IPA_CLIENT_WLAN1_PROD]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_REP_SEQ_TYPE_2PKT_PROC_PASS_NO_DEC_UCP_DMAP,
-			QMB_MASTER_SELECT_DDR,
-			{ 6, 2, 8, 16, IPA_EE_UC } },
-	[IPA_4_1][IPA_CLIENT_WLAN2_PROD]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_REP_SEQ_TYPE_2PKT_PROC_PASS_NO_DEC_UCP_DMAP,
-			QMB_MASTER_SELECT_DDR,
-			{ 7, 9, 8, 16, IPA_EE_AP } },
-	[IPA_4_1][IPA_CLIENT_USB_PROD]            = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_REP_SEQ_TYPE_2PKT_PROC_PASS_NO_DEC_UCP_DMAP,
-			QMB_MASTER_SELECT_DDR,
-			{ 0, 8, 8, 16, IPA_EE_AP } },
-	[IPA_4_1][IPA_CLIENT_APPS_LAN_PROD]   = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 8, 10, 8, 16, IPA_EE_AP } },
-	[IPA_4_1][IPA_CLIENT_APPS_WAN_PROD] = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_REP_SEQ_TYPE_2PKT_PROC_PASS_NO_DEC_UCP_DMAP,
-			QMB_MASTER_SELECT_DDR,
-			{ 2, 3, 16, 32, IPA_EE_AP } },
-	[IPA_4_1][IPA_CLIENT_APPS_CMD_PROD]	  = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_DMA_ONLY,
-			QMB_MASTER_SELECT_DDR,
-			{ 5, 4, 20, 24, IPA_EE_AP } },
-	[IPA_4_1][IPA_CLIENT_ODU_PROD]            = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_REP_SEQ_TYPE_2PKT_PROC_PASS_NO_DEC_UCP_DMAP,
-			QMB_MASTER_SELECT_DDR,
-			{ 1, 0, 8, 16, IPA_EE_AP } },
-	[IPA_4_1][IPA_CLIENT_ETHERNET_PROD] = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_REP_SEQ_TYPE_2PKT_PROC_PASS_NO_DEC_UCP_DMAP,
-			QMB_MASTER_SELECT_DDR,
-			{ 9, 0, 8, 16, IPA_EE_UC } },
-	[IPA_4_1][IPA_CLIENT_Q6_WAN_PROD]         = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 3, 0, 16, 32, IPA_EE_Q6 } },
-	[IPA_4_1][IPA_CLIENT_Q6_CMD_PROD]	  = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 4, 1, 20, 24, IPA_EE_Q6 } },
-	/* Only for test purpose */
-	[IPA_4_1][IPA_CLIENT_TEST_PROD]           = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{0, 8, 8, 16, IPA_EE_AP } },
-	[IPA_4_1][IPA_CLIENT_TEST1_PROD]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 0, 8, 8, 16, IPA_EE_AP } },
-	[IPA_4_1][IPA_CLIENT_TEST2_PROD]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 1, 0, 8, 16, IPA_EE_AP } },
-	[IPA_4_1][IPA_CLIENT_TEST3_PROD]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{7, 9, 8, 16, IPA_EE_AP } },
-	[IPA_4_1][IPA_CLIENT_TEST4_PROD]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 8, 10, 8, 16, IPA_EE_AP } },
-
-
-	[IPA_4_1][IPA_CLIENT_WLAN1_CONS]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 18, 3, 9, 9, IPA_EE_UC } },
-	[IPA_4_1][IPA_CLIENT_WLAN2_CONS]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 17, 1, 8, 13, IPA_EE_AP } },
-	[IPA_4_1][IPA_CLIENT_WLAN3_CONS]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 21, 14, 9, 9, IPA_EE_AP } },
-	[IPA_4_1][IPA_CLIENT_USB_CONS]            = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 19, 12, 9, 9, IPA_EE_AP } },
-	[IPA_4_1][IPA_CLIENT_USB_DPL_CONS]        = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 15, 7, 5, 5, IPA_EE_AP } },
-	[IPA_4_1][IPA_CLIENT_APPS_LAN_CONS]       = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 10, 5, 9, 9, IPA_EE_AP } },
-	[IPA_4_1][IPA_CLIENT_APPS_WAN_CONS]       = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 11, 6, 9, 9, IPA_EE_AP } },
-	[IPA_4_1][IPA_CLIENT_ODL_DPL_CONS]        = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 12, 2, 9, 9, IPA_EE_AP } },
-	[IPA_4_1][IPA_CLIENT_ETHERNET_CONS]	  = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 22, 1, 9, 9, IPA_EE_UC } },
-	[IPA_4_1][IPA_CLIENT_Q6_LAN_CONS]         = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 14, 4, 9, 9, IPA_EE_Q6 } },
-	[IPA_4_1][IPA_CLIENT_Q6_WAN_CONS]         = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 13, 3, 9, 9, IPA_EE_Q6 } },
-	[IPA_4_1][IPA_CLIENT_Q6_LTE_WIFI_AGGR_CONS] = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 16, 5, 9, 9, IPA_EE_Q6 } },
-	/* Only for test purpose */
-	/* MBIM aggregation test pipes should have the same QMB as USB_CONS */
-	[IPA_4_1][IPA_CLIENT_TEST_CONS]           = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 11, 6, 9, 9, IPA_EE_AP } },
-	[IPA_4_1][IPA_CLIENT_TEST1_CONS]           = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 11, 6, 9, 9, IPA_EE_AP } },
-	[IPA_4_1][IPA_CLIENT_TEST2_CONS]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 12, 2, 9, 9, IPA_EE_AP } },
-	[IPA_4_1][IPA_CLIENT_TEST3_CONS]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 19, 12, 9, 9, IPA_EE_AP } },
-	[IPA_4_1][IPA_CLIENT_TEST4_CONS]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 21, 14, 9, 9, IPA_EE_AP } },
-	/* Dummy consumer (pipe 31) is used in L2TP rt rule */
-	[IPA_4_1][IPA_CLIENT_DUMMY_CONS]          = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 31, 31, 8, 8, IPA_EE_AP } },
-
-	/* MHI PRIME PIPES - Client producer / IPA Consumer pipes */
-	[IPA_4_1_APQ][IPA_CLIENT_MHI_PRIME_DPL_PROD] = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_DMA_ONLY,
-			QMB_MASTER_SELECT_DDR,
-			{7, 9, 8, 16, IPA_EE_AP } },
-	[IPA_4_1_APQ][IPA_CLIENT_MHI_PRIME_TETH_PROD] = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 1, 0, 8, 16, IPA_EE_AP } },
-	[IPA_4_1_APQ][IPA_CLIENT_MHI_PRIME_RMNET_PROD] = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_DMA_ONLY,
-			QMB_MASTER_SELECT_DDR,
-			{ 2, 3, 16, 32, IPA_EE_AP } },
-	/* MHI PRIME PIPES - Client Consumer / IPA Producer pipes */
-	[IPA_4_1_APQ][IPA_CLIENT_MHI_PRIME_TETH_CONS] = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 20, 13, 9, 9, IPA_EE_AP } },
-	[IPA_4_1_APQ][IPA_CLIENT_MHI_PRIME_RMNET_CONS] = {
-			true, IPA_v4_0_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 17, 14, 9, 9, IPA_EE_AP } },
-
-	/* IPA_4_2 */
-	[IPA_4_2][IPA_CLIENT_WLAN1_PROD]          = {
-			true, IPA_v4_2_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_REP_SEQ_TYPE_PKT_PROCESS_NO_DEC_NO_UCP_DMAP,
-			QMB_MASTER_SELECT_DDR,
-			{ 3, 7, 6, 7, IPA_EE_AP, GSI_USE_PREFETCH_BUFS} },
-	[IPA_4_2][IPA_CLIENT_USB_PROD]            = {
-			true, IPA_v4_2_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_REP_SEQ_TYPE_PKT_PROCESS_NO_DEC_NO_UCP_DMAP,
-			QMB_MASTER_SELECT_DDR,
-			{ 0, 5, 8, 9, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY} },
-	[IPA_4_2][IPA_CLIENT_APPS_LAN_PROD]   = {
-			true, IPA_v4_2_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_NO_DEC_NO_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 2, 6, 8, 9, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY} },
-	[IPA_4_2][IPA_CLIENT_APPS_WAN_PROD] = {
-			true, IPA_v4_2_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_REP_SEQ_TYPE_PKT_PROCESS_NO_DEC_NO_UCP_DMAP,
-			QMB_MASTER_SELECT_DDR,
-			{ 1, 0, 8, 12, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY} },
-	[IPA_4_2][IPA_CLIENT_APPS_CMD_PROD]	  = {
-			true, IPA_v4_2_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_DMA_ONLY,
-			QMB_MASTER_SELECT_DDR,
-			{ 6, 1, 20, 20, IPA_EE_AP, GSI_USE_PREFETCH_BUFS} },
-	[IPA_4_2][IPA_CLIENT_Q6_WAN_PROD]         = {
-			true, IPA_v4_2_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_NO_DEC_NO_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 4, 0, 8, 12, IPA_EE_Q6, GSI_USE_PREFETCH_BUFS} },
-	[IPA_4_2][IPA_CLIENT_Q6_CMD_PROD]	  = {
-			true, IPA_v4_2_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_NO_DEC_NO_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 5, 1, 20, 20, IPA_EE_Q6, GSI_USE_PREFETCH_BUFS} },
-	[IPA_4_2][IPA_CLIENT_ETHERNET_PROD] = {
-			true, IPA_v4_2_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_NO_DEC_NO_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 7, 0, 8, 10, IPA_EE_UC, GSI_USE_PREFETCH_BUFS} },
-	/* Only for test purpose */
-	[IPA_4_2][IPA_CLIENT_TEST_PROD]           = {
-			true, IPA_v4_2_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_NO_DEC_NO_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{0, 5, 8, 9, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY} },
-	[IPA_4_2][IPA_CLIENT_TEST1_PROD]          = {
-			true, IPA_v4_2_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_NO_DEC_NO_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 0, 5, 8, 9, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY} },
-	[IPA_4_2][IPA_CLIENT_TEST2_PROD]          = {
-			true, IPA_v4_2_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_NO_DEC_NO_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 3, 7, 6, 7, IPA_EE_AP, GSI_USE_PREFETCH_BUFS} },
-	[IPA_4_2][IPA_CLIENT_TEST3_PROD]          = {
-			true, IPA_v4_2_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_NO_DEC_NO_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{1, 0, 8, 12, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY} },
-	[IPA_4_2][IPA_CLIENT_TEST4_PROD]          = {
-			true, IPA_v4_2_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_NO_DEC_NO_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 7, 0, 8, 10, IPA_EE_AP, GSI_USE_PREFETCH_BUFS} },
-
-
-	[IPA_4_2][IPA_CLIENT_WLAN1_CONS]          = {
-			true, IPA_v4_2_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 14, 8, 6, 9, IPA_EE_AP, GSI_USE_PREFETCH_BUFS} },
-	[IPA_4_2][IPA_CLIENT_USB_CONS]            = {
-			true, IPA_v4_2_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 15, 9, 6, 6, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY} },
-	[IPA_4_2][IPA_CLIENT_USB_DPL_CONS]        = {
-			true, IPA_v4_2_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 12, 4, 4, 4, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY} },
-	[IPA_4_2][IPA_CLIENT_APPS_LAN_CONS]       = {
-			true, IPA_v4_2_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 8, 2, 6, 6, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY} },
-	[IPA_4_2][IPA_CLIENT_APPS_WAN_CONS]       = {
-			true, IPA_v4_2_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 9, 3, 6, 6, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY} },
-	[IPA_4_2][IPA_CLIENT_Q6_LAN_CONS]         = {
-			true, IPA_v4_2_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 11, 3, 6, 6, IPA_EE_Q6, GSI_ESCAPE_BUF_ONLY} },
-	[IPA_4_2][IPA_CLIENT_Q6_WAN_CONS]         = {
-			true, IPA_v4_2_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 10, 2, 6, 6, IPA_EE_Q6,  GSI_ESCAPE_BUF_ONLY} },
-	[IPA_4_2][IPA_CLIENT_Q6_LTE_WIFI_AGGR_CONS] = {
-			true, IPA_v4_2_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 13, 4, 6, 6, IPA_EE_Q6, GSI_ESCAPE_BUF_ONLY} },
-	[IPA_4_2][IPA_CLIENT_ETHERNET_CONS] = {
-			true, IPA_v4_2_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 16, 1, 6, 6, IPA_EE_UC, GSI_USE_PREFETCH_BUFS} },
-	/* Only for test purpose */
-	/* MBIM aggregation test pipes should have the same QMB as USB_CONS */
-	[IPA_4_2][IPA_CLIENT_TEST_CONS]           = {
-			true, IPA_v4_2_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 15, 9, 6, 6, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY} },
-	[IPA_4_2][IPA_CLIENT_TEST1_CONS]           = {
-			true, IPA_v4_2_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 15, 9, 6, 6, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY} },
-	[IPA_4_2][IPA_CLIENT_TEST2_CONS]          = {
-			true, IPA_v4_2_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 12, 4, 4, 4, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY} },
-	[IPA_4_2][IPA_CLIENT_TEST3_CONS]          = {
-			true, IPA_v4_2_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 14, 8, 6, 9, IPA_EE_AP, GSI_USE_PREFETCH_BUFS} },
-	[IPA_4_2][IPA_CLIENT_TEST4_CONS]          = {
-			true, IPA_v4_2_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 9, 3, 6, 6, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY} },
-	/* Dummy consumer (pipe 31) is used in L2TP rt rule */
-	[IPA_4_2][IPA_CLIENT_DUMMY_CONS]          = {
-			true, IPA_v4_2_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 31, 31, 8, 8, IPA_EE_AP, GSI_USE_PREFETCH_BUFS} },
-
-	/* IPA_4_5 */
-	[IPA_4_5][IPA_CLIENT_WLAN2_PROD]          = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 9, 12, 8, 16, IPA_EE_AP, GSI_FREE_PRE_FETCH, 2 } },
-	[IPA_4_5][IPA_CLIENT_USB_PROD]            = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 1, 0, 8, 16, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_5][IPA_CLIENT_APPS_LAN_PROD]	  = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 11, 14, 10, 16, IPA_EE_AP, GSI_SMART_PRE_FETCH, 2 } },
-	[IPA_4_5][IPA_CLIENT_APPS_WAN_PROD]	  = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 2, 7, 16, 32, IPA_EE_AP, GSI_SMART_PRE_FETCH, 7 } },
-	[IPA_4_5][IPA_CLIENT_APPS_CMD_PROD]	  = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_DMA_ONLY,
-			QMB_MASTER_SELECT_DDR,
-			{ 7, 9, 20, 24, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_5][IPA_CLIENT_ODU_PROD]            = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 3, 5, 8, 16, IPA_EE_AP, GSI_SMART_PRE_FETCH, 3 } },
-	[IPA_4_5][IPA_CLIENT_ETHERNET_PROD]	  = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 12, 0, 8, 16, IPA_EE_UC, GSI_SMART_PRE_FETCH, 3 } },
-	[IPA_4_5][IPA_CLIENT_Q6_WAN_PROD]         = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 5, 0, 16, 28, IPA_EE_Q6, GSI_SMART_PRE_FETCH, 2 } },
-	[IPA_4_5][IPA_CLIENT_Q6_CMD_PROD]	  = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 6, 1, 20, 24, IPA_EE_Q6, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_5][IPA_CLIENT_Q6_DL_NLO_DATA_PROD] = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 8, 2, 27, 32, IPA_EE_Q6, GSI_FREE_PRE_FETCH, 3 } },
-	/* Only for test purpose */
-	[IPA_4_5][IPA_CLIENT_TEST_PROD]           = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 1, 0, 8, 16, IPA_EE_AP } },
-	[IPA_4_5][IPA_CLIENT_TEST1_PROD]          = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 1, 0, 8, 16, IPA_EE_AP } },
-	[IPA_4_5][IPA_CLIENT_TEST2_PROD]          = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 3, 5, 8, 16, IPA_EE_AP } },
-	[IPA_4_5][IPA_CLIENT_TEST3_PROD]          = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 9, 12, 8, 16, IPA_EE_AP } },
-	[IPA_4_5][IPA_CLIENT_TEST4_PROD]          = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 11, 14, 8, 16, IPA_EE_AP } },
-
-	[IPA_4_5][IPA_CLIENT_WLAN2_CONS]          = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 24, 3, 8, 14, IPA_EE_AP, GSI_SMART_PRE_FETCH, 3 } },
-	[IPA_4_5][IPA_CLIENT_USB_CONS]            = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 26, 17, 9, 9, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_5][IPA_CLIENT_USB_DPL_CONS]        = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 15, 15, 5, 5, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_5][IPA_CLIENT_ODL_DPL_CONS]        = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 22, 2, 5, 5, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_5][IPA_CLIENT_APPS_LAN_CONS]       = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 16, 10, 9, 9, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_5][IPA_CLIENT_APPS_WAN_COAL_CONS]       = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 13, 4, 8, 11, IPA_EE_AP, GSI_SMART_PRE_FETCH, 3 } },
-	[IPA_4_5][IPA_CLIENT_APPS_WAN_CONS]       = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 14, 1, 9, 9, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_5][IPA_CLIENT_ODU_EMB_CONS]        = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 30, 6, 9, 9, IPA_EE_AP, GSI_SMART_PRE_FETCH, 4 } },
-	[IPA_4_5][IPA_CLIENT_ETHERNET_CONS]	  = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 28, 1, 9, 9, IPA_EE_UC, GSI_SMART_PRE_FETCH, 4 } },
-	[IPA_4_5][IPA_CLIENT_Q6_LAN_CONS]         = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 17, 3, 9, 9, IPA_EE_Q6, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_5][IPA_CLIENT_Q6_WAN_CONS]         = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 21, 7, 9, 9, IPA_EE_Q6, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_5][IPA_CLIENT_Q6_UL_NLO_DATA_CONS] = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 19, 5, 5, 5, IPA_EE_Q6, GSI_SMART_PRE_FETCH, 2 } },
-	[IPA_4_5][IPA_CLIENT_Q6_UL_NLO_ACK_CONS]  = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 20, 6, 5, 5, IPA_EE_Q6, GSI_SMART_PRE_FETCH, 2 } },
-	[IPA_4_5][IPA_CLIENT_Q6_QBAP_STATUS_CONS] = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 18, 4, 9, 9, IPA_EE_Q6, GSI_ESCAPE_BUF_ONLY, 0 } },
-	/* Only for test purpose */
-	/* MBIM aggregation test pipes should have the same QMB as USB_CONS */
-	[IPA_4_5][IPA_CLIENT_TEST_CONS]           = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 14, 1, 9, 9, IPA_EE_AP } },
-	[IPA_4_5][IPA_CLIENT_TEST1_CONS]          = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 14, 1, 9, 9, IPA_EE_AP } },
-	[IPA_4_5][IPA_CLIENT_TEST2_CONS]          = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 24, 3, 8, 14, IPA_EE_AP } },
-	[IPA_4_5][IPA_CLIENT_TEST3_CONS]          = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 26, 17, 9, 9, IPA_EE_AP } },
-	[IPA_4_5][IPA_CLIENT_TEST4_CONS]          = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 27, 18, 9, 9, IPA_EE_AP } },
-	/* Dummy consumer (pipe 31) is used in L2TP rt rule */
-	[IPA_4_5][IPA_CLIENT_DUMMY_CONS]          = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 31, 31, 8, 8, IPA_EE_AP } },
-
-	/* IPA_4_5_MHI */
-	[IPA_4_5_MHI][IPA_CLIENT_APPS_CMD_PROD]		= {
-			true, IPA_v4_5_MHI_GROUP_DDR,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_DMA_ONLY,
-			QMB_MASTER_SELECT_DDR,
-			{ 7, 9, 20, 24, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_5_MHI][IPA_CLIENT_Q6_WAN_PROD]		= {
-			true, IPA_v4_5_MHI_GROUP_DDR,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 5, 0, 16, 28, IPA_EE_Q6, GSI_SMART_PRE_FETCH, 2 } },
-	[IPA_4_5_MHI][IPA_CLIENT_Q6_CMD_PROD]		= {
-			true, IPA_v4_5_MHI_GROUP_PCIE,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 6, 1, 20, 24, IPA_EE_Q6, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_5_MHI][IPA_CLIENT_Q6_DL_NLO_DATA_PROD]	= {
-			true, IPA_v4_5_MHI_GROUP_DDR,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 8, 2, 27, 32, IPA_EE_Q6, GSI_FREE_PRE_FETCH, 3 } },
-	[IPA_4_5_MHI][IPA_CLIENT_Q6_AUDIO_DMA_MHI_PROD]	= {
-			true, IPA_v4_5_MHI_GROUP_DMA,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_DMA_ONLY,
-			QMB_MASTER_SELECT_DDR,
-			{ 4, 8, 8, 16, IPA_EE_Q6, GSI_SMART_PRE_FETCH, 3 } },
-	[IPA_4_5_MHI][IPA_CLIENT_MHI_PROD]		= {
-			true, IPA_v4_5_MHI_GROUP_PCIE,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_PCIE,
-			{ 1, 0, 16, 20, IPA_EE_AP, GSI_SMART_PRE_FETCH, 7 } },
-	[IPA_4_5_MHI][IPA_CLIENT_MEMCPY_DMA_SYNC_PROD]	= {
-			true, IPA_v4_5_MHI_GROUP_DMA,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_DMA_ONLY,
-			QMB_MASTER_SELECT_DDR,
-			{ 9, 12, 8, 16, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_5_MHI][IPA_CLIENT_MEMCPY_DMA_ASYNC_PROD]	= {
-			true, IPA_v4_5_MHI_GROUP_DMA,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_DMA_ONLY,
-			QMB_MASTER_SELECT_DDR,
-			{ 10, 13, 8, 16, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY, 0 } },
-	/* Only for test purpose */
-	[IPA_4_5_MHI][IPA_CLIENT_TEST_PROD]           = {
-			true, QMB_MASTER_SELECT_DDR,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 1, 0, 8, 16, IPA_EE_AP } },
-
-	[IPA_4_5_MHI][IPA_CLIENT_APPS_LAN_CONS]		= {
-			true, IPA_v4_5_MHI_GROUP_DDR,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 16, 10, 9, 9, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_5][IPA_CLIENT_USB_DPL_CONS]        = {
-			true, IPA_v4_5_MHI_GROUP_DDR,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 15, 15, 5, 5, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_5_MHI][IPA_CLIENT_Q6_LAN_CONS]		= {
-			true, IPA_v4_5_MHI_GROUP_DDR,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 17, 3, 9, 9, IPA_EE_Q6, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_5_MHI][IPA_CLIENT_Q6_WAN_CONS]		= {
-			true, IPA_v4_5_MHI_GROUP_DDR,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 21, 7, 9, 9, IPA_EE_Q6, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_5_MHI][IPA_CLIENT_Q6_UL_NLO_DATA_CONS]	= {
-			true, IPA_v4_5_MHI_GROUP_DDR,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 19, 5, 5, 5, IPA_EE_Q6, GSI_SMART_PRE_FETCH, 2 } },
-	[IPA_4_5_MHI][IPA_CLIENT_Q6_UL_NLO_ACK_CONS]	= {
-			true, IPA_v4_5_MHI_GROUP_DDR,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 20, 6, 5, 5, IPA_EE_Q6, GSI_SMART_PRE_FETCH, 2 } },
-	[IPA_4_5_MHI][IPA_CLIENT_Q6_QBAP_STATUS_CONS]	= {
-			true, IPA_v4_5_MHI_GROUP_DDR,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 18, 4, 9, 9, IPA_EE_Q6, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_5_MHI][IPA_CLIENT_Q6_AUDIO_DMA_MHI_CONS]	= {
-			true, IPA_v4_5_MHI_GROUP_DMA,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_PCIE,
-			{ 29, 9, 9, 9, IPA_EE_Q6, GSI_SMART_PRE_FETCH, 4 } },
-	[IPA_4_5_MHI][IPA_CLIENT_MEMCPY_DMA_SYNC_CONS]	= {
-			true, IPA_v4_5_MHI_GROUP_DMA,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_PCIE,
-			{ 26, 17, 9, 9, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_5_MHI][IPA_CLIENT_MEMCPY_DMA_ASYNC_CONS]	= {
-			true, IPA_v4_5_MHI_GROUP_DMA,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_PCIE,
-			{ 27, 18, 9, 9, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_5_MHI][IPA_CLIENT_MHI_CONS]		= {
-			true, IPA_v4_5_MHI_GROUP_PCIE,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_PCIE,
-			{ 14, 1, 9, 9, IPA_EE_AP, GSI_SMART_PRE_FETCH, 4 } },
-	[IPA_4_5_MHI][IPA_CLIENT_MHI_DPL_CONS]		= {
-			true, IPA_v4_5_MHI_GROUP_PCIE,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_PCIE,
-			{ 22, 2, 5, 5, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY, 0 } },
-
-	/* Dummy consumer (pipe 31) is used in L2TP rt rule */
-	[IPA_4_5_MHI][IPA_CLIENT_DUMMY_CONS]          = {
-			true, QMB_MASTER_SELECT_DDR,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 31, 31, 8, 8, IPA_EE_AP } },
-
-	/* IPA_4_5 APQ */
-	[IPA_4_5_APQ][IPA_CLIENT_WLAN2_PROD]          = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 9, 3, 8, 16, IPA_EE_AP, GSI_FREE_PRE_FETCH, 2 } },
-	[IPA_4_5_APQ][IPA_CLIENT_WIGIG_PROD]          = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 1, 1, 8, 16, IPA_EE_AP, GSI_SMART_PRE_FETCH, 3 } },
-	[IPA_4_5_APQ][IPA_CLIENT_USB_PROD]            = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 0, 0, 8, 16, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_5_APQ][IPA_CLIENT_APPS_LAN_PROD]	  = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 11, 4, 8, 16, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_5_APQ][IPA_CLIENT_APPS_CMD_PROD]	  = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_DMA_ONLY,
-			QMB_MASTER_SELECT_DDR,
-			{ 7, 12, 20, 24, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY, 0 } },
-	/* MHI PRIME PIPES - Client producer / IPA Consumer pipes */
-	[IPA_4_5_APQ][IPA_CLIENT_MHI_PRIME_DPL_PROD] = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_DMA_ONLY,
-			QMB_MASTER_SELECT_DDR,
-			{3, 2, 8, 16, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_5_APQ][IPA_CLIENT_MHI_PRIME_TETH_PROD] = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 2, 7, 8, 16, IPA_EE_AP, GSI_SMART_PRE_FETCH, 3 } },
-	[IPA_4_5_APQ][IPA_CLIENT_MHI_PRIME_RMNET_PROD] = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_DMA_ONLY,
-			QMB_MASTER_SELECT_DDR,
-			{ 4, 11, 16, 32, IPA_EE_AP, GSI_SMART_PRE_FETCH, 7 } },
-	/* Only for test purpose */
-	[IPA_4_5_APQ][IPA_CLIENT_TEST_PROD]           = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 0, 0, 8, 16, IPA_EE_AP } },
-	[IPA_4_5_APQ][IPA_CLIENT_TEST1_PROD]          = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 0, 0, 8, 16, IPA_EE_AP } },
-	[IPA_4_5_APQ][IPA_CLIENT_TEST2_PROD]          = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 1, 1, 8, 16, IPA_EE_AP } },
-	[IPA_4_5_APQ][IPA_CLIENT_TEST3_PROD]          = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 9, 3, 8, 16, IPA_EE_AP } },
-	[IPA_4_5_APQ][IPA_CLIENT_TEST4_PROD]          = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 10, 10, 8, 16, IPA_EE_AP } },
-
-	[IPA_4_5_APQ][IPA_CLIENT_WLAN2_CONS]          = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 23, 8, 8, 14, IPA_EE_AP, GSI_SMART_PRE_FETCH, 3 } },
-	[IPA_4_5_APQ][IPA_CLIENT_WIGIG1_CONS]          = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 14, 14, 8, 14, IPA_EE_AP, GSI_SMART_PRE_FETCH, 3 } },
-	[IPA_4_5_APQ][IPA_CLIENT_WIGIG2_CONS]          = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 20, 18, 8, 14, IPA_EE_AP, GSI_SMART_PRE_FETCH, 3 } },
-	[IPA_4_5_APQ][IPA_CLIENT_WIGIG3_CONS]          = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 22, 5, 8, 14, IPA_EE_AP, GSI_SMART_PRE_FETCH, 3 } },
-	[IPA_4_5_APQ][IPA_CLIENT_WIGIG4_CONS]          = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 29, 10, 8, 14, IPA_EE_AP, GSI_SMART_PRE_FETCH, 3 } },
-	[IPA_4_5_APQ][IPA_CLIENT_USB_CONS]            = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 24, 9, 9, 9, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_5_APQ][IPA_CLIENT_USB_DPL_CONS]        = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 16, 16, 5, 5, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_5_APQ][IPA_CLIENT_APPS_LAN_CONS]       = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 13, 13, 9, 9, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_5_APQ][IPA_CLIENT_ODL_DPL_CONS]       = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 21, 19, 9, 9, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY, 0 } },
-	/* MHI PRIME PIPES - Client Consumer / IPA Producer pipes */
-	[IPA_4_5_APQ][IPA_CLIENT_MHI_PRIME_TETH_CONS] = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 28, 6, 8, 9, IPA_EE_AP, GSI_SMART_PRE_FETCH, 3 } },
-	[IPA_4_5_APQ][IPA_CLIENT_MHI_PRIME_RMNET_CONS] = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 17, 17, 8, 9, IPA_EE_AP, GSI_SMART_PRE_FETCH, 4 } },
-	/* Only for test purpose */
-	/* MBIM aggregation test pipes should have the same QMB as USB_CONS */
-	[IPA_4_5_APQ][IPA_CLIENT_TEST_CONS]           = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 16, 16, 5, 5, IPA_EE_AP } },
-	[IPA_4_5_APQ][IPA_CLIENT_TEST1_CONS]          = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 16, 16, 5, 5, IPA_EE_AP } },
-	[IPA_4_5_APQ][IPA_CLIENT_TEST2_CONS]          = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 22, 5, 9, 9, IPA_EE_AP } },
-	[IPA_4_5_APQ][IPA_CLIENT_TEST3_CONS]          = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 24, 9, 9, 9, IPA_EE_AP } },
-	[IPA_4_5_APQ][IPA_CLIENT_TEST4_CONS]          = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 23, 8, 8, 13, IPA_EE_AP } },
-	/* Dummy consumer (pipe 31) is used in L2TP rt rule */
-	[IPA_4_5_APQ][IPA_CLIENT_DUMMY_CONS]          = {
-			true, IPA_v4_5_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 31, 31, 8, 8, IPA_EE_AP } },
-
-	/* IPA_4_7 */
-	[IPA_4_7][IPA_CLIENT_WLAN1_PROD]          = {
-			true, IPA_v4_7_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 3, 3, 8, 16, IPA_EE_AP, GSI_SMART_PRE_FETCH, 3 } },
-	[IPA_4_7][IPA_CLIENT_USB_PROD]            = {
-			true, IPA_v4_7_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 0, 0, 8, 16, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_7][IPA_CLIENT_APPS_LAN_PROD]	  = {
-			true, IPA_v4_7_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 4, 4, 8, 16, IPA_EE_AP, GSI_SMART_PRE_FETCH, 3 } },
-	[IPA_4_7][IPA_CLIENT_APPS_WAN_PROD]	  = {
-			true, IPA_v4_7_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 2, 2, 16, 32, IPA_EE_AP, GSI_SMART_PRE_FETCH, 7 } },
-	[IPA_4_7][IPA_CLIENT_APPS_CMD_PROD]	  = {
-			true, IPA_v4_7_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_DMA_ONLY,
-			QMB_MASTER_SELECT_DDR,
-			{ 7, 5, 20, 24, IPA_EE_AP, GSI_SMART_PRE_FETCH, 8 } },
-	[IPA_4_7][IPA_CLIENT_Q6_WAN_PROD]         = {
-			true, IPA_v4_7_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 5, 0, 16, 28, IPA_EE_Q6, GSI_SMART_PRE_FETCH, 2 } },
-	[IPA_4_7][IPA_CLIENT_Q6_CMD_PROD]	  = {
-			true, IPA_v4_7_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 6, 1, 20, 24, IPA_EE_Q6, GSI_SMART_PRE_FETCH, 8 } },
-	[IPA_4_7][IPA_CLIENT_Q6_DL_NLO_DATA_PROD] = {
-			true, IPA_v4_7_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 8, 2, 27, 32, IPA_EE_Q6, GSI_FREE_PRE_FETCH, 3 } },
-	/* Only for test purpose */
-	[IPA_4_7][IPA_CLIENT_TEST_PROD]           = {
-			true, IPA_v4_7_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 0, 0, 8, 16, IPA_EE_AP } },
-	[IPA_4_7][IPA_CLIENT_TEST1_PROD]          = {
-			true, IPA_v4_7_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 0, 0, 8, 16, IPA_EE_AP } },
-	[IPA_4_7][IPA_CLIENT_TEST2_PROD]          = {
-			true, IPA_v4_7_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 1, 1, 8, 16, IPA_EE_AP } },
-	[IPA_4_7][IPA_CLIENT_TEST3_PROD]          = {
-			true, IPA_v4_7_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 2, 2, 16, 32, IPA_EE_AP } },
-	[IPA_4_7][IPA_CLIENT_TEST4_PROD]          = {
-			true, IPA_v4_7_GROUP_UL_DL,
-			true,
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP,
-			QMB_MASTER_SELECT_DDR,
-			{ 1, 1, 8, 16, IPA_EE_AP } },
-
-	[IPA_4_7][IPA_CLIENT_WLAN1_CONS]          = {
-			true, IPA_v4_7_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 18, 9, 8, 13, IPA_EE_AP, GSI_SMART_PRE_FETCH, 3 } },
-	[IPA_4_7][IPA_CLIENT_USB_CONS]            = {
-			true, IPA_v4_7_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 19, 10, 9, 9, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_7][IPA_CLIENT_USB_DPL_CONS]        = {
-			true, IPA_v4_7_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 17, 8, 5, 5, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_7][IPA_CLIENT_ODL_DPL_CONS]        = {
-			true, IPA_v4_7_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 22, 13, 9, 9, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_7][IPA_CLIENT_APPS_LAN_CONS]       = {
-			true, IPA_v4_7_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 9, 14, 9, 9, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_7][IPA_CLIENT_APPS_WAN_CONS]       = {
-			true, IPA_v4_7_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 16, 7, 9, 9, IPA_EE_AP, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_7][IPA_CLIENT_APPS_WAN_COAL_CONS]       = {
-			true, IPA_v4_7_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 15, 6, 8, 11, IPA_EE_AP, GSI_SMART_PRE_FETCH, 3 } },
-	[IPA_4_7][IPA_CLIENT_Q6_LAN_CONS]         = {
-			true, IPA_v4_7_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 10, 3, 9, 9, IPA_EE_Q6, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_7][IPA_CLIENT_Q6_WAN_CONS]         = {
-			true, IPA_v4_7_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 14, 7, 9, 9, IPA_EE_Q6, GSI_ESCAPE_BUF_ONLY, 0 } },
-	[IPA_4_7][IPA_CLIENT_Q6_UL_NLO_DATA_CONS] = {
-			true, IPA_v4_7_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 12, 5, 5, 5, IPA_EE_Q6, GSI_SMART_PRE_FETCH, 2 } },
-	[IPA_4_7][IPA_CLIENT_Q6_UL_NLO_ACK_CONS]  = {
-			true, IPA_v4_7_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 13, 6, 5, 5, IPA_EE_Q6, GSI_SMART_PRE_FETCH, 2 } },
-	[IPA_4_7][IPA_CLIENT_Q6_QBAP_STATUS_CONS] = {
-			true, IPA_v4_7_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 11, 4, 9, 9, IPA_EE_Q6, GSI_ESCAPE_BUF_ONLY, 0 } },
-	/* Only for test purpose */
-	/* MBIM aggregation test pipes should have the same QMB as USB_CONS */
-	[IPA_4_7][IPA_CLIENT_TEST_CONS]           = {
-			true, IPA_v4_7_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 16, 7, 9, 9, IPA_EE_AP } },
-	[IPA_4_7][IPA_CLIENT_TEST1_CONS]          = {
-			true, IPA_v4_7_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 16, 7, 9, 9, IPA_EE_AP } },
-	[IPA_4_7][IPA_CLIENT_TEST2_CONS]          = {
-			true, IPA_v4_7_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 21, 12, 9, 9, IPA_EE_AP } },
-	[IPA_4_7][IPA_CLIENT_TEST3_CONS]          = {
-			true, IPA_v4_7_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 19, 10, 9, 9, IPA_EE_AP } },
-	[IPA_4_7][IPA_CLIENT_TEST4_CONS]          = {
-			true, IPA_v4_7_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 20, 11, 9, 9, IPA_EE_AP } },
-	/* Dummy consumer (pipe 31) is used in L2TP rt rule */
-	[IPA_4_7][IPA_CLIENT_DUMMY_CONS]          = {
-			true, IPA_v4_7_GROUP_UL_DL,
-			false,
-			IPA_DPS_HPS_SEQ_TYPE_INVALID,
-			QMB_MASTER_SELECT_DDR,
-			{ 31, 31, 8, 8, IPA_EE_AP } },
-
-};
-
-static struct ipa3_mem_partition ipa_4_1_mem_part = {
-	.ofst_start				= 0x280,
-	.v4_flt_hash_ofst		= 0x288,
-	.v4_flt_hash_size		=  0x78,
-	.v4_flt_hash_size_ddr		= 0x4000,
-	.v4_flt_nhash_ofst		= 0x308,
-	.v4_flt_nhash_size		= 0x78,
-	.v4_flt_nhash_size_ddr		= 0x4000,
-	.v6_flt_hash_ofst		= 0x388,
-	.v6_flt_hash_size		= 0x78,
-	.v6_flt_hash_size_ddr		= 0x4000,
-	.v6_flt_nhash_ofst		= 0x408,
-	.v6_flt_nhash_size		= 0x78,
-	.v6_flt_nhash_size_ddr		= 0x4000,
-	.v4_rt_num_index		= 0xf,
-	.v4_modem_rt_index_lo		= 0x0,
-	.v4_modem_rt_index_hi		= 0x7,
-	.v4_apps_rt_index_lo		= 0x8,
-	.v4_apps_rt_index_hi		= 0xe,
-	.v4_rt_hash_ofst		= 0x488,
-	.v4_rt_hash_size		= 0x78,
-	.v4_rt_hash_size_ddr		= 0x4000,
-	.v4_rt_nhash_ofst		= 0x508,
-	.v4_rt_nhash_size		= 0x78,
-	.v4_rt_nhash_size_ddr		= 0x4000,
-	.v6_rt_num_index		= 0xf,
-	.v6_modem_rt_index_lo		= 0x0,
-	.v6_modem_rt_index_hi		= 0x7,
-	.v6_apps_rt_index_lo		= 0x8,
-	.v6_apps_rt_index_hi		= 0xe,
-	.v6_rt_hash_ofst		= 0x588,
-	.v6_rt_hash_size		= 0x78,
-	.v6_rt_hash_size_ddr		= 0x4000,
-	.v6_rt_nhash_ofst		= 0x608,
-	.v6_rt_nhash_size		= 0x78,
-	.v6_rt_nhash_size_ddr		= 0x4000,
-	.modem_hdr_ofst			= 0x688,
-	.modem_hdr_size			= 0x140,
-	.apps_hdr_ofst			= 0x7c8,
-	.apps_hdr_size			= 0x0,
-	.apps_hdr_size_ddr		= 0x800,
-	.modem_hdr_proc_ctx_ofst	= 0x7d0,
-	.modem_hdr_proc_ctx_size	= 0x200,
-	.apps_hdr_proc_ctx_ofst		= 0x9d0,
-	.apps_hdr_proc_ctx_size		= 0x200,
-	.apps_hdr_proc_ctx_size_ddr	= 0x0,
-	.modem_comp_decomp_ofst		= 0x0,
-	.modem_comp_decomp_size		= 0x0,
-	.modem_ofst			= 0x13f0,
-	.modem_size			= 0x100c,
-	.apps_v4_flt_hash_ofst		= 0x23fc,
-	.apps_v4_flt_hash_size		= 0x0,
-	.apps_v4_flt_nhash_ofst		= 0x23fc,
-	.apps_v4_flt_nhash_size		= 0x0,
-	.apps_v6_flt_hash_ofst		= 0x23fc,
-	.apps_v6_flt_hash_size		= 0x0,
-	.apps_v6_flt_nhash_ofst		= 0x23fc,
-	.apps_v6_flt_nhash_size		= 0x0,
-	.uc_info_ofst			= 0x80,
-	.uc_info_size			= 0x200,
-	.end_ofst			= 0x2800,
-	.apps_v4_rt_hash_ofst		= 0x23fc,
-	.apps_v4_rt_hash_size		= 0x0,
-	.apps_v4_rt_nhash_ofst		= 0x23fc,
-	.apps_v4_rt_nhash_size		= 0x0,
-	.apps_v6_rt_hash_ofst		= 0x23fc,
-	.apps_v6_rt_hash_size		= 0x0,
-	.apps_v6_rt_nhash_ofst		= 0x23fc,
-	.apps_v6_rt_nhash_size		= 0x0,
-	.uc_descriptor_ram_ofst		= 0x2400,
-	.uc_descriptor_ram_size		= 0x400,
-	.pdn_config_ofst		= 0xbd8,
-	.pdn_config_size		= 0x50,
-	.stats_quota_ofst		= 0xc30,
-	.stats_quota_size		= 0x60,
-	.stats_tethering_ofst		= 0xc90,
-	.stats_tethering_size		= 0x140,
-	.stats_flt_v4_ofst		= 0xdd0,
-	.stats_flt_v4_size		= 0x180,
-	.stats_flt_v6_ofst		= 0xf50,
-	.stats_flt_v6_size		= 0x180,
-	.stats_rt_v4_ofst		= 0x10d0,
-	.stats_rt_v4_size		= 0x180,
-	.stats_rt_v6_ofst		= 0x1250,
-	.stats_rt_v6_size		= 0x180,
-	.stats_drop_ofst		= 0x13d0,
-	.stats_drop_size		= 0x20,
-};
-
-static struct ipa3_mem_partition ipa_4_2_mem_part = {
-	.ofst_start				= 0x280,
-	.v4_flt_hash_ofst		= 0x288,
-	.v4_flt_hash_size		= 0x0,
-	.v4_flt_hash_size_ddr		= 0x0,
-	.v4_flt_nhash_ofst		= 0x290,
-	.v4_flt_nhash_size		= 0x78,
-	.v4_flt_nhash_size_ddr		= 0x4000,
-	.v6_flt_hash_ofst		= 0x310,
-	.v6_flt_hash_size		= 0x0,
-	.v6_flt_hash_size_ddr		= 0x0,
-	.v6_flt_nhash_ofst		= 0x318,
-	.v6_flt_nhash_size		= 0x78,
-	.v6_flt_nhash_size_ddr		= 0x4000,
-	.v4_rt_num_index		= 0xf,
-	.v4_modem_rt_index_lo		= 0x0,
-	.v4_modem_rt_index_hi		= 0x7,
-	.v4_apps_rt_index_lo		= 0x8,
-	.v4_apps_rt_index_hi		= 0xe,
-	.v4_rt_hash_ofst		= 0x398,
-	.v4_rt_hash_size		= 0x0,
-	.v4_rt_hash_size_ddr		= 0x0,
-	.v4_rt_nhash_ofst		= 0x3A0,
-	.v4_rt_nhash_size		= 0x78,
-	.v4_rt_nhash_size_ddr		= 0x4000,
-	.v6_rt_num_index		= 0xf,
-	.v6_modem_rt_index_lo		= 0x0,
-	.v6_modem_rt_index_hi		= 0x7,
-	.v6_apps_rt_index_lo		= 0x8,
-	.v6_apps_rt_index_hi		= 0xe,
-	.v6_rt_hash_ofst		= 0x420,
-	.v6_rt_hash_size		= 0x0,
-	.v6_rt_hash_size_ddr		= 0x0,
-	.v6_rt_nhash_ofst		= 0x428,
-	.v6_rt_nhash_size		= 0x78,
-	.v6_rt_nhash_size_ddr		= 0x4000,
-	.modem_hdr_ofst			= 0x4A8,
-	.modem_hdr_size			= 0x140,
-	.apps_hdr_ofst			= 0x5E8,
-	.apps_hdr_size			= 0x0,
-	.apps_hdr_size_ddr		= 0x800,
-	.modem_hdr_proc_ctx_ofst	= 0x5F0,
-	.modem_hdr_proc_ctx_size	= 0x200,
-	.apps_hdr_proc_ctx_ofst		= 0x7F0,
-	.apps_hdr_proc_ctx_size		= 0x200,
-	.apps_hdr_proc_ctx_size_ddr	= 0x0,
-	.modem_comp_decomp_ofst		= 0x0,
-	.modem_comp_decomp_size		= 0x0,
-	.modem_ofst			= 0xbf0,
-	.modem_size			= 0x140c,
-	.apps_v4_flt_hash_ofst		= 0x1bfc,
-	.apps_v4_flt_hash_size		= 0x0,
-	.apps_v4_flt_nhash_ofst		= 0x1bfc,
-	.apps_v4_flt_nhash_size		= 0x0,
-	.apps_v6_flt_hash_ofst		= 0x1bfc,
-	.apps_v6_flt_hash_size		= 0x0,
-	.apps_v6_flt_nhash_ofst		= 0x1bfc,
-	.apps_v6_flt_nhash_size		= 0x0,
-	.uc_info_ofst			= 0x80,
-	.uc_info_size			= 0x200,
-	.end_ofst			= 0x2000,
-	.apps_v4_rt_hash_ofst		= 0x1bfc,
-	.apps_v4_rt_hash_size		= 0x0,
-	.apps_v4_rt_nhash_ofst		= 0x1bfc,
-	.apps_v4_rt_nhash_size		= 0x0,
-	.apps_v6_rt_hash_ofst		= 0x1bfc,
-	.apps_v6_rt_hash_size		= 0x0,
-	.apps_v6_rt_nhash_ofst		= 0x1bfc,
-	.apps_v6_rt_nhash_size		= 0x0,
-	.uc_descriptor_ram_ofst		= 0x2000,
-	.uc_descriptor_ram_size		= 0x0,
-	.pdn_config_ofst		= 0x9F8,
-	.pdn_config_size		= 0x50,
-	.stats_quota_ofst		= 0xa50,
-	.stats_quota_size		= 0x60,
-	.stats_tethering_ofst		= 0xab0,
-	.stats_tethering_size		= 0x140,
-	.stats_flt_v4_ofst		= 0xbf0,
-	.stats_flt_v4_size		= 0x0,
-	.stats_flt_v6_ofst		= 0xbf0,
-	.stats_flt_v6_size		= 0x0,
-	.stats_rt_v4_ofst		= 0xbf0,
-	.stats_rt_v4_size		= 0x0,
-	.stats_rt_v6_ofst		= 0xbf0,
-	.stats_rt_v6_size		= 0x0,
-	.stats_drop_ofst		= 0xbf0,
-	.stats_drop_size		= 0x0,
-};
-
-static struct ipa3_mem_partition ipa_4_5_mem_part = {
-	.uc_info_ofst			= 0x80,
-	.uc_info_size			= 0x200,
-	.ofst_start			= 0x280,
-	.v4_flt_hash_ofst		= 0x288,
-	.v4_flt_hash_size		= 0x78,
-	.v4_flt_hash_size_ddr		= 0x4000,
-	.v4_flt_nhash_ofst		= 0x308,
-	.v4_flt_nhash_size		= 0x78,
-	.v4_flt_nhash_size_ddr		= 0x4000,
-	.v6_flt_hash_ofst		= 0x388,
-	.v6_flt_hash_size		= 0x78,
-	.v6_flt_hash_size_ddr		= 0x4000,
-	.v6_flt_nhash_ofst		= 0x408,
-	.v6_flt_nhash_size		= 0x78,
-	.v6_flt_nhash_size_ddr		= 0x4000,
-	.v4_rt_num_index		= 0xf,
-	.v4_modem_rt_index_lo		= 0x0,
-	.v4_modem_rt_index_hi		= 0x7,
-	.v4_apps_rt_index_lo		= 0x8,
-	.v4_apps_rt_index_hi		= 0xe,
-	.v4_rt_hash_ofst		= 0x488,
-	.v4_rt_hash_size		= 0x78,
-	.v4_rt_hash_size_ddr		= 0x4000,
-	.v4_rt_nhash_ofst		= 0x508,
-	.v4_rt_nhash_size		= 0x78,
-	.v4_rt_nhash_size_ddr		= 0x4000,
-	.v6_rt_num_index		= 0xf,
-	.v6_modem_rt_index_lo		= 0x0,
-	.v6_modem_rt_index_hi		= 0x7,
-	.v6_apps_rt_index_lo		= 0x8,
-	.v6_apps_rt_index_hi		= 0xe,
-	.v6_rt_hash_ofst		= 0x588,
-	.v6_rt_hash_size		= 0x78,
-	.v6_rt_hash_size_ddr		= 0x4000,
-	.v6_rt_nhash_ofst		= 0x608,
-	.v6_rt_nhash_size		= 0x78,
-	.v6_rt_nhash_size_ddr		= 0x4000,
-	.modem_hdr_ofst			= 0x688,
-	.modem_hdr_size			= 0x240,
-	.apps_hdr_ofst			= 0x8c8,
-	.apps_hdr_size			= 0x200,
-	.apps_hdr_size_ddr		= 0x800,
-	.modem_hdr_proc_ctx_ofst	= 0xad0,
-	.modem_hdr_proc_ctx_size	= 0xb20,
-	.apps_hdr_proc_ctx_ofst		= 0x15f0,
-	.apps_hdr_proc_ctx_size		= 0x200,
-	.apps_hdr_proc_ctx_size_ddr	= 0x0,
-	.nat_tbl_ofst			= 0x1800,
-	.nat_tbl_size			= 0x800,
-	.nat_index_tbl_ofst		= 0x2000,
-	.nat_index_tbl_size		= 0x100,
-	.nat_exp_tbl_ofst		= 0x2100,
-	.nat_exp_tbl_size		= 0x400,
-	.stats_quota_ofst		= 0x2510,
-	.stats_quota_size		= 0x78,
-	.stats_tethering_ofst		= 0x2588,
-	.stats_tethering_size		= 0x238,
-	.stats_flt_v4_ofst		= 0,
-	.stats_flt_v4_size		= 0,
-	.stats_flt_v6_ofst		= 0,
-	.stats_flt_v6_size		= 0,
-	.stats_rt_v4_ofst		= 0,
-	.stats_rt_v4_size		= 0,
-	.stats_rt_v6_ofst		= 0,
-	.stats_rt_v6_size		= 0,
-	.stats_fnr_ofst			= 0x27c0,
-	.stats_fnr_size			= 0x800,
-	.stats_drop_ofst		= 0x2fc0,
-	.stats_drop_size		= 0x20,
-	.modem_comp_decomp_ofst		= 0x0,
-	.modem_comp_decomp_size		= 0x0,
-	.modem_ofst			= 0x2fe8,
-	.modem_size			= 0x800,
-	.apps_v4_flt_hash_ofst	= 0x2718,
-	.apps_v4_flt_hash_size	= 0x0,
-	.apps_v4_flt_nhash_ofst	= 0x2718,
-	.apps_v4_flt_nhash_size	= 0x0,
-	.apps_v6_flt_hash_ofst	= 0x2718,
-	.apps_v6_flt_hash_size	= 0x0,
-	.apps_v6_flt_nhash_ofst	= 0x2718,
-	.apps_v6_flt_nhash_size	= 0x0,
-	.apps_v4_rt_hash_ofst	= 0x2718,
-	.apps_v4_rt_hash_size	= 0x0,
-	.apps_v4_rt_nhash_ofst	= 0x2718,
-	.apps_v4_rt_nhash_size	= 0x0,
-	.apps_v6_rt_hash_ofst	= 0x2718,
-	.apps_v6_rt_hash_size	= 0x0,
-	.apps_v6_rt_nhash_ofst	= 0x2718,
-	.apps_v6_rt_nhash_size	= 0x0,
-	.uc_descriptor_ram_ofst	= 0x3800,
-	.uc_descriptor_ram_size	= 0x1000,
-	.pdn_config_ofst	= 0x4800,
-	.pdn_config_size	= 0x50,
-	.end_ofst		= 0x4850,
-};
-
-static struct ipa3_mem_partition ipa_4_7_mem_part = {
-	.uc_info_ofst			= 0x80,
-	.uc_info_size			= 0x200,
-	.ofst_start			= 0x280,
-	.v4_flt_hash_ofst		= 0x288,
-	.v4_flt_hash_size		=  0x78,
-	.v4_flt_hash_size_ddr		= 0x4000,
-	.v4_flt_nhash_ofst		= 0x308,
-	.v4_flt_nhash_size		= 0x78,
-	.v4_flt_nhash_size_ddr		= 0x4000,
-	.v6_flt_hash_ofst		= 0x388,
-	.v6_flt_hash_size		= 0x78,
-	.v6_flt_hash_size_ddr		= 0x4000,
-	.v6_flt_nhash_ofst		= 0x408,
-	.v6_flt_nhash_size		= 0x78,
-	.v6_flt_nhash_size_ddr		= 0x4000,
-	.v4_rt_num_index		= 0xf,
-	.v4_modem_rt_index_lo		= 0x0,
-	.v4_modem_rt_index_hi		= 0x7,
-	.v4_apps_rt_index_lo		= 0x8,
-	.v4_apps_rt_index_hi		= 0xe,
-	.v4_rt_hash_ofst		= 0x488,
-	.v4_rt_hash_size		= 0x78,
-	.v4_rt_hash_size_ddr		= 0x4000,
-	.v4_rt_nhash_ofst		= 0x508,
-	.v4_rt_nhash_size		= 0x78,
-	.v4_rt_nhash_size_ddr		= 0x4000,
-	.v6_rt_num_index		= 0xf,
-	.v6_modem_rt_index_lo		= 0x0,
-	.v6_modem_rt_index_hi		= 0x7,
-	.v6_apps_rt_index_lo		= 0x8,
-	.v6_apps_rt_index_hi		= 0xe,
-	.v6_rt_hash_ofst		= 0x588,
-	.v6_rt_hash_size		= 0x78,
-	.v6_rt_hash_size_ddr		= 0x4000,
-	.v6_rt_nhash_ofst		= 0x608,
-	.v6_rt_nhash_size		= 0x78,
-	.v6_rt_nhash_size_ddr		= 0x4000,
-	.modem_hdr_ofst			= 0x688,
-	.modem_hdr_size			= 0x240,
-	.apps_hdr_ofst			= 0x8c8,
-	.apps_hdr_size			= 0x200,
-	.apps_hdr_size_ddr		= 0x800,
-	.modem_hdr_proc_ctx_ofst	= 0xad0,
-	.modem_hdr_proc_ctx_size	= 0x200,
-	.apps_hdr_proc_ctx_ofst		= 0xcd0,
-	.apps_hdr_proc_ctx_size		= 0x200,
-	.apps_hdr_proc_ctx_size_ddr	= 0x0,
-	.nat_tbl_ofst			= 0xee0,
-	.nat_tbl_size			= 0x800,
-	.nat_index_tbl_ofst		= 0x16e0,
-	.nat_index_tbl_size		= 0x100,
-	.nat_exp_tbl_ofst		= 0x17e0,
-	.nat_exp_tbl_size		= 0x400,
-	.pdn_config_ofst		= 0x1be8,
-	.pdn_config_size		= 0x50,
-	.stats_quota_ofst		= 0x1c40,
-	.stats_quota_size		= 0x78,
-	.stats_tethering_ofst		= 0x1cb8,
-	.stats_tethering_size		= 0x238,
-	.stats_flt_v4_ofst		= 0,
-	.stats_flt_v4_size		= 0,
-	.stats_flt_v6_ofst		= 0,
-	.stats_flt_v6_size		= 0,
-	.stats_rt_v4_ofst		= 0,
-	.stats_rt_v4_size		= 0,
-	.stats_rt_v6_ofst		= 0,
-	.stats_rt_v6_size		= 0,
-	.stats_fnr_ofst			= 0x1ef0,
-	.stats_fnr_size			= 0x0,
-	.stats_drop_ofst		= 0x1ef0,
-	.stats_drop_size		= 0x20,
-	.modem_comp_decomp_ofst		= 0x0,
-	.modem_comp_decomp_size		= 0x0,
-	.modem_ofst			= 0x1f18,
-	.modem_size			= 0x100c,
-	.apps_v4_flt_hash_ofst	= 0x1f18,
-	.apps_v4_flt_hash_size	= 0x0,
-	.apps_v4_flt_nhash_ofst	= 0x1f18,
-	.apps_v4_flt_nhash_size	= 0x0,
-	.apps_v6_flt_hash_ofst	= 0x1f18,
-	.apps_v6_flt_hash_size	= 0x0,
-	.apps_v6_flt_nhash_ofst	= 0x1f18,
-	.apps_v6_flt_nhash_size	= 0x0,
-	.apps_v4_rt_hash_ofst	= 0x1f18,
-	.apps_v4_rt_hash_size	= 0x0,
-	.apps_v4_rt_nhash_ofst	= 0x1f18,
-	.apps_v4_rt_nhash_size	= 0x0,
-	.apps_v6_rt_hash_ofst	= 0x1f18,
-	.apps_v6_rt_hash_size	= 0x0,
-	.apps_v6_rt_nhash_ofst	= 0x1f18,
-	.apps_v6_rt_nhash_size	= 0x0,
-	.uc_descriptor_ram_ofst	= 0x3000,
-	.uc_descriptor_ram_size	= 0x0000,
-	.end_ofst		= 0x3000,
-};
-
-/**
- * ipa3_get_clients_from_rm_resource() - get IPA clients which are related to an
- * IPA_RM resource
- *
- * @resource: [IN] IPA Resource Manager resource
- * @clients: [OUT] Empty array which will contain the list of clients. The
- *         caller must initialize this array.
- *
- * Return codes: 0 on success, negative on failure.
- */
-int ipa3_get_clients_from_rm_resource(
-	enum ipa_rm_resource_name resource,
-	struct ipa3_client_names *clients)
-{
-	int i = 0;
-
-	if (resource < 0 ||
-	    resource >= IPA_RM_RESOURCE_MAX ||
-	    !clients) {
-		IPAERR("Bad parameters\n");
-		return -EINVAL;
-	}
-
-	switch (resource) {
-	case IPA_RM_RESOURCE_USB_CONS:
-		if (ipa3_get_ep_mapping(IPA_CLIENT_USB_CONS) != -1)
-			clients->names[i++] = IPA_CLIENT_USB_CONS;
-		break;
-	case IPA_RM_RESOURCE_USB_DPL_CONS:
-		if (ipa3_get_ep_mapping(IPA_CLIENT_USB_DPL_CONS) != -1)
-			clients->names[i++] = IPA_CLIENT_USB_DPL_CONS;
-		break;
-	case IPA_RM_RESOURCE_HSIC_CONS:
-		clients->names[i++] = IPA_CLIENT_HSIC1_CONS;
-		break;
-	case IPA_RM_RESOURCE_WLAN_CONS:
-		clients->names[i++] = IPA_CLIENT_WLAN1_CONS;
-		clients->names[i++] = IPA_CLIENT_WLAN2_CONS;
-		clients->names[i++] = IPA_CLIENT_WLAN3_CONS;
-		break;
-	case IPA_RM_RESOURCE_MHI_CONS:
-		clients->names[i++] = IPA_CLIENT_MHI_CONS;
-		break;
-	case IPA_RM_RESOURCE_ODU_ADAPT_CONS:
-		clients->names[i++] = IPA_CLIENT_ODU_EMB_CONS;
-		clients->names[i++] = IPA_CLIENT_ODU_TETH_CONS;
-		break;
-	case IPA_RM_RESOURCE_ETHERNET_CONS:
-		clients->names[i++] = IPA_CLIENT_ETHERNET_CONS;
-		break;
-	case IPA_RM_RESOURCE_USB_PROD:
-		if (ipa3_get_ep_mapping(IPA_CLIENT_USB_PROD) != -1)
-			clients->names[i++] = IPA_CLIENT_USB_PROD;
-		break;
-	case IPA_RM_RESOURCE_HSIC_PROD:
-		clients->names[i++] = IPA_CLIENT_HSIC1_PROD;
-		break;
-	case IPA_RM_RESOURCE_MHI_PROD:
-		clients->names[i++] = IPA_CLIENT_MHI_PROD;
-		break;
-	case IPA_RM_RESOURCE_ODU_ADAPT_PROD:
-		clients->names[i++] = IPA_CLIENT_ODU_PROD;
-		break;
-	case IPA_RM_RESOURCE_ETHERNET_PROD:
-		clients->names[i++] = IPA_CLIENT_ETHERNET_PROD;
-		break;
-	default:
-		break;
-	}
-	clients->length = i;
-
-	return 0;
-}
-
-/**
- * ipa3_should_pipe_be_suspended() - returns true when the client's pipe should
- * be suspended during a power save scenario. False otherwise.
- *
- * @client: [IN] IPA client
- */
-bool ipa3_should_pipe_be_suspended(enum ipa_client_type client)
-{
-	struct ipa3_ep_context *ep;
-	int ipa_ep_idx;
-
-	ipa_ep_idx = ipa3_get_ep_mapping(client);
-	if (ipa_ep_idx == -1) {
-		IPAERR("Invalid client.\n");
-		WARN_ON(1);
-		return false;
-	}
-
-	ep = &ipa3_ctx->ep[ipa_ep_idx];
-
-	/*
-	 * starting IPA 4.0 pipe no longer can be suspended. Instead,
-	 * the corresponding GSI channel should be stopped. Usually client
-	 * driver will take care of stopping the channel. For client drivers
-	 * that are not stopping the channel, IPA RM will do that based on
-	 * ipa3_should_pipe_channel_be_stopped().
-	 */
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0)
-		return false;
-
-	if (ep->keep_ipa_awake)
-		return false;
-
-	if (client == IPA_CLIENT_USB_CONS     ||
-	    client == IPA_CLIENT_USB_DPL_CONS ||
-	    client == IPA_CLIENT_MHI_CONS     ||
-	    client == IPA_CLIENT_MHI_DPL_CONS ||
-	    client == IPA_CLIENT_HSIC1_CONS   ||
-	    client == IPA_CLIENT_WLAN1_CONS   ||
-	    client == IPA_CLIENT_WLAN2_CONS   ||
-	    client == IPA_CLIENT_WLAN3_CONS   ||
-	    client == IPA_CLIENT_WLAN4_CONS   ||
-	    client == IPA_CLIENT_ODU_EMB_CONS ||
-	    client == IPA_CLIENT_ODU_TETH_CONS ||
-	    client == IPA_CLIENT_ETHERNET_CONS)
-		return true;
-
-	return false;
-}
-
-/**
- * ipa3_should_pipe_channel_be_stopped() - returns true when the client's
- * channel should be stopped during a power save scenario. False otherwise.
- * Most client already stops the GSI channel on suspend, and are not included
- * in the list below.
- *
- * @client: [IN] IPA client
- */
-static bool ipa3_should_pipe_channel_be_stopped(enum ipa_client_type client)
-{
-	struct ipa3_ep_context *ep;
-	int ipa_ep_idx;
-
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_0)
-		return false;
-
-	ipa_ep_idx = ipa3_get_ep_mapping(client);
-	if (ipa_ep_idx == -1) {
-		IPAERR("Invalid client.\n");
-		WARN_ON(1);
-		return false;
-	}
-
-	ep = &ipa3_ctx->ep[ipa_ep_idx];
-
-	if (ep->keep_ipa_awake)
-		return false;
-
-	if (client == IPA_CLIENT_ODU_EMB_CONS ||
-	    client == IPA_CLIENT_ODU_TETH_CONS)
-		return true;
-
-	return false;
-}
-
-/**
- * ipa3_suspend_resource_sync() - suspend client endpoints related to the IPA_RM
- * resource and decrement active clients counter, which may result in clock
- * gating of IPA clocks.
- *
- * @resource: [IN] IPA Resource Manager resource
- *
- * Return codes: 0 on success, negative on failure.
- */
-int ipa3_suspend_resource_sync(enum ipa_rm_resource_name resource)
-{
-	struct ipa3_client_names clients;
-	int res;
-	int index;
-	struct ipa_ep_cfg_ctrl suspend;
-	enum ipa_client_type client;
-	int ipa_ep_idx;
-	bool pipe_suspended = false;
-
-	memset(&clients, 0, sizeof(clients));
-	res = ipa3_get_clients_from_rm_resource(resource, &clients);
-	if (res) {
-		IPAERR("Bad params.\n");
-		return res;
-	}
-
-	for (index = 0; index < clients.length; index++) {
-		client = clients.names[index];
-		ipa_ep_idx = ipa3_get_ep_mapping(client);
-		if (ipa_ep_idx == -1) {
-			IPAERR("Invalid client.\n");
-			res = -EINVAL;
-			continue;
-		}
-		ipa3_ctx->resume_on_connect[client] = false;
-		if (ipa3_ctx->ep[ipa_ep_idx].client == client &&
-		    ipa3_should_pipe_be_suspended(client)) {
-			if (ipa3_ctx->ep[ipa_ep_idx].valid) {
-				/* suspend endpoint */
-				memset(&suspend, 0, sizeof(suspend));
-				suspend.ipa_ep_suspend = true;
-				ipa3_cfg_ep_ctrl(ipa_ep_idx, &suspend);
-				pipe_suspended = true;
-			}
-		}
-
-		if (ipa3_ctx->ep[ipa_ep_idx].client == client &&
-			ipa3_should_pipe_channel_be_stopped(client)) {
-			if (ipa3_ctx->ep[ipa_ep_idx].valid) {
-				/* Stop GSI channel */
-				res = ipa3_stop_gsi_channel(ipa_ep_idx);
-				if (res) {
-					IPAERR("failed stop gsi ch %lu\n",
-					ipa3_ctx->ep[ipa_ep_idx].gsi_chan_hdl);
-					return res;
-				}
-			}
-		}
-	}
-	/* Sleep ~1 msec */
-	if (pipe_suspended)
-		usleep_range(1000, 2000);
-
-	/* before gating IPA clocks do TAG process */
-	ipa3_ctx->tag_process_before_gating = true;
-	IPA_ACTIVE_CLIENTS_DEC_RESOURCE(ipa_rm_resource_str(resource));
-
-	return 0;
-}
-
-/**
- * ipa3_suspend_resource_no_block() - suspend client endpoints related to the
- * IPA_RM resource and decrement active clients counter. This function is
- * guaranteed to avoid sleeping.
- *
- * @resource: [IN] IPA Resource Manager resource
- *
- * Return codes: 0 on success, negative on failure.
- */
-int ipa3_suspend_resource_no_block(enum ipa_rm_resource_name resource)
-{
-	int res;
-	struct ipa3_client_names clients;
-	int index;
-	enum ipa_client_type client;
-	struct ipa_ep_cfg_ctrl suspend;
-	int ipa_ep_idx;
-	struct ipa_active_client_logging_info log_info;
-
-	memset(&clients, 0, sizeof(clients));
-	res = ipa3_get_clients_from_rm_resource(resource, &clients);
-	if (res) {
-		IPAERR(
-			"ipa3_get_clients_from_rm_resource() failed, name = %d.\n",
-			resource);
-		goto bail;
-	}
-
-	for (index = 0; index < clients.length; index++) {
-		client = clients.names[index];
-		ipa_ep_idx = ipa3_get_ep_mapping(client);
-		if (ipa_ep_idx == -1) {
-			IPAERR("Invalid client.\n");
-			res = -EINVAL;
-			continue;
-		}
-		ipa3_ctx->resume_on_connect[client] = false;
-		if (ipa3_ctx->ep[ipa_ep_idx].client == client &&
-		    ipa3_should_pipe_be_suspended(client)) {
-			if (ipa3_ctx->ep[ipa_ep_idx].valid) {
-				/* suspend endpoint */
-				memset(&suspend, 0, sizeof(suspend));
-				suspend.ipa_ep_suspend = true;
-				ipa3_cfg_ep_ctrl(ipa_ep_idx, &suspend);
-			}
-		}
-
-		if (ipa3_ctx->ep[ipa_ep_idx].client == client &&
-			ipa3_should_pipe_channel_be_stopped(client)) {
-			res = -EPERM;
-			goto bail;
-		}
-	}
-
-	if (res == 0) {
-		IPA_ACTIVE_CLIENTS_PREP_RESOURCE(log_info,
-				ipa_rm_resource_str(resource));
-		/* before gating IPA clocks do TAG process */
-		ipa3_ctx->tag_process_before_gating = true;
-		ipa3_dec_client_disable_clks_no_block(&log_info);
-	}
-bail:
-	return res;
-}
-
-/**
- * ipa3_resume_resource() - resume client endpoints related to the IPA_RM
- * resource.
- *
- * @resource: [IN] IPA Resource Manager resource
- *
- * Return codes: 0 on success, negative on failure.
- */
-int ipa3_resume_resource(enum ipa_rm_resource_name resource)
-{
-
-	struct ipa3_client_names clients;
-	int res;
-	int index;
-	struct ipa_ep_cfg_ctrl suspend;
-	enum ipa_client_type client;
-	int ipa_ep_idx;
-
-	memset(&clients, 0, sizeof(clients));
-	res = ipa3_get_clients_from_rm_resource(resource, &clients);
-	if (res) {
-		IPAERR("ipa3_get_clients_from_rm_resource() failed.\n");
-		return res;
-	}
-
-	for (index = 0; index < clients.length; index++) {
-		client = clients.names[index];
-		ipa_ep_idx = ipa3_get_ep_mapping(client);
-		if (ipa_ep_idx == -1) {
-			IPAERR("Invalid client.\n");
-			res = -EINVAL;
-			continue;
-		}
-		/*
-		 * The related ep, will be resumed on connect
-		 * while its resource is granted
-		 */
-		ipa3_ctx->resume_on_connect[client] = true;
-		IPADBG("%d will be resumed on connect.\n", client);
-		if (ipa3_ctx->ep[ipa_ep_idx].client == client &&
-		    ipa3_should_pipe_be_suspended(client)) {
-			if (ipa3_ctx->ep[ipa_ep_idx].valid) {
-				memset(&suspend, 0, sizeof(suspend));
-				suspend.ipa_ep_suspend = false;
-				ipa3_cfg_ep_ctrl(ipa_ep_idx, &suspend);
-			}
-		}
-
-		if (ipa3_ctx->ep[ipa_ep_idx].client == client &&
-			ipa3_should_pipe_channel_be_stopped(client)) {
-			if (ipa3_ctx->ep[ipa_ep_idx].valid) {
-				res = gsi_start_channel(
-					ipa3_ctx->ep[ipa_ep_idx].gsi_chan_hdl);
-				if (res) {
-					IPAERR("failed to start gsi ch %lu\n",
-					ipa3_ctx->ep[ipa_ep_idx].gsi_chan_hdl);
-					return res;
-				}
-			}
-		}
-	}
-
-	return res;
-}
-
-/**
- * ipa3_get_hw_type_index() - Get HW type index which is used as the entry index
- *	for ep\resource groups related arrays .
- *
- * Return value: HW type index
- */
-static u8 ipa3_get_hw_type_index(void)
-{
-	u8 hw_type_index;
-
-	switch (ipa3_ctx->ipa_hw_type) {
-	case IPA_HW_v3_0:
-	case IPA_HW_v3_1:
-		hw_type_index = IPA_3_0;
-		break;
-	case IPA_HW_v3_5:
-		hw_type_index = IPA_3_5;
-		/*
-		 *this flag is initialized only after fw load trigger from
-		 * user space (ipa3_write)
-		 */
-		if (ipa3_ctx->ipa_config_is_mhi)
-			hw_type_index = IPA_3_5_MHI;
-		break;
-	case IPA_HW_v3_5_1:
-		hw_type_index = IPA_3_5_1;
-		break;
-	case IPA_HW_v4_0:
-		hw_type_index = IPA_4_0;
-		/*
-		 *this flag is initialized only after fw load trigger from
-		 * user space (ipa3_write)
-		 */
-		if (ipa3_ctx->ipa_config_is_mhi)
-			hw_type_index = IPA_4_0_MHI;
-		break;
-	case IPA_HW_v4_1:
-		hw_type_index = IPA_4_1;
-		break;
-	case IPA_HW_v4_2:
-		hw_type_index = IPA_4_2;
-		break;
-	case IPA_HW_v4_5:
-		hw_type_index = IPA_4_5;
-		if (ipa3_ctx->ipa_config_is_mhi)
-			hw_type_index = IPA_4_5_MHI;
-		if (ipa3_ctx->platform_type == IPA_PLAT_TYPE_APQ)
-			hw_type_index = IPA_4_5_APQ;
-		break;
-	case IPA_HW_v4_7:
-		hw_type_index = IPA_4_7;
-		break;
-	default:
-		IPAERR("Incorrect IPA version %d\n", ipa3_ctx->ipa_hw_type);
-		hw_type_index = IPA_3_0;
-		break;
-	}
-
-	return hw_type_index;
-}
-
-/**
- * _ipa_sram_settings_read_v3_0() - Read SRAM settings from HW
- *
- * Returns:	None
- */
-void _ipa_sram_settings_read_v3_0(void)
-{
-	struct ipahal_reg_shared_mem_size smem_sz;
-
-	memset(&smem_sz, 0, sizeof(smem_sz));
-
-	ipahal_read_reg_fields(IPA_SHARED_MEM_SIZE, &smem_sz);
-
-	ipa3_ctx->smem_restricted_bytes = smem_sz.shared_mem_baddr;
-	ipa3_ctx->smem_sz = smem_sz.shared_mem_sz;
-
-	/* reg fields are in 8B units */
-	ipa3_ctx->smem_restricted_bytes *= 8;
-	ipa3_ctx->smem_sz *= 8;
-	ipa3_ctx->smem_reqd_sz = IPA_MEM_PART(end_ofst);
-	ipa3_ctx->hdr_tbl_lcl = false;
-	ipa3_ctx->hdr_proc_ctx_tbl_lcl = true;
-
-	/*
-	 * when proc ctx table is located in internal memory,
-	 * modem entries resides first.
-	 */
-	if (ipa3_ctx->hdr_proc_ctx_tbl_lcl) {
-		ipa3_ctx->hdr_proc_ctx_tbl.start_offset =
-			IPA_MEM_PART(modem_hdr_proc_ctx_size);
-	}
-	ipa3_ctx->ip4_rt_tbl_hash_lcl =	false;
-	ipa3_ctx->ip4_rt_tbl_nhash_lcl = false;
-	ipa3_ctx->ip6_rt_tbl_hash_lcl = false;
-	ipa3_ctx->ip6_rt_tbl_nhash_lcl = false;
-	ipa3_ctx->ip4_flt_tbl_hash_lcl = false;
-	ipa3_ctx->ip4_flt_tbl_nhash_lcl = false;
-	ipa3_ctx->ip6_flt_tbl_hash_lcl = false;
-	ipa3_ctx->ip6_flt_tbl_nhash_lcl = false;
-}
-
-/**
- * ipa3_cfg_route() - configure IPA route
- * @route: IPA route
- *
- * Return codes:
- * 0: success
- */
-int ipa3_cfg_route(struct ipahal_reg_route *route)
-{
-
-	IPADBG("disable_route_block=%d, default_pipe=%d, default_hdr_tbl=%d\n",
-		route->route_dis,
-		route->route_def_pipe,
-		route->route_def_hdr_table);
-	IPADBG("default_hdr_ofst=%d, default_frag_pipe=%d\n",
-		route->route_def_hdr_ofst,
-		route->route_frag_def_pipe);
-
-	IPADBG("default_retain_hdr=%d\n",
-		route->route_def_retain_hdr);
-
-	if (route->route_dis) {
-		IPAERR("Route disable is not supported!\n");
-		return -EPERM;
-	}
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	ipahal_write_reg_fields(IPA_ROUTE, route);
-
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-
-	return 0;
-}
-
-/**
- * ipa3_cfg_filter() - configure filter
- * @disable: disable value
- *
- * Return codes:
- * 0: success
- */
-int ipa3_cfg_filter(u32 disable)
-{
-	IPAERR_RL("Filter disable is not supported!\n");
-	return -EPERM;
-}
-
-/**
- * ipa_disable_hashing_rt_flt_v4_2() - Disable filer and route hashing.
- *
- * Return codes: 0 for success, negative value for failure
- */
-static int ipa_disable_hashing_rt_flt_v4_2(void)
-{
-
-	IPADBG("Disable hashing for filter and route table in IPA 4.2 HW\n");
-	ipahal_write_reg(IPA_FILT_ROUT_HASH_EN,
-					IPA_FILT_ROUT_HASH_REG_VAL_v4_2);
-	return 0;
-}
-
-
-/**
- * ipa_comp_cfg() - Configure QMB/Master port selection
- *
- * Returns:	None
- */
-static void ipa_comp_cfg(void)
-{
-	struct ipahal_reg_comp_cfg comp_cfg;
-
-	/* IPAv4 specific, on NON-MHI config*/
-	if ((ipa3_ctx->ipa_hw_type == IPA_HW_v4_0) &&
-		(!ipa3_ctx->ipa_config_is_mhi)) {
-
-		ipahal_read_reg_fields(IPA_COMP_CFG, &comp_cfg);
-		IPADBG("Before comp config\n");
-		IPADBG("ipa_qmb_select_by_address_global_en = %d\n",
-			comp_cfg.ipa_qmb_select_by_address_global_en);
-
-		IPADBG("ipa_qmb_select_by_address_prod_en = %d\n",
-				comp_cfg.ipa_qmb_select_by_address_prod_en);
-
-		IPADBG("ipa_qmb_select_by_address_cons_en = %d\n",
-				comp_cfg.ipa_qmb_select_by_address_cons_en);
-
-		comp_cfg.ipa_qmb_select_by_address_global_en = false;
-		comp_cfg.ipa_qmb_select_by_address_prod_en = false;
-		comp_cfg.ipa_qmb_select_by_address_cons_en = false;
-
-		ipahal_write_reg_fields(IPA_COMP_CFG, &comp_cfg);
-
-		ipahal_read_reg_fields(IPA_COMP_CFG, &comp_cfg);
-		IPADBG("After comp config\n");
-		IPADBG("ipa_qmb_select_by_address_global_en = %d\n",
-			comp_cfg.ipa_qmb_select_by_address_global_en);
-
-		IPADBG("ipa_qmb_select_by_address_prod_en = %d\n",
-				comp_cfg.ipa_qmb_select_by_address_prod_en);
-
-		IPADBG("ipa_qmb_select_by_address_cons_en = %d\n",
-				comp_cfg.ipa_qmb_select_by_address_cons_en);
-	}
-
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0) {
-		ipahal_read_reg_fields(IPA_COMP_CFG, &comp_cfg);
-		IPADBG("Before comp config\n");
-		IPADBG("gsi_multi_inorder_rd_dis = %d\n",
-			comp_cfg.gsi_multi_inorder_rd_dis);
-
-		IPADBG("gsi_multi_inorder_wr_dis = %d\n",
-			comp_cfg.gsi_multi_inorder_wr_dis);
-
-		comp_cfg.gsi_multi_inorder_rd_dis = true;
-		comp_cfg.gsi_multi_inorder_wr_dis = true;
-
-		ipahal_write_reg_fields(IPA_COMP_CFG, &comp_cfg);
-
-		ipahal_read_reg_fields(IPA_COMP_CFG, &comp_cfg);
-		IPADBG("After comp config\n");
-		IPADBG("gsi_multi_inorder_rd_dis = %d\n",
-			comp_cfg.gsi_multi_inorder_rd_dis);
-
-		IPADBG("gsi_multi_inorder_wr_dis = %d\n",
-			comp_cfg.gsi_multi_inorder_wr_dis);
-	}
-
-	/* set GSI_MULTI_AXI_MASTERS_DIS = true after HW.4.1 */
-	if ((ipa3_ctx->ipa_hw_type == IPA_HW_v4_1) ||
-		(ipa3_ctx->ipa_hw_type == IPA_HW_v4_2)) {
-		ipahal_read_reg_fields(IPA_COMP_CFG, &comp_cfg);
-		IPADBG("Before comp config\n");
-		IPADBG("gsi_multi_axi_masters_dis = %d\n",
-			comp_cfg.gsi_multi_axi_masters_dis);
-
-		comp_cfg.gsi_multi_axi_masters_dis = true;
-
-		ipahal_write_reg_fields(IPA_COMP_CFG, &comp_cfg);
-
-		ipahal_read_reg_fields(IPA_COMP_CFG, &comp_cfg);
-		IPADBG("After comp config\n");
-		IPADBG("gsi_multi_axi_masters_dis = %d\n",
-			comp_cfg.gsi_multi_axi_masters_dis);
-	}
-}
-
-/**
- * ipa3_cfg_qsb() - Configure IPA QSB maximal reads and writes
- *
- * Returns:	None
- */
-static void ipa3_cfg_qsb(void)
-{
-	u8 hw_type_idx;
-	const struct ipa_qmb_outstanding *qmb_ot;
-	struct ipahal_reg_qsb_max_reads max_reads = { 0 };
-	struct ipahal_reg_qsb_max_writes max_writes = { 0 };
-
-	hw_type_idx = ipa3_get_hw_type_index();
-
-	qmb_ot = &(ipa3_qmb_outstanding[hw_type_idx][IPA_QMB_INSTANCE_DDR]);
-	max_reads.qmb_0_max_reads = qmb_ot->ot_reads;
-	max_writes.qmb_0_max_writes = qmb_ot->ot_writes;
-
-	qmb_ot = &(ipa3_qmb_outstanding[hw_type_idx][IPA_QMB_INSTANCE_PCIE]);
-	max_reads.qmb_1_max_reads = qmb_ot->ot_reads;
-	max_writes.qmb_1_max_writes = qmb_ot->ot_writes;
-
-	ipahal_write_reg_fields(IPA_QSB_MAX_WRITES, &max_writes);
-	ipahal_write_reg_fields(IPA_QSB_MAX_READS, &max_reads);
-}
-
-/* relevant starting IPA4.5 */
-static void ipa_cfg_qtime(void)
-{
-	struct ipahal_reg_qtime_timestamp_cfg ts_cfg;
-	struct ipahal_reg_timers_pulse_gran_cfg gran_cfg;
-	struct ipahal_reg_timers_xo_clk_div_cfg div_cfg;
-	u32 val;
-
-	/* Configure timestamp resolution */
-	memset(&ts_cfg, 0, sizeof(ts_cfg));
-	ts_cfg.dpl_timestamp_lsb = 0;
-	ts_cfg.dpl_timestamp_sel = false; /* DPL: use legacy 1ms resolution */
-	ts_cfg.tag_timestamp_lsb = IPA_TAG_TIMER_TIMESTAMP_SHFT;
-	ts_cfg.nat_timestamp_lsb = IPA_NAT_TIMER_TIMESTAMP_SHFT;
-	val = ipahal_read_reg(IPA_QTIME_TIMESTAMP_CFG);
-	IPADBG("qtime timestamp before cfg: 0x%x\n", val);
-	ipahal_write_reg_fields(IPA_QTIME_TIMESTAMP_CFG, &ts_cfg);
-	val = ipahal_read_reg(IPA_QTIME_TIMESTAMP_CFG);
-	IPADBG("qtime timestamp after cfg: 0x%x\n", val);
-
-	/* Configure timers pulse generators granularity */
-	memset(&gran_cfg, 0, sizeof(gran_cfg));
-	gran_cfg.gran_0 = IPA_TIMERS_TIME_GRAN_100_USEC;
-	gran_cfg.gran_1 = IPA_TIMERS_TIME_GRAN_1_MSEC;
-	gran_cfg.gran_2 = IPA_TIMERS_TIME_GRAN_1_MSEC;
-	val = ipahal_read_reg(IPA_TIMERS_PULSE_GRAN_CFG);
-	IPADBG("timer pulse granularity before cfg: 0x%x\n", val);
-	ipahal_write_reg_fields(IPA_TIMERS_PULSE_GRAN_CFG, &gran_cfg);
-	val = ipahal_read_reg(IPA_TIMERS_PULSE_GRAN_CFG);
-	IPADBG("timer pulse granularity after cfg: 0x%x\n", val);
-
-	/* Configure timers XO Clock divider */
-	memset(&div_cfg, 0, sizeof(div_cfg));
-	ipahal_read_reg_fields(IPA_TIMERS_XO_CLK_DIV_CFG, &div_cfg);
-	IPADBG("timer XO clk divider before cfg: enabled=%d divider=%u\n",
-		div_cfg.enable, div_cfg.value);
-
-	/* Make sure divider is disabled */
-	if (div_cfg.enable) {
-		div_cfg.enable = false;
-		ipahal_write_reg_fields(IPA_TIMERS_XO_CLK_DIV_CFG, &div_cfg);
-	}
-
-	/* At emulation systems XO clock is lower than on real target.
-	 * (e.g. 19.2Mhz compared to 96Khz)
-	 * Use lowest possible divider.
-	 */
-	if (ipa3_ctx->ipa3_hw_mode == IPA_HW_MODE_VIRTUAL ||
-		ipa3_ctx->ipa3_hw_mode == IPA_HW_MODE_EMULATION) {
-		div_cfg.value = 0;
-	}
-
-	div_cfg.enable = true; /* Enable the divider */
-	ipahal_write_reg_fields(IPA_TIMERS_XO_CLK_DIV_CFG, &div_cfg);
-	ipahal_read_reg_fields(IPA_TIMERS_XO_CLK_DIV_CFG, &div_cfg);
-	IPADBG("timer XO clk divider after cfg: enabled=%d divider=%u\n",
-		div_cfg.enable, div_cfg.value);
-}
-
-/**
- * ipa3_init_hw() - initialize HW
- *
- * Return codes:
- * 0: success
- */
-int ipa3_init_hw(void)
-{
-	u32 ipa_version = 0;
-	struct ipahal_reg_counter_cfg cnt_cfg;
-
-	/* Read IPA version and make sure we have access to the registers */
-	ipa_version = ipahal_read_reg(IPA_VERSION);
-	IPADBG("IPA_VERSION=%u\n", ipa_version);
-	if (ipa_version == 0)
-		return -EFAULT;
-
-	switch (ipa3_ctx->ipa_hw_type) {
-	case IPA_HW_v3_0:
-	case IPA_HW_v3_1:
-		ipahal_write_reg(IPA_BCR, IPA_BCR_REG_VAL_v3_0);
-		break;
-	case IPA_HW_v3_5:
-	case IPA_HW_v3_5_1:
-		ipahal_write_reg(IPA_BCR, IPA_BCR_REG_VAL_v3_5);
-		break;
-	case IPA_HW_v4_0:
-	case IPA_HW_v4_1:
-		ipahal_write_reg(IPA_BCR, IPA_BCR_REG_VAL_v4_0);
-		break;
-	case IPA_HW_v4_2:
-		ipahal_write_reg(IPA_BCR, IPA_BCR_REG_VAL_v4_2);
-		break;
-	default:
-		IPADBG("Do not update BCR - hw_type=%d\n",
-			ipa3_ctx->ipa_hw_type);
-		break;
-	}
-
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0 &&
-		ipa3_ctx->ipa_hw_type < IPA_HW_v4_5) {
-		struct ipahal_reg_clkon_cfg clkon_cfg;
-		struct ipahal_reg_tx_cfg tx_cfg;
-
-		memset(&clkon_cfg, 0, sizeof(clkon_cfg));
-
-		/*enable open global clocks*/
-		clkon_cfg.open_global_2x_clk = true;
-		clkon_cfg.open_global = true;
-		ipahal_write_reg_fields(IPA_CLKON_CFG, &clkon_cfg);
-
-		ipahal_read_reg_fields(IPA_TX_CFG, &tx_cfg);
-		/* disable PA_MASK_EN to allow holb drop */
-		tx_cfg.pa_mask_en = 0;
-		ipahal_write_reg_fields(IPA_TX_CFG, &tx_cfg);
-	}
-
-	ipa3_cfg_qsb();
-
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_5) {
-		/* set aggr granularity for 0.5 msec*/
-		cnt_cfg.aggr_granularity = GRAN_VALUE_500_USEC;
-		ipahal_write_reg_fields(IPA_COUNTER_CFG, &cnt_cfg);
-	} else {
-		ipa_cfg_qtime();
-	}
-
-	ipa_comp_cfg();
-
-	/*
-	 * In IPA 4.2 filter and routing hashing not supported
-	 * disabling hash enable register.
-	 */
-	if (ipa3_ctx->ipa_fltrt_not_hashable)
-		ipa_disable_hashing_rt_flt_v4_2();
-
-	return 0;
-}
-
-/**
- * ipa3_get_ep_mapping() - provide endpoint mapping
- * @client: client type
- *
- * Return value: endpoint mapping
- */
-int ipa3_get_ep_mapping(enum ipa_client_type client)
-{
-	int ipa_ep_idx;
-	u8 hw_idx = ipa3_get_hw_type_index();
-
-	if (client >= IPA_CLIENT_MAX || client < 0) {
-		IPAERR_RL("Bad client number! client =%d\n", client);
-		return IPA_EP_NOT_ALLOCATED;
-	}
-
-	if (!ipa3_ep_mapping[hw_idx][client].valid)
-		return IPA_EP_NOT_ALLOCATED;
-
-	ipa_ep_idx =
-		ipa3_ep_mapping[hw_idx][client].ipa_gsi_ep_info.ipa_ep_num;
-	if (ipa_ep_idx < 0 || (ipa_ep_idx >= IPA3_MAX_NUM_PIPES
-		&& client != IPA_CLIENT_DUMMY_CONS))
-		return IPA_EP_NOT_ALLOCATED;
-
-	return ipa_ep_idx;
-}
-
-/**
- * ipa3_get_gsi_ep_info() - provide gsi ep information
- * @client: IPA client value
- *
- * Return value: pointer to ipa_gsi_ep_info
- */
-const struct ipa_gsi_ep_config *ipa3_get_gsi_ep_info
-	(enum ipa_client_type client)
-{
-	int ep_idx;
-
-	ep_idx = ipa3_get_ep_mapping(client);
-	if (ep_idx == IPA_EP_NOT_ALLOCATED)
-		return NULL;
-
-	if (!ipa3_ep_mapping[ipa3_get_hw_type_index()][client].valid)
-		return NULL;
-
-	return &(ipa3_ep_mapping[ipa3_get_hw_type_index()]
-		[client].ipa_gsi_ep_info);
-}
-
-/**
- * ipa_get_ep_group() - provide endpoint group by client
- * @client: client type
- *
- * Return value: endpoint group
- */
-int ipa_get_ep_group(enum ipa_client_type client)
-{
-	if (client >= IPA_CLIENT_MAX || client < 0) {
-		IPAERR("Bad client number! client =%d\n", client);
-		return -EINVAL;
-	}
-
-	if (!ipa3_ep_mapping[ipa3_get_hw_type_index()][client].valid)
-		return -EINVAL;
-
-	return ipa3_ep_mapping[ipa3_get_hw_type_index()][client].group_num;
-}
-
-/**
- * ipa3_get_qmb_master_sel() - provide QMB master selection for the client
- * @client: client type
- *
- * Return value: QMB master index
- */
-u8 ipa3_get_qmb_master_sel(enum ipa_client_type client)
-{
-	if (client >= IPA_CLIENT_MAX || client < 0) {
-		IPAERR("Bad client number! client =%d\n", client);
-		return -EINVAL;
-	}
-
-	if (!ipa3_ep_mapping[ipa3_get_hw_type_index()][client].valid)
-		return -EINVAL;
-
-	return ipa3_ep_mapping[ipa3_get_hw_type_index()]
-		[client].qmb_master_sel;
-}
-
-/* ipa3_set_client() - provide client mapping
- * @client: client type
- *
- * Return value: none
- */
-
-void ipa3_set_client(int index, enum ipacm_client_enum client, bool uplink)
-{
-	if (client > IPACM_CLIENT_MAX || client < IPACM_CLIENT_USB) {
-		IPAERR("Bad client number! client =%d\n", client);
-	} else if (index >= IPA3_MAX_NUM_PIPES || index < 0) {
-		IPAERR("Bad pipe index! index =%d\n", index);
-	} else {
-		ipa3_ctx->ipacm_client[index].client_enum = client;
-		ipa3_ctx->ipacm_client[index].uplink = uplink;
-	}
-}
-
-/* ipa3_get_wlan_stats() - get ipa wifi stats
- *
- * Return value: success or failure
- */
-int ipa3_get_wlan_stats(struct ipa_get_wdi_sap_stats *wdi_sap_stats)
-{
-	if (ipa3_ctx->uc_wdi_ctx.stats_notify) {
-		ipa3_ctx->uc_wdi_ctx.stats_notify(IPA_GET_WDI_SAP_STATS,
-			wdi_sap_stats);
-	} else {
-		IPAERR_RL("uc_wdi_ctx.stats_notify NULL\n");
-		return -EFAULT;
-	}
-	return 0;
-}
-
-int ipa3_set_wlan_quota(struct ipa_set_wifi_quota *wdi_quota)
-{
-	if (ipa3_ctx->uc_wdi_ctx.stats_notify) {
-		ipa3_ctx->uc_wdi_ctx.stats_notify(IPA_SET_WIFI_QUOTA,
-			wdi_quota);
-	} else {
-		IPAERR("uc_wdi_ctx.stats_notify NULL\n");
-		return -EFAULT;
-	}
-	return 0;
-}
-
-/**
- * ipa3_get_client() - provide client mapping
- * @client: client type
- *
- * Return value: client mapping enum
- */
-enum ipacm_client_enum ipa3_get_client(int pipe_idx)
-{
-	if (pipe_idx >= IPA3_MAX_NUM_PIPES || pipe_idx < 0) {
-		IPAERR("Bad pipe index! pipe_idx =%d\n", pipe_idx);
-		return IPACM_CLIENT_MAX;
-	} else {
-		return ipa3_ctx->ipacm_client[pipe_idx].client_enum;
-	}
-}
-
-/**
- * ipa2_get_client_uplink() - provide client mapping
- * @client: client type
- *
- * Return value: none
- */
-bool ipa3_get_client_uplink(int pipe_idx)
-{
-	if (pipe_idx < 0 || pipe_idx >= IPA3_MAX_NUM_PIPES) {
-		IPAERR("invalid pipe idx %d\n", pipe_idx);
-		return false;
-	}
-
-	return ipa3_ctx->ipacm_client[pipe_idx].uplink;
-}
-
-
-/**
- * ipa3_get_client_mapping() - provide client mapping
- * @pipe_idx: IPA end-point number
- *
- * Return value: client mapping
- */
-enum ipa_client_type ipa3_get_client_mapping(int pipe_idx)
-{
-	if (pipe_idx >= ipa3_ctx->ipa_num_pipes || pipe_idx < 0) {
-		IPAERR("Bad pipe index!\n");
-		WARN_ON(1);
-		return -EINVAL;
-	}
-
-	return ipa3_ctx->ep[pipe_idx].client;
-}
-
-/**
- * ipa3_get_client_by_pipe() - return client type relative to pipe
- * index
- * @pipe_idx: IPA end-point number
- *
- * Return value: client type
- */
-enum ipa_client_type ipa3_get_client_by_pipe(int pipe_idx)
-{
-	int j = 0;
-
-	for (j = 0; j < IPA_CLIENT_MAX; j++) {
-		const struct ipa_ep_configuration *iec_ptr =
-			&(ipa3_ep_mapping[ipa3_get_hw_type_index()][j]);
-		if (iec_ptr->valid &&
-		    iec_ptr->ipa_gsi_ep_info.ipa_ep_num == pipe_idx)
-			break;
-	}
-
-	if (j == IPA_CLIENT_MAX)
-		IPADBG("Got to IPA_CLIENT_MAX (%d) while searching for (%d)\n",
-		       j, pipe_idx);
-
-	return j;
-}
-
-/**
- * ipa_init_ep_flt_bitmap() - Initialize the bitmap
- * that represents the End-points that supports filtering
- */
-void ipa_init_ep_flt_bitmap(void)
-{
-	enum ipa_client_type cl;
-	u8 hw_idx = ipa3_get_hw_type_index();
-	u32 bitmap;
-	u32 pipe_num;
-	const struct ipa_gsi_ep_config *gsi_ep_ptr;
-
-	bitmap = 0;
-	if (ipa3_ctx->ep_flt_bitmap) {
-		WARN_ON(1);
-		return;
-	}
-
-	for (cl = 0; cl < IPA_CLIENT_MAX ; cl++) {
-		if (ipa3_ep_mapping[hw_idx][cl].support_flt) {
-			gsi_ep_ptr =
-				&ipa3_ep_mapping[hw_idx][cl].ipa_gsi_ep_info;
-			pipe_num =
-				gsi_ep_ptr->ipa_ep_num;
-			bitmap |= (1U << pipe_num);
-			if (bitmap != ipa3_ctx->ep_flt_bitmap) {
-				ipa3_ctx->ep_flt_bitmap = bitmap;
-				ipa3_ctx->ep_flt_num++;
-			}
-		}
-	}
-}
-
-/**
- * ipa_is_ep_support_flt() - Given an End-point check
- * whether it supports filtering or not.
- *
- * @pipe_idx:
- *
- * Return values:
- * true if supports and false if not
- */
-bool ipa_is_ep_support_flt(int pipe_idx)
-{
-	if (pipe_idx >= ipa3_ctx->ipa_num_pipes || pipe_idx < 0) {
-		IPAERR("Bad pipe index!\n");
-		return false;
-	}
-
-	return ipa3_ctx->ep_flt_bitmap & (1U<<pipe_idx);
-}
-
-/**
- * ipa3_cfg_ep_seq() - IPA end-point HPS/DPS sequencer type configuration
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_cfg_ep_seq(u32 clnt_hdl, const struct ipa_ep_cfg_seq *seq_cfg)
-{
-	int type;
-
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-	    ipa3_ctx->ep[clnt_hdl].valid == 0) {
-		IPAERR("bad param, clnt_hdl = %d", clnt_hdl);
-		return -EINVAL;
-	}
-
-	if (IPA_CLIENT_IS_CONS(ipa3_ctx->ep[clnt_hdl].client)) {
-		IPAERR("SEQ does not apply to IPA consumer EP %d\n", clnt_hdl);
-		return -EINVAL;
-	}
-
-	/*
-	 * Skip Configure sequencers type for test clients.
-	 * These are configured dynamically in ipa3_cfg_ep_mode
-	 */
-	if (IPA_CLIENT_IS_TEST(ipa3_ctx->ep[clnt_hdl].client)) {
-		IPADBG("Skip sequencers configuration for test clients\n");
-		return 0;
-	}
-
-	if (seq_cfg->set_dynamic)
-		type = seq_cfg->seq_type;
-	else
-		type = ipa3_ep_mapping[ipa3_get_hw_type_index()]
-			[ipa3_ctx->ep[clnt_hdl].client].sequencer_type;
-
-	if (type != IPA_DPS_HPS_SEQ_TYPE_INVALID) {
-		if (ipa3_ctx->ep[clnt_hdl].cfg.mode.mode == IPA_DMA &&
-			!IPA_DPS_HPS_SEQ_TYPE_IS_DMA(type)) {
-			IPAERR("Configuring non-DMA SEQ type to DMA pipe\n");
-			WARN_ON(1);
-			return -EINVAL;
-		}
-		IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-		/* Configure sequencers type*/
-
-		IPADBG("set sequencers to sequence 0x%x, ep = %d\n", type,
-				clnt_hdl);
-		ipahal_write_reg_n(IPA_ENDP_INIT_SEQ_n, clnt_hdl, type);
-
-		IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-	} else {
-		IPADBG("should not set sequencer type of ep = %d\n", clnt_hdl);
-	}
-
-	return 0;
-}
-
-/**
- * ipa3_cfg_ep - IPA end-point configuration
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- * @ipa_ep_cfg:	[in] IPA end-point configuration params
- *
- * This includes nat, IPv6CT, header, mode, aggregation and route settings and
- * is a one shot API to configure the IPA end-point fully
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_cfg_ep(u32 clnt_hdl, const struct ipa_ep_cfg *ipa_ep_cfg)
-{
-	int result = -EINVAL;
-
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-	    ipa3_ctx->ep[clnt_hdl].valid == 0 || ipa_ep_cfg == NULL) {
-		IPAERR("bad parm.\n");
-		return -EINVAL;
-	}
-
-	result = ipa3_cfg_ep_hdr(clnt_hdl, &ipa_ep_cfg->hdr);
-	if (result)
-		return result;
-
-	result = ipa3_cfg_ep_hdr_ext(clnt_hdl, &ipa_ep_cfg->hdr_ext);
-	if (result)
-		return result;
-
-	result = ipa3_cfg_ep_aggr(clnt_hdl, &ipa_ep_cfg->aggr);
-	if (result)
-		return result;
-
-	result = ipa3_cfg_ep_cfg(clnt_hdl, &ipa_ep_cfg->cfg);
-	if (result)
-		return result;
-
-	if (IPA_CLIENT_IS_PROD(ipa3_ctx->ep[clnt_hdl].client)) {
-		result = ipa3_cfg_ep_nat(clnt_hdl, &ipa_ep_cfg->nat);
-		if (result)
-			return result;
-
-		if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0) {
-			result = ipa3_cfg_ep_conn_track(clnt_hdl,
-				&ipa_ep_cfg->conn_track);
-			if (result)
-				return result;
-		}
-
-		result = ipa3_cfg_ep_mode(clnt_hdl, &ipa_ep_cfg->mode);
-		if (result)
-			return result;
-
-		result = ipa3_cfg_ep_seq(clnt_hdl, &ipa_ep_cfg->seq);
-		if (result)
-			return result;
-
-		result = ipa3_cfg_ep_route(clnt_hdl, &ipa_ep_cfg->route);
-		if (result)
-			return result;
-
-		result = ipa3_cfg_ep_deaggr(clnt_hdl, &ipa_ep_cfg->deaggr);
-		if (result)
-			return result;
-	} else {
-		result = ipa3_cfg_ep_metadata_mask(clnt_hdl,
-				&ipa_ep_cfg->metadata_mask);
-		if (result)
-			return result;
-	}
-
-	return 0;
-}
-
-static const char *ipa3_get_nat_en_str(enum ipa_nat_en_type nat_en)
-{
-	switch (nat_en) {
-	case (IPA_BYPASS_NAT):
-		return "NAT disabled";
-	case (IPA_SRC_NAT):
-		return "Source NAT";
-	case (IPA_DST_NAT):
-		return "Dst NAT";
-	}
-
-	return "undefined";
-}
-
-static const char *ipa3_get_ipv6ct_en_str(enum ipa_ipv6ct_en_type ipv6ct_en)
-{
-	switch (ipv6ct_en) {
-	case (IPA_BYPASS_IPV6CT):
-		return "ipv6ct disabled";
-	case (IPA_ENABLE_IPV6CT):
-		return "ipv6ct enabled";
-	}
-
-	return "undefined";
-}
-
-/**
- * ipa3_cfg_ep_nat() - IPA end-point NAT configuration
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- * @ep_nat:	[in] IPA NAT end-point configuration params
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_cfg_ep_nat(u32 clnt_hdl, const struct ipa_ep_cfg_nat *ep_nat)
-{
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-	    ipa3_ctx->ep[clnt_hdl].valid == 0 || ep_nat == NULL) {
-		IPAERR("bad parm, clnt_hdl = %d , ep_valid = %d\n",
-					clnt_hdl,
-					ipa3_ctx->ep[clnt_hdl].valid);
-		return -EINVAL;
-	}
-
-	if (IPA_CLIENT_IS_CONS(ipa3_ctx->ep[clnt_hdl].client)) {
-		IPAERR("NAT does not apply to IPA out EP %d\n", clnt_hdl);
-		return -EINVAL;
-	}
-
-	IPADBG("pipe=%d, nat_en=%d(%s)\n",
-			clnt_hdl,
-			ep_nat->nat_en,
-			ipa3_get_nat_en_str(ep_nat->nat_en));
-
-	/* copy over EP cfg */
-	ipa3_ctx->ep[clnt_hdl].cfg.nat = *ep_nat;
-
-	IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	ipahal_write_reg_n_fields(IPA_ENDP_INIT_NAT_n, clnt_hdl, ep_nat);
-
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	return 0;
-}
-
-/**
- * ipa3_cfg_ep_conn_track() - IPA end-point IPv6CT configuration
- * @clnt_hdl:		[in] opaque client handle assigned by IPA to client
- * @ep_conn_track:	[in] IPA IPv6CT end-point configuration params
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_cfg_ep_conn_track(u32 clnt_hdl,
-	const struct ipa_ep_cfg_conn_track *ep_conn_track)
-{
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-		ipa3_ctx->ep[clnt_hdl].valid == 0 || ep_conn_track == NULL) {
-		IPAERR("bad parm, clnt_hdl = %d , ep_valid = %d\n",
-			clnt_hdl,
-			ipa3_ctx->ep[clnt_hdl].valid);
-		return -EINVAL;
-	}
-
-	if (IPA_CLIENT_IS_CONS(ipa3_ctx->ep[clnt_hdl].client)) {
-		IPAERR("IPv6CT does not apply to IPA out EP %d\n", clnt_hdl);
-		return -EINVAL;
-	}
-
-	IPADBG("pipe=%d, conn_track_en=%d(%s)\n",
-		clnt_hdl,
-		ep_conn_track->conn_track_en,
-		ipa3_get_ipv6ct_en_str(ep_conn_track->conn_track_en));
-
-	/* copy over EP cfg */
-	ipa3_ctx->ep[clnt_hdl].cfg.conn_track = *ep_conn_track;
-
-	IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	ipahal_write_reg_n_fields(IPA_ENDP_INIT_CONN_TRACK_n, clnt_hdl,
-		ep_conn_track);
-
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	return 0;
-}
-
-
-/**
- * ipa3_cfg_ep_status() - IPA end-point status configuration
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- * @ipa_ep_cfg:	[in] IPA end-point configuration params
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_cfg_ep_status(u32 clnt_hdl,
-	const struct ipahal_reg_ep_cfg_status *ep_status)
-{
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-	    ipa3_ctx->ep[clnt_hdl].valid == 0 || ep_status == NULL) {
-		IPAERR("bad parm, clnt_hdl = %d , ep_valid = %d\n",
-					clnt_hdl,
-					ipa3_ctx->ep[clnt_hdl].valid);
-		return -EINVAL;
-	}
-
-	IPADBG("pipe=%d, status_en=%d status_ep=%d status_location=%d\n",
-			clnt_hdl,
-			ep_status->status_en,
-			ep_status->status_ep,
-			ep_status->status_location);
-
-	/* copy over EP cfg */
-	ipa3_ctx->ep[clnt_hdl].status = *ep_status;
-
-	IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	ipahal_write_reg_n_fields(IPA_ENDP_STATUS_n, clnt_hdl, ep_status);
-
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	return 0;
-}
-
-/**
- * ipa3_cfg_ep_cfg() - IPA end-point cfg configuration
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- * @ipa_ep_cfg:	[in] IPA end-point configuration params
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_cfg_ep_cfg(u32 clnt_hdl, const struct ipa_ep_cfg_cfg *cfg)
-{
-	u8 qmb_master_sel;
-
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-	    ipa3_ctx->ep[clnt_hdl].valid == 0 || cfg == NULL) {
-		IPAERR("bad parm, clnt_hdl = %d , ep_valid = %d\n",
-					clnt_hdl,
-					ipa3_ctx->ep[clnt_hdl].valid);
-		return -EINVAL;
-	}
-
-	/* copy over EP cfg */
-	ipa3_ctx->ep[clnt_hdl].cfg.cfg = *cfg;
-
-	/* Override QMB master selection */
-	qmb_master_sel = ipa3_get_qmb_master_sel(ipa3_ctx->ep[clnt_hdl].client);
-	ipa3_ctx->ep[clnt_hdl].cfg.cfg.gen_qmb_master_sel = qmb_master_sel;
-	IPADBG(
-	       "pipe=%d, frag_ofld_en=%d cs_ofld_en=%d mdata_hdr_ofst=%d gen_qmb_master_sel=%d\n",
-			clnt_hdl,
-			ipa3_ctx->ep[clnt_hdl].cfg.cfg.frag_offload_en,
-			ipa3_ctx->ep[clnt_hdl].cfg.cfg.cs_offload_en,
-			ipa3_ctx->ep[clnt_hdl].cfg.cfg.cs_metadata_hdr_offset,
-			ipa3_ctx->ep[clnt_hdl].cfg.cfg.gen_qmb_master_sel);
-
-	IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	ipahal_write_reg_n_fields(IPA_ENDP_INIT_CFG_n, clnt_hdl,
-				  &ipa3_ctx->ep[clnt_hdl].cfg.cfg);
-
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	return 0;
-}
-
-/**
- * ipa3_cfg_ep_metadata_mask() - IPA end-point meta-data mask configuration
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- * @ipa_ep_cfg:	[in] IPA end-point configuration params
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_cfg_ep_metadata_mask(u32 clnt_hdl,
-		const struct ipa_ep_cfg_metadata_mask
-		*metadata_mask)
-{
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-	    ipa3_ctx->ep[clnt_hdl].valid == 0 || metadata_mask == NULL) {
-		IPAERR("bad parm, clnt_hdl = %d , ep_valid = %d\n",
-					clnt_hdl,
-					ipa3_ctx->ep[clnt_hdl].valid);
-		return -EINVAL;
-	}
-
-	IPADBG("pipe=%d, metadata_mask=0x%x\n",
-			clnt_hdl,
-			metadata_mask->metadata_mask);
-
-	/* copy over EP cfg */
-	ipa3_ctx->ep[clnt_hdl].cfg.metadata_mask = *metadata_mask;
-
-	IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	ipahal_write_reg_n_fields(IPA_ENDP_INIT_HDR_METADATA_MASK_n,
-		clnt_hdl, metadata_mask);
-
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	return 0;
-}
-
-/**
- * ipa3_cfg_ep_hdr() -  IPA end-point header configuration
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- * @ipa_ep_cfg:	[in] IPA end-point configuration params
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_cfg_ep_hdr(u32 clnt_hdl, const struct ipa_ep_cfg_hdr *ep_hdr)
-{
-	struct ipa3_ep_context *ep;
-
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-	    ipa3_ctx->ep[clnt_hdl].valid == 0 || ep_hdr == NULL) {
-		IPAERR("bad parm, clnt_hdl = %d , ep_valid = %d\n",
-				clnt_hdl, ipa3_ctx->ep[clnt_hdl].valid);
-		return -EINVAL;
-	}
-	IPADBG("pipe=%d metadata_reg_valid=%d\n",
-		clnt_hdl,
-		ep_hdr->hdr_metadata_reg_valid);
-
-	IPADBG("remove_additional=%d, a5_mux=%d, ofst_pkt_size=0x%x\n",
-		ep_hdr->hdr_remove_additional,
-		ep_hdr->hdr_a5_mux,
-		ep_hdr->hdr_ofst_pkt_size);
-
-	IPADBG("ofst_pkt_size_valid=%d, additional_const_len=0x%x\n",
-		ep_hdr->hdr_ofst_pkt_size_valid,
-		ep_hdr->hdr_additional_const_len);
-
-	IPADBG("ofst_metadata=0x%x, ofst_metadata_valid=%d, len=0x%x\n",
-		ep_hdr->hdr_ofst_metadata,
-		ep_hdr->hdr_ofst_metadata_valid,
-		ep_hdr->hdr_len);
-
-	ep = &ipa3_ctx->ep[clnt_hdl];
-
-	/* copy over EP cfg */
-	ep->cfg.hdr = *ep_hdr;
-
-	IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	ipahal_write_reg_n_fields(IPA_ENDP_INIT_HDR_n, clnt_hdl, &ep->cfg.hdr);
-
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	return 0;
-}
-
-/**
- * ipa3_cfg_ep_hdr_ext() -  IPA end-point extended header configuration
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- * @ep_hdr_ext:	[in] IPA end-point configuration params
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_cfg_ep_hdr_ext(u32 clnt_hdl,
-		       const struct ipa_ep_cfg_hdr_ext *ep_hdr_ext)
-{
-	struct ipa3_ep_context *ep;
-
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-	    ipa3_ctx->ep[clnt_hdl].valid == 0 || ep_hdr_ext == NULL) {
-		IPAERR("bad parm, clnt_hdl = %d , ep_valid = %d\n",
-				clnt_hdl, ipa3_ctx->ep[clnt_hdl].valid);
-		return -EINVAL;
-	}
-
-	IPADBG("pipe=%d hdr_pad_to_alignment=%d\n",
-		clnt_hdl,
-		ep_hdr_ext->hdr_pad_to_alignment);
-
-	IPADBG("hdr_total_len_or_pad_offset=%d\n",
-		ep_hdr_ext->hdr_total_len_or_pad_offset);
-
-	IPADBG("hdr_payload_len_inc_padding=%d hdr_total_len_or_pad=%d\n",
-		ep_hdr_ext->hdr_payload_len_inc_padding,
-		ep_hdr_ext->hdr_total_len_or_pad);
-
-	IPADBG("hdr_total_len_or_pad_valid=%d hdr_little_endian=%d\n",
-		ep_hdr_ext->hdr_total_len_or_pad_valid,
-		ep_hdr_ext->hdr_little_endian);
-
-	ep = &ipa3_ctx->ep[clnt_hdl];
-
-	/* copy over EP cfg */
-	ep->cfg.hdr_ext = *ep_hdr_ext;
-	ep->cfg.hdr_ext.hdr = &ep->cfg.hdr;
-
-	IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	ipahal_write_reg_n_fields(IPA_ENDP_INIT_HDR_EXT_n, clnt_hdl,
-		&ep->cfg.hdr_ext);
-
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	return 0;
-}
-
-/**
- * ipa3_cfg_ep_ctrl() -  IPA end-point Control configuration
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- * @ipa_ep_cfg_ctrl:	[in] IPA end-point configuration params
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa3_cfg_ep_ctrl(u32 clnt_hdl, const struct ipa_ep_cfg_ctrl *ep_ctrl)
-{
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes || ep_ctrl == NULL) {
-		IPAERR("bad parm, clnt_hdl = %d\n", clnt_hdl);
-		return -EINVAL;
-	}
-
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0 && ep_ctrl->ipa_ep_suspend) {
-		IPAERR("pipe suspend is not supported\n");
-		WARN_ON(1);
-		return -EPERM;
-	}
-
-	if (ipa3_ctx->ipa_endp_delay_wa) {
-		IPAERR("pipe setting delay is not supported\n");
-		return 0;
-	}
-
-	IPADBG("pipe=%d ep_suspend=%d, ep_delay=%d\n",
-		clnt_hdl,
-		ep_ctrl->ipa_ep_suspend,
-		ep_ctrl->ipa_ep_delay);
-
-	ipahal_write_reg_n_fields(IPA_ENDP_INIT_CTRL_n, clnt_hdl, ep_ctrl);
-
-	if (ep_ctrl->ipa_ep_suspend == true &&
-			IPA_CLIENT_IS_CONS(ipa3_ctx->ep[clnt_hdl].client))
-		ipa3_suspend_active_aggr_wa(clnt_hdl);
-
-	return 0;
-}
-
-const char *ipa3_get_mode_type_str(enum ipa_mode_type mode)
-{
-	switch (mode) {
-	case (IPA_BASIC):
-		return "Basic";
-	case (IPA_ENABLE_FRAMING_HDLC):
-		return "HDLC framing";
-	case (IPA_ENABLE_DEFRAMING_HDLC):
-		return "HDLC de-framing";
-	case (IPA_DMA):
-		return "DMA";
-	}
-
-	return "undefined";
-}
-
-/**
- * ipa3_cfg_ep_mode() - IPA end-point mode configuration
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- * @ipa_ep_cfg:	[in] IPA end-point configuration params
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_cfg_ep_mode(u32 clnt_hdl, const struct ipa_ep_cfg_mode *ep_mode)
-{
-	int ep;
-	int type;
-	struct ipahal_reg_endp_init_mode init_mode;
-
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-	    ipa3_ctx->ep[clnt_hdl].valid == 0 || ep_mode == NULL) {
-		IPAERR("bad params clnt_hdl=%d , ep_valid=%d ep_mode=%pK\n",
-				clnt_hdl, ipa3_ctx->ep[clnt_hdl].valid,
-				ep_mode);
-		return -EINVAL;
-	}
-
-	if (IPA_CLIENT_IS_CONS(ipa3_ctx->ep[clnt_hdl].client)) {
-		IPAERR("MODE does not apply to IPA out EP %d\n", clnt_hdl);
-		return -EINVAL;
-	}
-
-	ep = ipa3_get_ep_mapping(ep_mode->dst);
-	if (ep == -1 && ep_mode->mode == IPA_DMA) {
-		IPAERR("dst %d does not exist in DMA mode\n", ep_mode->dst);
-		return -EINVAL;
-	}
-
-	WARN_ON(ep_mode->mode == IPA_DMA && IPA_CLIENT_IS_PROD(ep_mode->dst));
-
-	if (!IPA_CLIENT_IS_CONS(ep_mode->dst))
-		ep = ipa3_get_ep_mapping(IPA_CLIENT_APPS_LAN_CONS);
-
-	IPADBG("pipe=%d mode=%d(%s), dst_client_number=%d\n",
-			clnt_hdl,
-			ep_mode->mode,
-			ipa3_get_mode_type_str(ep_mode->mode),
-			ep_mode->dst);
-
-	/* copy over EP cfg */
-	ipa3_ctx->ep[clnt_hdl].cfg.mode = *ep_mode;
-	ipa3_ctx->ep[clnt_hdl].dst_pipe_index = ep;
-
-	IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	init_mode.dst_pipe_number = ipa3_ctx->ep[clnt_hdl].dst_pipe_index;
-	init_mode.ep_mode = *ep_mode;
-	ipahal_write_reg_n_fields(IPA_ENDP_INIT_MODE_n, clnt_hdl, &init_mode);
-
-	 /* Configure sequencers type for test clients*/
-	if (IPA_CLIENT_IS_TEST(ipa3_ctx->ep[clnt_hdl].client)) {
-		if (ep_mode->mode == IPA_DMA)
-			type = IPA_DPS_HPS_SEQ_TYPE_DMA_ONLY;
-		else
-			/* In IPA4.2 only single pass only supported*/
-			if (ipa3_ctx->ipa_hw_type == IPA_HW_v4_2)
-				type =
-				IPA_DPS_HPS_SEQ_TYPE_PKT_PROCESS_NO_DEC_NO_UCP;
-			else
-				type =
-			IPA_DPS_HPS_SEQ_TYPE_2ND_PKT_PROCESS_PASS_NO_DEC_UCP;
-
-		IPADBG(" set sequencers to sequance 0x%x, ep = %d\n", type,
-				clnt_hdl);
-		ipahal_write_reg_n(IPA_ENDP_INIT_SEQ_n, clnt_hdl, type);
-	}
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	return 0;
-}
-
-const char *ipa3_get_aggr_enable_str(enum ipa_aggr_en_type aggr_en)
-{
-	switch (aggr_en) {
-	case (IPA_BYPASS_AGGR):
-			return "no aggregation";
-	case (IPA_ENABLE_AGGR):
-			return "aggregation enabled";
-	case (IPA_ENABLE_DEAGGR):
-		return "de-aggregation enabled";
-	}
-
-	return "undefined";
-}
-
-const char *ipa3_get_aggr_type_str(enum ipa_aggr_type aggr_type)
-{
-	switch (aggr_type) {
-	case (IPA_MBIM_16):
-			return "MBIM_16";
-	case (IPA_HDLC):
-		return "HDLC";
-	case (IPA_TLP):
-			return "TLP";
-	case (IPA_RNDIS):
-			return "RNDIS";
-	case (IPA_GENERIC):
-			return "GENERIC";
-	case (IPA_QCMAP):
-			return "QCMAP";
-	case (IPA_COALESCE):
-			return "COALESCE";
-	}
-	return "undefined";
-}
-
-static u32 ipa3_time_gran_usec_step(enum ipa_timers_time_gran_type gran)
-{
-	switch (gran) {
-	case IPA_TIMERS_TIME_GRAN_10_USEC:		return 10;
-	case IPA_TIMERS_TIME_GRAN_20_USEC:		return 20;
-	case IPA_TIMERS_TIME_GRAN_50_USEC:		return 50;
-	case IPA_TIMERS_TIME_GRAN_100_USEC:		return 100;
-	case IPA_TIMERS_TIME_GRAN_1_MSEC:		return 1000;
-	case IPA_TIMERS_TIME_GRAN_10_MSEC:		return 10000;
-	case IPA_TIMERS_TIME_GRAN_100_MSEC:		return 100000;
-	case IPA_TIMERS_TIME_GRAN_NEAR_HALF_SEC:	return 655350;
-	default:
-		IPAERR("Invalid granularity time unit %d\n", gran);
-		ipa_assert();
-		break;
-	}
-
-	return 100;
-}
-
-/*
- * ipa3_process_timer_cfg() - Check and produce timer config
- *
- * Relevant for IPA 4.5 and above
- *
- * Assumes clocks are voted
- */
-static int ipa3_process_timer_cfg(u32 time_us,
-	u8 *pulse_gen, u8 *time_units)
-{
-	struct ipahal_reg_timers_pulse_gran_cfg gran_cfg;
-	u32 gran0_step, gran1_step;
-
-	IPADBG("time in usec=%u\n", time_us);
-
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_5) {
-		IPAERR("Invalid IPA version %d\n", ipa3_ctx->ipa_hw_type);
-		return -EPERM;
-	}
-
-	if (!time_us) {
-		*pulse_gen = 0;
-		*time_units = 0;
-		return 0;
-	}
-
-	ipahal_read_reg_fields(IPA_TIMERS_PULSE_GRAN_CFG, &gran_cfg);
-
-	gran0_step = ipa3_time_gran_usec_step(gran_cfg.gran_0);
-	gran1_step = ipa3_time_gran_usec_step(gran_cfg.gran_1);
-	/* gran_2 is not used by AP */
-
-	IPADBG("gran0 usec step=%u  gran1 usec step=%u\n",
-		gran0_step, gran1_step);
-
-	/* Lets try pulse generator #0 granularity */
-	if (!(time_us % gran0_step)) {
-		if ((time_us / gran0_step) <= IPA_TIMER_SCALED_TIME_LIMIT) {
-			*pulse_gen = 0;
-			*time_units = time_us / gran0_step;
-			IPADBG("Matched: generator=0, units=%u\n",
-				*time_units);
-			return 0;
-		}
-		IPADBG("gran0 cannot be used due to range limit\n");
-	}
-
-	/* Lets try pulse generator #1 granularity */
-	if (!(time_us % gran1_step)) {
-		if ((time_us / gran1_step) <= IPA_TIMER_SCALED_TIME_LIMIT) {
-			*pulse_gen = 1;
-			*time_units = time_us / gran1_step;
-			IPADBG("Matched: generator=1, units=%u\n",
-				*time_units);
-			return 0;
-		}
-		IPADBG("gran1 cannot be used due to range limit\n");
-	}
-
-	IPAERR("Cannot match requested time to configured granularities\n");
-	return -EPERM;
-}
-
-/**
- * ipa3_cfg_ep_aggr() - IPA end-point aggregation configuration
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- * @ipa_ep_cfg:	[in] IPA end-point configuration params
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_cfg_ep_aggr(u32 clnt_hdl, const struct ipa_ep_cfg_aggr *ep_aggr)
-{
-	int res = 0;
-
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-	    ipa3_ctx->ep[clnt_hdl].valid == 0 || ep_aggr == NULL) {
-		IPAERR("bad parm, clnt_hdl = %d , ep_valid = %d\n",
-			clnt_hdl, ipa3_ctx->ep[clnt_hdl].valid);
-		return -EINVAL;
-	}
-
-	if (ep_aggr->aggr_en == IPA_ENABLE_DEAGGR &&
-	    !IPA_EP_SUPPORTS_DEAGGR(clnt_hdl)) {
-		IPAERR("pipe=%d cannot be configured to DEAGGR\n", clnt_hdl);
-		WARN_ON(1);
-		return -EINVAL;
-	}
-
-	IPADBG("pipe=%d en=%d(%s), type=%d(%s), byte_limit=%d, time_limit=%d\n",
-			clnt_hdl,
-			ep_aggr->aggr_en,
-			ipa3_get_aggr_enable_str(ep_aggr->aggr_en),
-			ep_aggr->aggr,
-			ipa3_get_aggr_type_str(ep_aggr->aggr),
-			ep_aggr->aggr_byte_limit,
-			ep_aggr->aggr_time_limit);
-	IPADBG("hard_byte_limit_en=%d aggr_sw_eof_active=%d\n",
-		ep_aggr->aggr_hard_byte_limit_en,
-		ep_aggr->aggr_sw_eof_active);
-
-	/* copy over EP cfg */
-	ipa3_ctx->ep[clnt_hdl].cfg.aggr = *ep_aggr;
-
-	IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_5) {
-		res = ipa3_process_timer_cfg(ep_aggr->aggr_time_limit,
-			&ipa3_ctx->ep[clnt_hdl].cfg.aggr.pulse_generator,
-			&ipa3_ctx->ep[clnt_hdl].cfg.aggr.scaled_time);
-		if (res) {
-			IPAERR("failed to process AGGR timer tmr=%u\n",
-				ep_aggr->aggr_time_limit);
-			ipa_assert();
-			res = -EINVAL;
-			goto complete;
-		}
-		/*
-		 * HW bug on IPA4.5 where gran is used from pipe 0 instead of
-		 * coal pipe. Add this check to make sure that RSC pipe will use
-		 * gran 0 per the requested time needed; pipe 0 will use always
-		 * gran 0 as gran 0 is the POR value of it and s/w never change
-		 * it.
-		 */
-		if (ipa3_ctx->ipa_hw_type == IPA_HW_v4_5 &&
-		    ipa3_get_client_mapping(clnt_hdl) ==
-		    IPA_CLIENT_APPS_WAN_COAL_CONS &&
-		    ipa3_ctx->ep[clnt_hdl].cfg.aggr.pulse_generator != 0) {
-			IPAERR("coal pipe using GRAN_SEL = %d\n",
-			       ipa3_ctx->ep[clnt_hdl].cfg.aggr.pulse_generator);
-			ipa_assert();
-		}
-	} else {
-		/*
-		 * Global aggregation granularity is 0.5msec.
-		 * So if H/W programmed with 1msec, it will be
-		 *  0.5msec defacto.
-		 * So finest granularity is 0.5msec
-		 */
-		if (ep_aggr->aggr_time_limit % 500) {
-			IPAERR("given time limit %u is not in 0.5msec\n",
-				ep_aggr->aggr_time_limit);
-			WARN_ON(1);
-			res = -EINVAL;
-			goto complete;
-		}
-
-		/* Due to described above global granularity */
-		ipa3_ctx->ep[clnt_hdl].cfg.aggr.aggr_time_limit *= 2;
-	}
-
-	ipahal_write_reg_n_fields(IPA_ENDP_INIT_AGGR_n, clnt_hdl,
-			&ipa3_ctx->ep[clnt_hdl].cfg.aggr);
-complete:
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-	return res;
-}
-
-/**
- * ipa3_cfg_ep_route() - IPA end-point routing configuration
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- * @ipa_ep_cfg:	[in] IPA end-point configuration params
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_cfg_ep_route(u32 clnt_hdl, const struct ipa_ep_cfg_route *ep_route)
-{
-	struct ipahal_reg_endp_init_route init_rt;
-
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-	    ipa3_ctx->ep[clnt_hdl].valid == 0 || ep_route == NULL) {
-		IPAERR("bad parm, clnt_hdl = %d , ep_valid = %d\n",
-			clnt_hdl, ipa3_ctx->ep[clnt_hdl].valid);
-		return -EINVAL;
-	}
-
-	if (IPA_CLIENT_IS_CONS(ipa3_ctx->ep[clnt_hdl].client)) {
-		IPAERR("ROUTE does not apply to IPA out EP %d\n",
-				clnt_hdl);
-		return -EINVAL;
-	}
-
-	/*
-	 * if DMA mode was configured previously for this EP, return with
-	 * success
-	 */
-	if (ipa3_ctx->ep[clnt_hdl].cfg.mode.mode == IPA_DMA) {
-		IPADBG("DMA enabled for ep %d, dst pipe is part of DMA\n",
-				clnt_hdl);
-		return 0;
-	}
-
-	if (ep_route->rt_tbl_hdl)
-		IPAERR("client specified non-zero RT TBL hdl - ignore it\n");
-
-	IPADBG("pipe=%d, rt_tbl_hdl=%d\n",
-			clnt_hdl,
-			ep_route->rt_tbl_hdl);
-
-	/* always use "default" routing table when programming EP ROUTE reg */
-	ipa3_ctx->ep[clnt_hdl].rt_tbl_idx =
-		IPA_MEM_PART(v4_apps_rt_index_lo);
-
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_0) {
-		IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-		init_rt.route_table_index = ipa3_ctx->ep[clnt_hdl].rt_tbl_idx;
-		ipahal_write_reg_n_fields(IPA_ENDP_INIT_ROUTE_n,
-			clnt_hdl, &init_rt);
-
-		IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-	}
-
-	return 0;
-}
-
-#define MAX_ALLOWED_BASE_VAL 0x1f
-#define MAX_ALLOWED_SCALE_VAL 0x1f
-
-/**
- * ipa3_cal_ep_holb_scale_base_val - calculate base and scale value from tmr_val
- *
- * In IPA4.2 HW version need configure base and scale value in HOL timer reg
- * @tmr_val: [in] timer value for HOL timer
- * @ipa_ep_cfg: [out] Fill IPA end-point configuration base and scale value
- *			and return
- */
-void ipa3_cal_ep_holb_scale_base_val(u32 tmr_val,
-				struct ipa_ep_cfg_holb *ep_holb)
-{
-	u32 base_val, scale, scale_val = 1, base = 2;
-
-	for (scale = 0; scale <= MAX_ALLOWED_SCALE_VAL; scale++) {
-		base_val = tmr_val/scale_val;
-		if (scale != 0)
-			scale_val *= base;
-		if (base_val <= MAX_ALLOWED_BASE_VAL)
-			break;
-	}
-	ep_holb->base_val = base_val;
-	ep_holb->scale = scale_val;
-
-}
-
-/**
- * ipa3_cfg_ep_holb() - IPA end-point holb configuration
- *
- * If an IPA producer pipe is full, IPA HW by default will block
- * indefinitely till space opens up. During this time no packets
- * including those from unrelated pipes will be processed. Enabling
- * HOLB means IPA HW will be allowed to drop packets as/when needed
- * and indefinite blocking is avoided.
- *
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- * @ipa_ep_cfg:	[in] IPA end-point configuration params
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa3_cfg_ep_holb(u32 clnt_hdl, const struct ipa_ep_cfg_holb *ep_holb)
-{
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-	    ipa3_ctx->ep[clnt_hdl].valid == 0 || ep_holb == NULL ||
-	    ep_holb->tmr_val > ipa3_ctx->ctrl->max_holb_tmr_val ||
-	    ep_holb->en > 1) {
-		IPAERR("bad parm.\n");
-		return -EINVAL;
-	}
-
-	if (IPA_CLIENT_IS_PROD(ipa3_ctx->ep[clnt_hdl].client)) {
-		IPAERR("HOLB does not apply to IPA in EP %d\n", clnt_hdl);
-		return -EINVAL;
-	}
-
-	ipa3_ctx->ep[clnt_hdl].holb = *ep_holb;
-
-	IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	ipahal_write_reg_n_fields(IPA_ENDP_INIT_HOL_BLOCK_EN_n, clnt_hdl,
-		ep_holb);
-
-	/* IPA4.5 issue requires HOLB_EN to be written twice */
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_5)
-		ipahal_write_reg_n_fields(IPA_ENDP_INIT_HOL_BLOCK_EN_n,
-			clnt_hdl, ep_holb);
-
-	/* Configure timer */
-	if (ipa3_ctx->ipa_hw_type == IPA_HW_v4_2) {
-		ipa3_cal_ep_holb_scale_base_val(ep_holb->tmr_val,
-				&ipa3_ctx->ep[clnt_hdl].holb);
-		goto success;
-	}
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_5) {
-		int res;
-
-		res = ipa3_process_timer_cfg(ep_holb->tmr_val * 1000,
-			&ipa3_ctx->ep[clnt_hdl].holb.pulse_generator,
-			&ipa3_ctx->ep[clnt_hdl].holb.scaled_time);
-		if (res) {
-			IPAERR("failed to process HOLB timer tmr=%u\n",
-				ep_holb->tmr_val);
-			ipa_assert();
-			return res;
-		}
-	}
-
-success:
-	ipahal_write_reg_n_fields(IPA_ENDP_INIT_HOL_BLOCK_TIMER_n,
-		clnt_hdl, &ipa3_ctx->ep[clnt_hdl].holb);
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-	IPADBG("cfg holb %u ep=%d tmr=%d\n", ep_holb->en, clnt_hdl,
-		ep_holb->tmr_val);
-	return 0;
-}
-
-/**
- * ipa3_cfg_ep_holb_by_client() - IPA end-point holb configuration
- *
- * Wrapper function for ipa3_cfg_ep_holb() with client name instead of
- * client handle. This function is used for clients that does not have
- * client handle.
- *
- * @client:	[in] client name
- * @ipa_ep_cfg:	[in] IPA end-point configuration params
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa3_cfg_ep_holb_by_client(enum ipa_client_type client,
-				const struct ipa_ep_cfg_holb *ep_holb)
-{
-	return ipa3_cfg_ep_holb(ipa3_get_ep_mapping(client), ep_holb);
-}
-
-/**
- * ipa3_cfg_ep_deaggr() -  IPA end-point deaggregation configuration
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- * @ep_deaggr:	[in] IPA end-point configuration params
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_cfg_ep_deaggr(u32 clnt_hdl,
-			const struct ipa_ep_cfg_deaggr *ep_deaggr)
-{
-	struct ipa3_ep_context *ep;
-
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-	    ipa3_ctx->ep[clnt_hdl].valid == 0 || ep_deaggr == NULL) {
-		IPAERR("bad parm, clnt_hdl = %d , ep_valid = %d\n",
-				clnt_hdl, ipa3_ctx->ep[clnt_hdl].valid);
-		return -EINVAL;
-	}
-
-	IPADBG("pipe=%d deaggr_hdr_len=%d\n",
-		clnt_hdl,
-		ep_deaggr->deaggr_hdr_len);
-
-	IPADBG("packet_offset_valid=%d\n",
-		ep_deaggr->packet_offset_valid);
-
-	IPADBG("packet_offset_location=%d max_packet_len=%d\n",
-		ep_deaggr->packet_offset_location,
-		ep_deaggr->max_packet_len);
-
-	ep = &ipa3_ctx->ep[clnt_hdl];
-
-	/* copy over EP cfg */
-	ep->cfg.deaggr = *ep_deaggr;
-
-	IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	ipahal_write_reg_n_fields(IPA_ENDP_INIT_DEAGGR_n, clnt_hdl,
-		&ep->cfg.deaggr);
-
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	return 0;
-}
-
-/**
- * ipa3_cfg_ep_metadata() - IPA end-point metadata configuration
- * @clnt_hdl:	[in] opaque client handle assigned by IPA to client
- * @ipa_ep_cfg:	[in] IPA end-point configuration params
- *
- * Returns:	0 on success, negative on failure
- *
- * Note:	Should not be called from atomic context
- */
-int ipa3_cfg_ep_metadata(u32 clnt_hdl, const struct ipa_ep_cfg_metadata *ep_md)
-{
-	u32 qmap_id = 0;
-	struct ipa_ep_cfg_metadata ep_md_reg_wrt;
-
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-		ipa3_ctx->ep[clnt_hdl].valid == 0 || ep_md == NULL) {
-		IPAERR("bad parm, clnt_hdl = %d , ep_valid = %d\n",
-					clnt_hdl, ipa3_ctx->ep[clnt_hdl].valid);
-		return -EINVAL;
-	}
-
-	IPADBG("pipe=%d, mux id=%d\n", clnt_hdl, ep_md->qmap_id);
-
-	/* copy over EP cfg */
-	ipa3_ctx->ep[clnt_hdl].cfg.meta = *ep_md;
-
-	IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	ep_md_reg_wrt = *ep_md;
-	qmap_id = (ep_md->qmap_id <<
-		IPA_ENDP_INIT_HDR_METADATA_n_MUX_ID_SHFT) &
-		IPA_ENDP_INIT_HDR_METADATA_n_MUX_ID_BMASK;
-
-	/* mark tethering bit for remote modem */
-	if (ipa3_ctx->ipa_hw_type == IPA_HW_v4_1)
-		qmap_id |= IPA_QMAP_TETH_BIT;
-
-	ep_md_reg_wrt.qmap_id = qmap_id;
-	ipahal_write_reg_n_fields(IPA_ENDP_INIT_HDR_METADATA_n, clnt_hdl,
-		&ep_md_reg_wrt);
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_5) {
-		ipa3_ctx->ep[clnt_hdl].cfg.hdr.hdr_metadata_reg_valid = 1;
-		ipahal_write_reg_n_fields(IPA_ENDP_INIT_HDR_n, clnt_hdl,
-			&ipa3_ctx->ep[clnt_hdl].cfg.hdr);
-	}
-
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	return 0;
-}
-
-int ipa3_write_qmap_id(struct ipa_ioc_write_qmapid *param_in)
-{
-	struct ipa_ep_cfg_metadata meta;
-	struct ipa3_ep_context *ep;
-	int ipa_ep_idx;
-	int result = -EINVAL;
-
-	if (param_in->client  >= IPA_CLIENT_MAX) {
-		IPAERR_RL("bad parm client:%d\n", param_in->client);
-		goto fail;
-	}
-
-	ipa_ep_idx = ipa3_get_ep_mapping(param_in->client);
-	if (ipa_ep_idx == -1) {
-		IPAERR_RL("Invalid client.\n");
-		goto fail;
-	}
-
-	ep = &ipa3_ctx->ep[ipa_ep_idx];
-	if (!ep->valid) {
-		IPAERR_RL("EP not allocated.\n");
-		goto fail;
-	}
-
-	meta.qmap_id = param_in->qmap_id;
-	if (param_in->client == IPA_CLIENT_USB_PROD ||
-	    param_in->client == IPA_CLIENT_HSIC1_PROD ||
-	    param_in->client == IPA_CLIENT_ODU_PROD ||
-	    param_in->client == IPA_CLIENT_ETHERNET_PROD ||
-		param_in->client == IPA_CLIENT_WIGIG_PROD) {
-		result = ipa3_cfg_ep_metadata(ipa_ep_idx, &meta);
-	} else if (param_in->client == IPA_CLIENT_WLAN1_PROD ||
-			   param_in->client == IPA_CLIENT_WLAN2_PROD) {
-		ipa3_ctx->ep[ipa_ep_idx].cfg.meta = meta;
-		if (param_in->client == IPA_CLIENT_WLAN2_PROD)
-			result = ipa3_write_qmapid_wdi3_gsi_pipe(
-				ipa_ep_idx, meta.qmap_id);
-		else
-			result = ipa3_write_qmapid_wdi_pipe(
-				ipa_ep_idx, meta.qmap_id);
-		if (result)
-			IPAERR_RL("qmap_id %d write failed on ep=%d\n",
-					meta.qmap_id, ipa_ep_idx);
-		result = 0;
-	}
-
-fail:
-	return result;
-}
-
-/**
- * ipa3_dump_buff_internal() - dumps buffer for debug purposes
- * @base: buffer base address
- * @phy_base: buffer physical base address
- * @size: size of the buffer
- */
-void ipa3_dump_buff_internal(void *base, dma_addr_t phy_base, u32 size)
-{
-	int i;
-	u32 *cur = (u32 *)base;
-	u8 *byt;
-
-	IPADBG("system phys addr=%pa len=%u\n", &phy_base, size);
-	for (i = 0; i < size / 4; i++) {
-		byt = (u8 *)(cur + i);
-		IPADBG("%2d %08x   %02x %02x %02x %02x\n", i, *(cur + i),
-				byt[0], byt[1], byt[2], byt[3]);
-	}
-	IPADBG("END\n");
-}
-
-/**
- * ipa3_set_aggr_mode() - Set the aggregation mode which is a global setting
- * @mode:	[in] the desired aggregation mode for e.g. straight MBIM, QCNCM,
- * etc
- *
- * Returns:	0 on success
- */
-int ipa3_set_aggr_mode(enum ipa_aggr_mode mode)
-{
-	struct ipahal_reg_qcncm qcncm;
-
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0) {
-		if (mode != IPA_MBIM_AGGR) {
-			IPAERR("Only MBIM mode is supported staring 4.0\n");
-			return -EPERM;
-		}
-	} else {
-		IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-		ipahal_read_reg_fields(IPA_QCNCM, &qcncm);
-		qcncm.mode_en = mode;
-		ipahal_write_reg_fields(IPA_QCNCM, &qcncm);
-		IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-	}
-
-	return 0;
-}
-
-/**
- * ipa3_set_qcncm_ndp_sig() - Set the NDP signature used for QCNCM aggregation
- * mode
- * @sig:	[in] the first 3 bytes of QCNCM NDP signature (expected to be
- * "QND")
- *
- * Set the NDP signature used for QCNCM aggregation mode. The fourth byte
- * (expected to be 'P') needs to be set using the header addition mechanism
- *
- * Returns:	0 on success, negative on failure
- */
-int ipa3_set_qcncm_ndp_sig(char sig[3])
-{
-	struct ipahal_reg_qcncm qcncm;
-
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0) {
-		IPAERR("QCNCM mode is not supported staring 4.0\n");
-		return -EPERM;
-	}
-
-	if (sig == NULL) {
-		IPAERR("bad argument\n");
-		return -EINVAL;
-	}
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-	ipahal_read_reg_fields(IPA_QCNCM, &qcncm);
-	qcncm.mode_val = ((sig[0] << 16) | (sig[1] << 8) | sig[2]);
-	ipahal_write_reg_fields(IPA_QCNCM, &qcncm);
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-
-	return 0;
-}
-
-/**
- * ipa3_set_single_ndp_per_mbim() - Enable/disable single NDP per MBIM frame
- * configuration
- * @enable:	[in] true for single NDP/MBIM; false otherwise
- *
- * Returns:	0 on success
- */
-int ipa3_set_single_ndp_per_mbim(bool enable)
-{
-	struct ipahal_reg_single_ndp_mode mode;
-
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0) {
-		IPAERR("QCNCM mode is not supported staring 4.0\n");
-		return -EPERM;
-	}
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-	ipahal_read_reg_fields(IPA_SINGLE_NDP_MODE, &mode);
-	mode.single_ndp_en = enable;
-	ipahal_write_reg_fields(IPA_SINGLE_NDP_MODE, &mode);
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-
-	return 0;
-}
-
-/**
- * ipa3_straddle_boundary() - Checks whether a memory buffer straddles a
- * boundary
- * @start: start address of the memory buffer
- * @end: end address of the memory buffer
- * @boundary: boundary
- *
- * Return value:
- * 1: if the interval [start, end] straddles boundary
- * 0: otherwise
- */
-int ipa3_straddle_boundary(u32 start, u32 end, u32 boundary)
-{
-	u32 next_start;
-	u32 prev_end;
-
-	IPADBG("start=%u end=%u boundary=%u\n", start, end, boundary);
-
-	next_start = (start + (boundary - 1)) & ~(boundary - 1);
-	prev_end = ((end + (boundary - 1)) & ~(boundary - 1)) - boundary;
-
-	while (next_start < prev_end)
-		next_start += boundary;
-
-	if (next_start == prev_end)
-		return 1;
-	else
-		return 0;
-}
-
-/**
- * ipa3_init_mem_partition() - Assigns the static memory partition
- * based on the IPA version
- *
- * Returns:	0 on success
- */
-int ipa3_init_mem_partition(enum ipa_hw_type type)
-{
-	switch (type) {
-	case IPA_HW_v4_1:
-		ipa3_ctx->ctrl->mem_partition = &ipa_4_1_mem_part;
-		break;
-	case IPA_HW_v4_2:
-		ipa3_ctx->ctrl->mem_partition = &ipa_4_2_mem_part;
-		break;
-	case IPA_HW_v4_5:
-		ipa3_ctx->ctrl->mem_partition = &ipa_4_5_mem_part;
-		break;
-	case IPA_HW_v4_7:
-		ipa3_ctx->ctrl->mem_partition = &ipa_4_7_mem_part;
-		break;
-	case IPA_HW_None:
-	case IPA_HW_v1_0:
-	case IPA_HW_v1_1:
-	case IPA_HW_v2_0:
-	case IPA_HW_v2_1:
-	case IPA_HW_v2_5:
-	case IPA_HW_v2_6L:
-	case IPA_HW_v3_0:
-	case IPA_HW_v3_1:
-	case IPA_HW_v3_5:
-	case IPA_HW_v3_5_1:
-	case IPA_HW_v4_0:
-		IPAERR("unsupported version %d\n", type);
-		return -EPERM;
-	}
-
-	if (IPA_MEM_PART(uc_info_ofst) & 3) {
-		IPAERR("UC INFO OFST 0x%x is unaligned\n",
-			IPA_MEM_PART(uc_info_ofst));
-		return -ENODEV;
-	}
-
-	IPADBG("UC INFO OFST 0x%x SIZE 0x%x\n",
-		IPA_MEM_PART(uc_info_ofst), IPA_MEM_PART(uc_info_size));
-
-	IPADBG("RAM OFST 0x%x\n", IPA_MEM_PART(ofst_start));
-
-	if (IPA_MEM_PART(v4_flt_hash_ofst) & 7) {
-		IPAERR("V4 FLT HASHABLE OFST 0x%x is unaligned\n",
-			IPA_MEM_PART(v4_flt_hash_ofst));
-		return -ENODEV;
-	}
-
-	IPADBG("V4 FLT HASHABLE OFST 0x%x SIZE 0x%x DDR SIZE 0x%x\n",
-		IPA_MEM_PART(v4_flt_hash_ofst),
-		IPA_MEM_PART(v4_flt_hash_size),
-		IPA_MEM_PART(v4_flt_hash_size_ddr));
-
-	if (IPA_MEM_PART(v4_flt_nhash_ofst) & 7) {
-		IPAERR("V4 FLT NON-HASHABLE OFST 0x%x is unaligned\n",
-			IPA_MEM_PART(v4_flt_nhash_ofst));
-		return -ENODEV;
-	}
-
-	IPADBG("V4 FLT NON-HASHABLE OFST 0x%x SIZE 0x%x DDR SIZE 0x%x\n",
-		IPA_MEM_PART(v4_flt_nhash_ofst),
-		IPA_MEM_PART(v4_flt_nhash_size),
-		IPA_MEM_PART(v4_flt_nhash_size_ddr));
-
-	if (IPA_MEM_PART(v6_flt_hash_ofst) & 7) {
-		IPAERR("V6 FLT HASHABLE OFST 0x%x is unaligned\n",
-			IPA_MEM_PART(v6_flt_hash_ofst));
-		return -ENODEV;
-	}
-
-	IPADBG("V6 FLT HASHABLE OFST 0x%x SIZE 0x%x DDR SIZE 0x%x\n",
-		IPA_MEM_PART(v6_flt_hash_ofst), IPA_MEM_PART(v6_flt_hash_size),
-		IPA_MEM_PART(v6_flt_hash_size_ddr));
-
-	if (IPA_MEM_PART(v6_flt_nhash_ofst) & 7) {
-		IPAERR("V6 FLT NON-HASHABLE OFST 0x%x is unaligned\n",
-			IPA_MEM_PART(v6_flt_nhash_ofst));
-		return -ENODEV;
-	}
-
-	IPADBG("V6 FLT NON-HASHABLE OFST 0x%x SIZE 0x%x DDR SIZE 0x%x\n",
-		IPA_MEM_PART(v6_flt_nhash_ofst),
-		IPA_MEM_PART(v6_flt_nhash_size),
-		IPA_MEM_PART(v6_flt_nhash_size_ddr));
-
-	IPADBG("V4 RT NUM INDEX 0x%x\n", IPA_MEM_PART(v4_rt_num_index));
-
-	IPADBG("V4 RT MODEM INDEXES 0x%x - 0x%x\n",
-		IPA_MEM_PART(v4_modem_rt_index_lo),
-		IPA_MEM_PART(v4_modem_rt_index_hi));
-
-	IPADBG("V4 RT APPS INDEXES 0x%x - 0x%x\n",
-		IPA_MEM_PART(v4_apps_rt_index_lo),
-		IPA_MEM_PART(v4_apps_rt_index_hi));
-
-	if (IPA_MEM_PART(v4_rt_hash_ofst) & 7) {
-		IPAERR("V4 RT HASHABLE OFST 0x%x is unaligned\n",
-			IPA_MEM_PART(v4_rt_hash_ofst));
-		return -ENODEV;
-	}
-
-	IPADBG("V4 RT HASHABLE OFST 0x%x\n", IPA_MEM_PART(v4_rt_hash_ofst));
-
-	IPADBG("V4 RT HASHABLE SIZE 0x%x DDR SIZE 0x%x\n",
-		IPA_MEM_PART(v4_rt_hash_size),
-		IPA_MEM_PART(v4_rt_hash_size_ddr));
-
-	if (IPA_MEM_PART(v4_rt_nhash_ofst) & 7) {
-		IPAERR("V4 RT NON-HASHABLE OFST 0x%x is unaligned\n",
-			IPA_MEM_PART(v4_rt_nhash_ofst));
-		return -ENODEV;
-	}
-
-	IPADBG("V4 RT NON-HASHABLE OFST 0x%x\n",
-		IPA_MEM_PART(v4_rt_nhash_ofst));
-
-	IPADBG("V4 RT HASHABLE SIZE 0x%x DDR SIZE 0x%x\n",
-		IPA_MEM_PART(v4_rt_nhash_size),
-		IPA_MEM_PART(v4_rt_nhash_size_ddr));
-
-	IPADBG("V6 RT NUM INDEX 0x%x\n", IPA_MEM_PART(v6_rt_num_index));
-
-	IPADBG("V6 RT MODEM INDEXES 0x%x - 0x%x\n",
-		IPA_MEM_PART(v6_modem_rt_index_lo),
-		IPA_MEM_PART(v6_modem_rt_index_hi));
-
-	IPADBG("V6 RT APPS INDEXES 0x%x - 0x%x\n",
-		IPA_MEM_PART(v6_apps_rt_index_lo),
-		IPA_MEM_PART(v6_apps_rt_index_hi));
-
-	if (IPA_MEM_PART(v6_rt_hash_ofst) & 7) {
-		IPAERR("V6 RT HASHABLE OFST 0x%x is unaligned\n",
-			IPA_MEM_PART(v6_rt_hash_ofst));
-		return -ENODEV;
-	}
-
-	IPADBG("V6 RT HASHABLE OFST 0x%x\n", IPA_MEM_PART(v6_rt_hash_ofst));
-
-	IPADBG("V6 RT HASHABLE SIZE 0x%x DDR SIZE 0x%x\n",
-		IPA_MEM_PART(v6_rt_hash_size),
-		IPA_MEM_PART(v6_rt_hash_size_ddr));
-
-	if (IPA_MEM_PART(v6_rt_nhash_ofst) & 7) {
-		IPAERR("V6 RT NON-HASHABLE OFST 0x%x is unaligned\n",
-			IPA_MEM_PART(v6_rt_nhash_ofst));
-		return -ENODEV;
-	}
-
-	IPADBG("V6 RT NON-HASHABLE OFST 0x%x\n",
-		IPA_MEM_PART(v6_rt_nhash_ofst));
-
-	IPADBG("V6 RT NON-HASHABLE SIZE 0x%x DDR SIZE 0x%x\n",
-		IPA_MEM_PART(v6_rt_nhash_size),
-		IPA_MEM_PART(v6_rt_nhash_size_ddr));
-
-	if (IPA_MEM_PART(modem_hdr_ofst) & 7) {
-		IPAERR("MODEM HDR OFST 0x%x is unaligned\n",
-			IPA_MEM_PART(modem_hdr_ofst));
-		return -ENODEV;
-	}
-
-	IPADBG("MODEM HDR OFST 0x%x SIZE 0x%x\n",
-		IPA_MEM_PART(modem_hdr_ofst), IPA_MEM_PART(modem_hdr_size));
-
-	if (IPA_MEM_PART(apps_hdr_ofst) & 7) {
-		IPAERR("APPS HDR OFST 0x%x is unaligned\n",
-			IPA_MEM_PART(apps_hdr_ofst));
-		return -ENODEV;
-	}
-
-	IPADBG("APPS HDR OFST 0x%x SIZE 0x%x DDR SIZE 0x%x\n",
-		IPA_MEM_PART(apps_hdr_ofst), IPA_MEM_PART(apps_hdr_size),
-		IPA_MEM_PART(apps_hdr_size_ddr));
-
-	if (IPA_MEM_PART(modem_hdr_proc_ctx_ofst) & 7) {
-		IPAERR("MODEM HDR PROC CTX OFST 0x%x is unaligned\n",
-			IPA_MEM_PART(modem_hdr_proc_ctx_ofst));
-		return -ENODEV;
-	}
-
-	IPADBG("MODEM HDR PROC CTX OFST 0x%x SIZE 0x%x\n",
-		IPA_MEM_PART(modem_hdr_proc_ctx_ofst),
-		IPA_MEM_PART(modem_hdr_proc_ctx_size));
-
-	if (IPA_MEM_PART(apps_hdr_proc_ctx_ofst) & 7) {
-		IPAERR("APPS HDR PROC CTX OFST 0x%x is unaligned\n",
-			IPA_MEM_PART(apps_hdr_proc_ctx_ofst));
-		return -ENODEV;
-	}
-
-	IPADBG("APPS HDR PROC CTX OFST 0x%x SIZE 0x%x DDR SIZE 0x%x\n",
-		IPA_MEM_PART(apps_hdr_proc_ctx_ofst),
-		IPA_MEM_PART(apps_hdr_proc_ctx_size),
-		IPA_MEM_PART(apps_hdr_proc_ctx_size_ddr));
-
-	if (IPA_MEM_PART(pdn_config_ofst) & 7) {
-		IPAERR("PDN CONFIG OFST 0x%x is unaligned\n",
-			IPA_MEM_PART(pdn_config_ofst));
-		return -ENODEV;
-	}
-
-	/*
-	 * Routing rules points to hdr_proc_ctx in 32byte offsets from base.
-	 * Base is modem hdr_proc_ctx first address.
-	 * AP driver install APPS hdr_proc_ctx starting at the beginning of
-	 * apps hdr_proc_ctx part.
-	 * So first apps hdr_proc_ctx offset at some routing
-	 * rule will be modem_hdr_proc_ctx_size >> 5 (32B).
-	 */
-	if (IPA_MEM_PART(modem_hdr_proc_ctx_size) & 31) {
-		IPAERR("MODEM HDR PROC CTX SIZE 0x%x is not 32B aligned\n",
-			IPA_MEM_PART(modem_hdr_proc_ctx_size));
-		return -ENODEV;
-	}
-
-	/*
-	 * AP driver when installing routing rule, it calcs the hdr_proc_ctx
-	 * offset by local offset (from base of apps part) +
-	 * modem_hdr_proc_ctx_size. This is to get offset from modem part base.
-	 * Thus apps part must be adjacent to modem part
-	 */
-	if (IPA_MEM_PART(apps_hdr_proc_ctx_ofst) !=
-		IPA_MEM_PART(modem_hdr_proc_ctx_ofst) +
-		IPA_MEM_PART(modem_hdr_proc_ctx_size)) {
-		IPAERR("APPS HDR PROC CTX SIZE not adjacent to MODEM one!\n");
-		return -ENODEV;
-	}
-
-	IPADBG("NAT TBL OFST 0x%x SIZE 0x%x\n",
-		IPA_MEM_PART(nat_tbl_ofst),
-		IPA_MEM_PART(nat_tbl_size));
-
-	if (IPA_MEM_PART(nat_tbl_ofst) & 31) {
-		IPAERR("NAT TBL OFST 0x%x is unaligned\n",
-			IPA_MEM_PART(nat_tbl_ofst));
-		return -ENODEV;
-	}
-
-	IPADBG("NAT INDEX TBL OFST 0x%x SIZE 0x%x\n",
-		IPA_MEM_PART(nat_index_tbl_ofst),
-		IPA_MEM_PART(nat_index_tbl_size));
-
-	if (IPA_MEM_PART(nat_index_tbl_ofst) & 3) {
-		IPAERR("NAT INDEX TBL OFST 0x%x is unaligned\n",
-			IPA_MEM_PART(nat_index_tbl_ofst));
-		return -ENODEV;
-	}
-
-	IPADBG("NAT EXP TBL OFST 0x%x SIZE 0x%x\n",
-		IPA_MEM_PART(nat_exp_tbl_ofst),
-		IPA_MEM_PART(nat_exp_tbl_size));
-
-	if (IPA_MEM_PART(nat_exp_tbl_ofst) & 31) {
-		IPAERR("NAT EXP TBL OFST 0x%x is unaligned\n",
-			IPA_MEM_PART(nat_exp_tbl_ofst));
-		return -ENODEV;
-	}
-
-	IPADBG("PDN CONFIG OFST 0x%x SIZE 0x%x\n",
-		IPA_MEM_PART(pdn_config_ofst),
-		IPA_MEM_PART(pdn_config_size));
-
-	if (IPA_MEM_PART(pdn_config_ofst) & 7) {
-		IPAERR("PDN CONFIG OFST 0x%x is unaligned\n",
-			IPA_MEM_PART(pdn_config_ofst));
-		return -ENODEV;
-	}
-
-	IPADBG("QUOTA STATS OFST 0x%x SIZE 0x%x\n",
-		IPA_MEM_PART(stats_quota_ofst),
-		IPA_MEM_PART(stats_quota_size));
-
-	if (IPA_MEM_PART(stats_quota_ofst) & 7) {
-		IPAERR("QUOTA STATS OFST 0x%x is unaligned\n",
-			IPA_MEM_PART(stats_quota_ofst));
-		return -ENODEV;
-	}
-
-	IPADBG("TETHERING STATS OFST 0x%x SIZE 0x%x\n",
-		IPA_MEM_PART(stats_tethering_ofst),
-		IPA_MEM_PART(stats_tethering_size));
-
-	if (IPA_MEM_PART(stats_tethering_ofst) & 7) {
-		IPAERR("TETHERING STATS OFST 0x%x is unaligned\n",
-			IPA_MEM_PART(stats_tethering_ofst));
-		return -ENODEV;
-	}
-
-	IPADBG("FILTER AND ROUTING STATS OFST 0x%x SIZE 0x%x\n",
-		IPA_MEM_PART(stats_fnr_ofst),
-		IPA_MEM_PART(stats_fnr_size));
-
-	if (IPA_MEM_PART(stats_fnr_ofst) & 7) {
-		IPAERR("FILTER AND ROUTING STATS OFST 0x%x is unaligned\n",
-			IPA_MEM_PART(stats_fnr_ofst));
-		return -ENODEV;
-	}
-
-	IPADBG("DROP STATS OFST 0x%x SIZE 0x%x\n",
-	IPA_MEM_PART(stats_drop_ofst),
-		IPA_MEM_PART(stats_drop_size));
-
-	if (IPA_MEM_PART(stats_drop_ofst) & 7) {
-		IPAERR("DROP STATS OFST 0x%x is unaligned\n",
-			IPA_MEM_PART(stats_drop_ofst));
-		return -ENODEV;
-	}
-
-	IPADBG("V4 APPS HASHABLE FLT OFST 0x%x SIZE 0x%x\n",
-		IPA_MEM_PART(apps_v4_flt_hash_ofst),
-		IPA_MEM_PART(apps_v4_flt_hash_size));
-
-	IPADBG("V4 APPS NON-HASHABLE FLT OFST 0x%x SIZE 0x%x\n",
-		IPA_MEM_PART(apps_v4_flt_nhash_ofst),
-		IPA_MEM_PART(apps_v4_flt_nhash_size));
-
-	IPADBG("V6 APPS HASHABLE FLT OFST 0x%x SIZE 0x%x\n",
-		IPA_MEM_PART(apps_v6_flt_hash_ofst),
-		IPA_MEM_PART(apps_v6_flt_hash_size));
-
-	IPADBG("V6 APPS NON-HASHABLE FLT OFST 0x%x SIZE 0x%x\n",
-		IPA_MEM_PART(apps_v6_flt_nhash_ofst),
-		IPA_MEM_PART(apps_v6_flt_nhash_size));
-
-	IPADBG("RAM END OFST 0x%x\n",
-		IPA_MEM_PART(end_ofst));
-
-	IPADBG("V4 APPS HASHABLE RT OFST 0x%x SIZE 0x%x\n",
-		IPA_MEM_PART(apps_v4_rt_hash_ofst),
-		IPA_MEM_PART(apps_v4_rt_hash_size));
-
-	IPADBG("V4 APPS NON-HASHABLE RT OFST 0x%x SIZE 0x%x\n",
-		IPA_MEM_PART(apps_v4_rt_nhash_ofst),
-		IPA_MEM_PART(apps_v4_rt_nhash_size));
-
-	IPADBG("V6 APPS HASHABLE RT OFST 0x%x SIZE 0x%x\n",
-		IPA_MEM_PART(apps_v6_rt_hash_ofst),
-		IPA_MEM_PART(apps_v6_rt_hash_size));
-
-	IPADBG("V6 APPS NON-HASHABLE RT OFST 0x%x SIZE 0x%x\n",
-		IPA_MEM_PART(apps_v6_rt_nhash_ofst),
-		IPA_MEM_PART(apps_v6_rt_nhash_size));
-
-	if (IPA_MEM_PART(modem_ofst) & 7) {
-		IPAERR("MODEM OFST 0x%x is unaligned\n",
-			IPA_MEM_PART(modem_ofst));
-		return -ENODEV;
-	}
-
-	IPADBG("MODEM OFST 0x%x SIZE 0x%x\n", IPA_MEM_PART(modem_ofst),
-		IPA_MEM_PART(modem_size));
-
-	if (IPA_MEM_PART(uc_descriptor_ram_ofst) & 1023) {
-		IPAERR("UC DESCRIPTOR RAM OFST 0x%x is unaligned\n",
-			IPA_MEM_PART(uc_descriptor_ram_ofst));
-		return -ENODEV;
-	}
-
-	IPADBG("UC DESCRIPTOR RAM OFST 0x%x SIZE 0x%x\n",
-		IPA_MEM_PART(uc_descriptor_ram_ofst),
-		IPA_MEM_PART(uc_descriptor_ram_size));
-
-	return 0;
-}
-
-/**
- * ipa_ctrl_static_bind() - set the appropriate methods for
- *  IPA Driver based on the HW version
- *
- *  @ctrl: data structure which holds the function pointers
- *  @hw_type: the HW type in use
- *
- *  This function can avoid the runtime assignment by using C99 special
- *  struct initialization - hard decision... time.vs.mem
- */
-int ipa3_controller_static_bind(struct ipa3_controller *ctrl,
-		enum ipa_hw_type hw_type)
-{
-	if (hw_type >= IPA_HW_v4_0) {
-		ctrl->ipa_clk_rate_turbo = IPA_V4_0_CLK_RATE_TURBO;
-		ctrl->ipa_clk_rate_nominal = IPA_V4_0_CLK_RATE_NOMINAL;
-		ctrl->ipa_clk_rate_svs = IPA_V4_0_CLK_RATE_SVS;
-		ctrl->ipa_clk_rate_svs2 = IPA_V4_0_CLK_RATE_SVS2;
-	} else if (hw_type >= IPA_HW_v3_5) {
-		ctrl->ipa_clk_rate_turbo = IPA_V3_5_CLK_RATE_TURBO;
-		ctrl->ipa_clk_rate_nominal = IPA_V3_5_CLK_RATE_NOMINAL;
-		ctrl->ipa_clk_rate_svs = IPA_V3_5_CLK_RATE_SVS;
-		ctrl->ipa_clk_rate_svs2 = IPA_V3_5_CLK_RATE_SVS2;
-	} else {
-		ctrl->ipa_clk_rate_turbo = IPA_V3_0_CLK_RATE_TURBO;
-		ctrl->ipa_clk_rate_nominal = IPA_V3_0_CLK_RATE_NOMINAL;
-		ctrl->ipa_clk_rate_svs = IPA_V3_0_CLK_RATE_SVS;
-		ctrl->ipa_clk_rate_svs2 = IPA_V3_0_CLK_RATE_SVS2;
-	}
-
-	ctrl->ipa_init_rt4 = _ipa_init_rt4_v3;
-	ctrl->ipa_init_rt6 = _ipa_init_rt6_v3;
-	ctrl->ipa_init_flt4 = _ipa_init_flt4_v3;
-	ctrl->ipa_init_flt6 = _ipa_init_flt6_v3;
-	ctrl->ipa3_read_ep_reg = _ipa_read_ep_reg_v3_0;
-	ctrl->ipa3_commit_flt = __ipa_commit_flt_v3;
-	ctrl->ipa3_commit_rt = __ipa_commit_rt_v3;
-	ctrl->ipa3_commit_hdr = __ipa_commit_hdr_v3_0;
-	ctrl->ipa3_enable_clks = _ipa_enable_clks_v3_0;
-	ctrl->ipa3_disable_clks = _ipa_disable_clks_v3_0;
-	ctrl->clock_scaling_bw_threshold_svs =
-		IPA_V3_0_BW_THRESHOLD_SVS_MBPS;
-	ctrl->clock_scaling_bw_threshold_nominal =
-		IPA_V3_0_BW_THRESHOLD_NOMINAL_MBPS;
-	ctrl->clock_scaling_bw_threshold_turbo =
-		IPA_V3_0_BW_THRESHOLD_TURBO_MBPS;
-	ctrl->ipa_reg_base_ofst = ipahal_get_reg_base();
-	ctrl->ipa_init_sram = _ipa_init_sram_v3;
-	ctrl->ipa_sram_read_settings = _ipa_sram_settings_read_v3_0;
-	ctrl->ipa_init_hdr = _ipa_init_hdr_v3_0;
-
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0)
-		ctrl->ipa3_read_ep_reg = _ipa_read_ep_reg_v4_0;
-
-	return 0;
-}
-
-void ipa3_skb_recycle(struct sk_buff *skb)
-{
-	struct skb_shared_info *shinfo;
-
-	shinfo = skb_shinfo(skb);
-	memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
-	atomic_set(&shinfo->dataref, 1);
-
-	memset(skb, 0, offsetof(struct sk_buff, tail));
-	skb->data = skb->head + NET_SKB_PAD;
-	skb_reset_tail_pointer(skb);
-}
-
-int ipa3_alloc_rule_id(struct idr *rule_ids)
-{
-	/* There is two groups of rule-Ids, Modem ones and Apps ones.
-	 * Distinction by high bit: Modem Ids are high bit asserted.
-	 */
-	return idr_alloc(rule_ids, NULL,
-		ipahal_get_low_rule_id(),
-		ipahal_get_rule_id_hi_bit(),
-		GFP_KERNEL);
-}
-
-static int __ipa3_alloc_counter_hdl
-	(struct ipa_ioc_flt_rt_counter_alloc *counter)
-{
-	int id;
-
-	/* assign a handle using idr to this counter block */
-	id = idr_alloc(&ipa3_ctx->flt_rt_counters.hdl, counter,
-		ipahal_get_low_hdl_id(), ipahal_get_high_hdl_id(),
-		GFP_ATOMIC);
-
-	return id;
-}
-
-int ipa3_alloc_counter_id(struct ipa_ioc_flt_rt_counter_alloc *counter)
-{
-	int i, unused_cnt, unused_max, unused_start_id;
-
-	idr_preload(GFP_KERNEL);
-	spin_lock(&ipa3_ctx->flt_rt_counters.hdl_lock);
-
-	/* allocate hw counters */
-	counter->hw_counter.start_id = 0;
-	counter->hw_counter.end_id = 0;
-	unused_cnt = 0;
-	unused_max = 0;
-	unused_start_id = 0;
-	if (counter->hw_counter.num_counters == 0)
-		goto sw_counter_alloc;
-	/* find the start id which can be used for the block */
-	for (i = 0; i < IPA_FLT_RT_HW_COUNTER; i++) {
-		if (!ipa3_ctx->flt_rt_counters.used_hw[i])
-			unused_cnt++;
-		else {
-			/* tracking max unused block in case allow less */
-			if (unused_cnt > unused_max) {
-				unused_start_id = i - unused_cnt + 2;
-				unused_max = unused_cnt;
-			}
-			unused_cnt = 0;
-		}
-		/* find it, break and use this 1st possible block */
-		if (unused_cnt == counter->hw_counter.num_counters) {
-			counter->hw_counter.start_id = i - unused_cnt + 2;
-			counter->hw_counter.end_id = i + 1;
-			break;
-		}
-	}
-	if (counter->hw_counter.start_id == 0) {
-		/* if not able to find such a block but allow less */
-		if (counter->hw_counter.allow_less && unused_max) {
-			/* give the max possible unused blocks */
-			counter->hw_counter.num_counters = unused_max;
-			counter->hw_counter.start_id = unused_start_id;
-			counter->hw_counter.end_id =
-				unused_start_id + unused_max - 1;
-		} else {
-			/* not able to find such a block */
-			counter->hw_counter.num_counters = 0;
-			counter->hw_counter.start_id = 0;
-			counter->hw_counter.end_id = 0;
-			goto err;
-		}
-	}
-
-sw_counter_alloc:
-	/* allocate sw counters */
-	counter->sw_counter.start_id = 0;
-	counter->sw_counter.end_id = 0;
-	unused_cnt = 0;
-	unused_max = 0;
-	unused_start_id = 0;
-	if (counter->sw_counter.num_counters == 0)
-		goto mark_hw_cnt;
-	/* find the start id which can be used for the block */
-	for (i = 0; i < IPA_FLT_RT_SW_COUNTER; i++) {
-		if (!ipa3_ctx->flt_rt_counters.used_sw[i])
-			unused_cnt++;
-		else {
-			/* tracking max unused block in case allow less */
-			if (unused_cnt > unused_max) {
-				unused_start_id = i - unused_cnt +
-					2 + IPA_FLT_RT_HW_COUNTER;
-				unused_max = unused_cnt;
-			}
-			unused_cnt = 0;
-		}
-		/* find it, break and use this 1st possible block */
-		if (unused_cnt == counter->sw_counter.num_counters) {
-			counter->sw_counter.start_id = i - unused_cnt +
-				2 + IPA_FLT_RT_HW_COUNTER;
-			counter->sw_counter.end_id =
-				i + 1 + IPA_FLT_RT_HW_COUNTER;
-			break;
-		}
-	}
-	if (counter->sw_counter.start_id == 0) {
-		/* if not able to find such a block but allow less */
-		if (counter->sw_counter.allow_less && unused_max) {
-			/* give the max possible unused blocks */
-			counter->sw_counter.num_counters = unused_max;
-			counter->sw_counter.start_id = unused_start_id;
-			counter->sw_counter.end_id =
-				unused_start_id + unused_max - 1;
-		} else {
-			/* not able to find such a block */
-			counter->sw_counter.num_counters = 0;
-			counter->sw_counter.start_id = 0;
-			counter->sw_counter.end_id = 0;
-			goto err;
-		}
-	}
-
-mark_hw_cnt:
-	/* add hw counters, set used to 1 */
-	if (counter->hw_counter.num_counters == 0)
-		goto mark_sw_cnt;
-	unused_start_id = counter->hw_counter.start_id;
-	if (unused_start_id < 1 ||
-		unused_start_id > IPA_FLT_RT_HW_COUNTER) {
-		IPAERR("unexpected hw_counter start id %d\n",
-			   unused_start_id);
-		goto err;
-	}
-	for (i = 0; i < counter->hw_counter.num_counters; i++)
-		ipa3_ctx->flt_rt_counters.used_hw[unused_start_id + i - 1]
-			= true;
-mark_sw_cnt:
-	/* add sw counters, set used to 1 */
-	if (counter->sw_counter.num_counters == 0)
-		goto done;
-	unused_start_id = counter->sw_counter.start_id
-		- IPA_FLT_RT_HW_COUNTER;
-	if (unused_start_id < 1 ||
-		unused_start_id > IPA_FLT_RT_SW_COUNTER) {
-		IPAERR("unexpected sw_counter start id %d\n",
-			   unused_start_id);
-		goto err;
-	}
-	for (i = 0; i < counter->sw_counter.num_counters; i++)
-		ipa3_ctx->flt_rt_counters.used_sw[unused_start_id + i - 1]
-			= true;
-done:
-	/* get a handle from idr for dealloc */
-	counter->hdl = __ipa3_alloc_counter_hdl(counter);
-	spin_unlock(&ipa3_ctx->flt_rt_counters.hdl_lock);
-	idr_preload_end();
-	return 0;
-
-err:
-	counter->hdl = -1;
-	spin_unlock(&ipa3_ctx->flt_rt_counters.hdl_lock);
-	idr_preload_end();
-	return -ENOMEM;
-}
-
-void ipa3_counter_remove_hdl(int hdl)
-{
-	struct ipa_ioc_flt_rt_counter_alloc *counter;
-	int offset = 0;
-
-	spin_lock(&ipa3_ctx->flt_rt_counters.hdl_lock);
-	counter = idr_find(&ipa3_ctx->flt_rt_counters.hdl, hdl);
-	if (counter == NULL) {
-		IPAERR("unexpected hdl %d\n", hdl);
-		goto err;
-	}
-	/* remove counters belong to this hdl, set used back to 0 */
-	offset = counter->hw_counter.start_id - 1;
-	if (offset >= 0 && offset + counter->hw_counter.num_counters
-		< IPA_FLT_RT_HW_COUNTER) {
-		memset(&ipa3_ctx->flt_rt_counters.used_hw + offset,
-			   0, counter->hw_counter.num_counters * sizeof(bool));
-	} else {
-		IPAERR("unexpected hdl %d\n", hdl);
-		goto err;
-	}
-	offset = counter->sw_counter.start_id - 1 - IPA_FLT_RT_HW_COUNTER;
-	if (offset >= 0 && offset + counter->sw_counter.num_counters
-		< IPA_FLT_RT_SW_COUNTER) {
-		memset(&ipa3_ctx->flt_rt_counters.used_sw + offset,
-		   0, counter->sw_counter.num_counters * sizeof(bool));
-	} else {
-		IPAERR("unexpected hdl %d\n", hdl);
-		goto err;
-	}
-	/* remove the handle */
-	idr_remove(&ipa3_ctx->flt_rt_counters.hdl, hdl);
-err:
-	spin_unlock(&ipa3_ctx->flt_rt_counters.hdl_lock);
-}
-
-void ipa3_counter_id_remove_all(void)
-{
-	struct ipa_ioc_flt_rt_counter_alloc *counter;
-	int hdl;
-
-	spin_lock(&ipa3_ctx->flt_rt_counters.hdl_lock);
-	/* remove all counters, set used back to 0 */
-	memset(&ipa3_ctx->flt_rt_counters.used_hw, 0,
-		   sizeof(ipa3_ctx->flt_rt_counters.used_hw));
-	memset(&ipa3_ctx->flt_rt_counters.used_sw, 0,
-		   sizeof(ipa3_ctx->flt_rt_counters.used_sw));
-	/* remove all handles */
-	idr_for_each_entry(&ipa3_ctx->flt_rt_counters.hdl, counter, hdl)
-		idr_remove(&ipa3_ctx->flt_rt_counters.hdl, hdl);
-	spin_unlock(&ipa3_ctx->flt_rt_counters.hdl_lock);
-}
-
-int ipa3_id_alloc(void *ptr)
-{
-	int id;
-
-	idr_preload(GFP_KERNEL);
-	spin_lock(&ipa3_ctx->idr_lock);
-	id = idr_alloc(&ipa3_ctx->ipa_idr, ptr, 0, 0, GFP_NOWAIT);
-	spin_unlock(&ipa3_ctx->idr_lock);
-	idr_preload_end();
-
-	return id;
-}
-
-void *ipa3_id_find(u32 id)
-{
-	void *ptr;
-
-	spin_lock(&ipa3_ctx->idr_lock);
-	ptr = idr_find(&ipa3_ctx->ipa_idr, id);
-	spin_unlock(&ipa3_ctx->idr_lock);
-
-	return ptr;
-}
-
-void ipa3_id_remove(u32 id)
-{
-	spin_lock(&ipa3_ctx->idr_lock);
-	idr_remove(&ipa3_ctx->ipa_idr, id);
-	spin_unlock(&ipa3_ctx->idr_lock);
-}
-
-void ipa3_tag_destroy_imm(void *user1, int user2)
-{
-	ipahal_destroy_imm_cmd(user1);
-}
-
-static void ipa3_tag_free_skb(void *user1, int user2)
-{
-	dev_kfree_skb_any((struct sk_buff *)user1);
-}
-
-#define REQUIRED_TAG_PROCESS_DESCRIPTORS 4
-#define MAX_RETRY_ALLOC 10
-#define ALLOC_MIN_SLEEP_RX 100000
-#define ALLOC_MAX_SLEEP_RX 200000
-
-/* ipa3_tag_process() - Initiates a tag process. Incorporates the input
- * descriptors
- *
- * @desc:	descriptors with commands for IC
- * @desc_size:	amount of descriptors in the above variable
- *
- * Note: The descriptors are copied (if there's room), the client needs to
- * free his descriptors afterwards
- *
- * Return: 0 or negative in case of failure
- */
-int ipa3_tag_process(struct ipa3_desc desc[],
-	int descs_num,
-	unsigned long timeout)
-{
-	struct ipa3_sys_context *sys;
-	struct ipa3_desc *tag_desc;
-	int desc_idx = 0;
-	struct ipahal_imm_cmd_ip_packet_init pktinit_cmd;
-	struct ipahal_imm_cmd_pyld *cmd_pyld = NULL;
-	struct ipahal_imm_cmd_ip_packet_tag_status status;
-	int i;
-	struct sk_buff *dummy_skb;
-	int res;
-	struct ipa3_tag_completion *comp;
-	int ep_idx;
-	u32 retry_cnt = 0;
-
-	/* Not enough room for the required descriptors for the tag process */
-	if (IPA_TAG_MAX_DESC - descs_num < REQUIRED_TAG_PROCESS_DESCRIPTORS) {
-		IPAERR("up to %d descriptors are allowed (received %d)\n",
-		       IPA_TAG_MAX_DESC - REQUIRED_TAG_PROCESS_DESCRIPTORS,
-		       descs_num);
-		return -ENOMEM;
-	}
-
-	ep_idx = ipa3_get_ep_mapping(IPA_CLIENT_APPS_CMD_PROD);
-	if (-1 == ep_idx) {
-		IPAERR("Client %u is not mapped\n",
-			IPA_CLIENT_APPS_CMD_PROD);
-		return -EFAULT;
-	}
-	sys = ipa3_ctx->ep[ep_idx].sys;
-
-	tag_desc = kzalloc(sizeof(*tag_desc) * IPA_TAG_MAX_DESC, GFP_KERNEL);
-	if (!tag_desc) {
-		IPAERR("failed to allocate memory\n");
-		return -ENOMEM;
-	}
-
-	/* Copy the required descriptors from the client now */
-	if (desc) {
-		memcpy(&(tag_desc[0]), desc, descs_num *
-			sizeof(tag_desc[0]));
-		desc_idx += descs_num;
-	}
-
-	/* NO-OP IC for ensuring that IPA pipeline is empty */
-	cmd_pyld = ipahal_construct_nop_imm_cmd(
-		false, IPAHAL_FULL_PIPELINE_CLEAR, false);
-	if (!cmd_pyld) {
-		IPAERR("failed to construct NOP imm cmd\n");
-		res = -ENOMEM;
-		goto fail_free_tag_desc;
-	}
-	ipa3_init_imm_cmd_desc(&tag_desc[desc_idx], cmd_pyld);
-	tag_desc[desc_idx].callback = ipa3_tag_destroy_imm;
-	tag_desc[desc_idx].user1 = cmd_pyld;
-	++desc_idx;
-
-	/* IP_PACKET_INIT IC for tag status to be sent to apps */
-	pktinit_cmd.destination_pipe_index =
-		ipa3_get_ep_mapping(IPA_CLIENT_APPS_LAN_CONS);
-	cmd_pyld = ipahal_construct_imm_cmd(
-		IPA_IMM_CMD_IP_PACKET_INIT, &pktinit_cmd, false);
-	if (!cmd_pyld) {
-		IPAERR("failed to construct ip_packet_init imm cmd\n");
-		res = -ENOMEM;
-		goto fail_free_desc;
-	}
-	ipa3_init_imm_cmd_desc(&tag_desc[desc_idx], cmd_pyld);
-	tag_desc[desc_idx].callback = ipa3_tag_destroy_imm;
-	tag_desc[desc_idx].user1 = cmd_pyld;
-	++desc_idx;
-
-	/* status IC */
-	status.tag = IPA_COOKIE;
-	cmd_pyld = ipahal_construct_imm_cmd(
-		IPA_IMM_CMD_IP_PACKET_TAG_STATUS, &status, false);
-	if (!cmd_pyld) {
-		IPAERR("failed to construct ip_packet_tag_status imm cmd\n");
-		res = -ENOMEM;
-		goto fail_free_desc;
-	}
-	ipa3_init_imm_cmd_desc(&tag_desc[desc_idx], cmd_pyld);
-	tag_desc[desc_idx].callback = ipa3_tag_destroy_imm;
-	tag_desc[desc_idx].user1 = cmd_pyld;
-	++desc_idx;
-
-	comp = kzalloc(sizeof(*comp), GFP_KERNEL);
-	if (!comp) {
-		IPAERR("no mem\n");
-		res = -ENOMEM;
-		goto fail_free_desc;
-	}
-	init_completion(&comp->comp);
-
-	/* completion needs to be released from both here and rx handler */
-	atomic_set(&comp->cnt, 2);
-
-	/* dummy packet to send to IPA. packet payload is a completion object */
-	dummy_skb = alloc_skb(sizeof(comp), GFP_KERNEL);
-	if (!dummy_skb) {
-		IPAERR("failed to allocate memory\n");
-		res = -ENOMEM;
-		goto fail_free_comp;
-	}
-
-	memcpy(skb_put(dummy_skb, sizeof(comp)), &comp, sizeof(comp));
-
-	if (desc_idx >= IPA_TAG_MAX_DESC) {
-		IPAERR("number of commands is out of range\n");
-		res = -ENOBUFS;
-		goto fail_free_skb;
-	}
-
-	tag_desc[desc_idx].pyld = dummy_skb->data;
-	tag_desc[desc_idx].len = dummy_skb->len;
-	tag_desc[desc_idx].type = IPA_DATA_DESC_SKB;
-	tag_desc[desc_idx].callback = ipa3_tag_free_skb;
-	tag_desc[desc_idx].user1 = dummy_skb;
-	desc_idx++;
-retry_alloc:
-	/* send all descriptors to IPA with single EOT */
-	res = ipa3_send(sys, desc_idx, tag_desc, true);
-	if (res) {
-		if (res == -ENOMEM) {
-			if (retry_cnt < MAX_RETRY_ALLOC) {
-				IPADBG(
-				"failed to alloc memory retry cnt = %d\n",
-					retry_cnt);
-				retry_cnt++;
-				usleep_range(ALLOC_MIN_SLEEP_RX,
-					ALLOC_MAX_SLEEP_RX);
-				goto retry_alloc;
-			}
-
-		}
-		IPAERR("failed to send TAG packets %d\n", res);
-		res = -ENOMEM;
-		goto fail_free_skb;
-	}
-	kfree(tag_desc);
-	tag_desc = NULL;
-	ipa3_ctx->tag_process_before_gating = false;
-
-	IPADBG("waiting for TAG response\n");
-	res = wait_for_completion_timeout(&comp->comp, timeout);
-	if (res == 0) {
-		IPAERR("timeout (%lu msec) on waiting for TAG response\n",
-			timeout);
-		WARN_ON(1);
-		if (atomic_dec_return(&comp->cnt) == 0)
-			kfree(comp);
-		return -ETIME;
-	}
-
-	IPADBG("TAG response arrived!\n");
-	if (atomic_dec_return(&comp->cnt) == 0)
-		kfree(comp);
-
-	/*
-	 * sleep for short period to ensure IPA wrote all packets to
-	 * the transport
-	 */
-	usleep_range(IPA_TAG_SLEEP_MIN_USEC, IPA_TAG_SLEEP_MAX_USEC);
-
-	return 0;
-
-fail_free_skb:
-	kfree_skb(dummy_skb);
-fail_free_comp:
-	kfree(comp);
-fail_free_desc:
-	/*
-	 * Free only the first descriptors allocated here.
-	 * [nop, pkt_init, status, dummy_skb]
-	 * The user is responsible to free his allocations
-	 * in case of failure.
-	 * The min is required because we may fail during
-	 * of the initial allocations above
-	 */
-	for (i = descs_num;
-		i < min(REQUIRED_TAG_PROCESS_DESCRIPTORS, desc_idx); i++)
-		if (tag_desc[i].callback)
-			tag_desc[i].callback(tag_desc[i].user1,
-				tag_desc[i].user2);
-fail_free_tag_desc:
-	kfree(tag_desc);
-	return res;
-}
-
-/**
- * ipa3_tag_generate_force_close_desc() - generate descriptors for force close
- *					 immediate command
- *
- * @desc: descriptors for IC
- * @desc_size: desc array size
- * @start_pipe: first pipe to close aggregation
- * @end_pipe: last (non-inclusive) pipe to close aggregation
- *
- * Return: number of descriptors written or negative in case of failure
- */
-static int ipa3_tag_generate_force_close_desc(struct ipa3_desc desc[],
-	int desc_size, int start_pipe, int end_pipe)
-{
-	int i;
-	struct ipa_ep_cfg_aggr ep_aggr;
-	int desc_idx = 0;
-	int res;
-	struct ipahal_imm_cmd_register_write reg_write_agg_close;
-	struct ipahal_imm_cmd_pyld *cmd_pyld;
-	struct ipahal_reg_valmask valmask;
-
-	for (i = start_pipe; i < end_pipe; i++) {
-		ipahal_read_reg_n_fields(IPA_ENDP_INIT_AGGR_n, i, &ep_aggr);
-		if (!ep_aggr.aggr_en)
-			continue;
-		IPADBG("Force close ep: %d\n", i);
-		if (desc_idx + 1 > desc_size) {
-			IPAERR("Internal error - no descriptors\n");
-			res = -EFAULT;
-			goto fail_no_desc;
-		}
-
-		reg_write_agg_close.skip_pipeline_clear = false;
-		reg_write_agg_close.pipeline_clear_options =
-			IPAHAL_FULL_PIPELINE_CLEAR;
-		reg_write_agg_close.offset =
-			ipahal_get_reg_ofst(IPA_AGGR_FORCE_CLOSE);
-		ipahal_get_aggr_force_close_valmask(i, &valmask);
-		reg_write_agg_close.value = valmask.val;
-		reg_write_agg_close.value_mask = valmask.mask;
-		cmd_pyld = ipahal_construct_imm_cmd(IPA_IMM_CMD_REGISTER_WRITE,
-			&reg_write_agg_close, false);
-		if (!cmd_pyld) {
-			IPAERR("failed to construct register_write imm cmd\n");
-			res = -ENOMEM;
-			goto fail_alloc_reg_write_agg_close;
-		}
-
-		ipa3_init_imm_cmd_desc(&desc[desc_idx], cmd_pyld);
-		desc[desc_idx].callback = ipa3_tag_destroy_imm;
-		desc[desc_idx].user1 = cmd_pyld;
-		++desc_idx;
-	}
-
-	return desc_idx;
-
-fail_alloc_reg_write_agg_close:
-	for (i = 0; i < desc_idx; ++i)
-		if (desc[desc_idx].callback)
-			desc[desc_idx].callback(desc[desc_idx].user1,
-				desc[desc_idx].user2);
-fail_no_desc:
-	return res;
-}
-
-/**
- * ipa3_tag_aggr_force_close() - Force close aggregation
- *
- * @pipe_num: pipe number or -1 for all pipes
- */
-int ipa3_tag_aggr_force_close(int pipe_num)
-{
-	struct ipa3_desc *desc;
-	int res = -1;
-	int start_pipe;
-	int end_pipe;
-	int num_descs;
-	int num_aggr_descs;
-
-	if (pipe_num < -1 || pipe_num >= (int)ipa3_ctx->ipa_num_pipes) {
-		IPAERR("Invalid pipe number %d\n", pipe_num);
-		return -EINVAL;
-	}
-
-	if (pipe_num == -1) {
-		start_pipe = 0;
-		end_pipe = ipa3_ctx->ipa_num_pipes;
-	} else {
-		start_pipe = pipe_num;
-		end_pipe = pipe_num + 1;
-	}
-
-	num_descs = end_pipe - start_pipe;
-
-	desc = kcalloc(num_descs, sizeof(*desc), GFP_KERNEL);
-	if (!desc) {
-		IPAERR("no mem\n");
-		return -ENOMEM;
-	}
-
-	/* Force close aggregation on all valid pipes with aggregation */
-	num_aggr_descs = ipa3_tag_generate_force_close_desc(desc, num_descs,
-						start_pipe, end_pipe);
-	if (num_aggr_descs < 0) {
-		IPAERR("ipa3_tag_generate_force_close_desc failed %d\n",
-			num_aggr_descs);
-		goto fail_free_desc;
-	}
-
-	res = ipa3_tag_process(desc, num_aggr_descs,
-			      IPA_FORCE_CLOSE_TAG_PROCESS_TIMEOUT);
-
-fail_free_desc:
-	kfree(desc);
-
-	return res;
-}
-
-/**
- * ipa3_is_ready() - check if IPA module was initialized
- * successfully
- *
- * Return value: true for yes; false for no
- */
-bool ipa3_is_ready(void)
-{
-	bool complete;
-
-	if (ipa3_ctx == NULL)
-		return false;
-	mutex_lock(&ipa3_ctx->lock);
-	complete = ipa3_ctx->ipa_initialization_complete;
-	mutex_unlock(&ipa3_ctx->lock);
-	return complete;
-}
-
-/**
- * ipa3_is_client_handle_valid() - check if IPA client handle is valid handle
- *
- * Return value: true for yes; false for no
- */
-bool ipa3_is_client_handle_valid(u32 clnt_hdl)
-{
-	if (clnt_hdl >= 0 && clnt_hdl < ipa3_ctx->ipa_num_pipes)
-		return true;
-	return false;
-}
-
-/**
- * ipa3_proxy_clk_unvote() - called to remove IPA clock proxy vote
- *
- * Return value: none
- */
-void ipa3_proxy_clk_unvote(void)
-{
-	if (ipa3_ctx == NULL)
-		return;
-	mutex_lock(&ipa3_ctx->q6_proxy_clk_vote_mutex);
-	if (ipa3_ctx->q6_proxy_clk_vote_valid) {
-		IPA_ACTIVE_CLIENTS_DEC_SPECIAL("PROXY_CLK_VOTE");
-		ipa3_ctx->q6_proxy_clk_vote_cnt--;
-		if (ipa3_ctx->q6_proxy_clk_vote_cnt == 0)
-			ipa3_ctx->q6_proxy_clk_vote_valid = false;
-	}
-	mutex_unlock(&ipa3_ctx->q6_proxy_clk_vote_mutex);
-}
-
-/**
- * ipa3_proxy_clk_vote() - called to add IPA clock proxy vote
- *
- * Return value: none
- */
-void ipa3_proxy_clk_vote(void)
-{
-	if (ipa3_ctx == NULL)
-		return;
-	mutex_lock(&ipa3_ctx->q6_proxy_clk_vote_mutex);
-	if (!ipa3_ctx->q6_proxy_clk_vote_valid ||
-		(ipa3_ctx->q6_proxy_clk_vote_cnt > 0)) {
-		IPA_ACTIVE_CLIENTS_INC_SPECIAL("PROXY_CLK_VOTE");
-		ipa3_ctx->q6_proxy_clk_vote_cnt++;
-		ipa3_ctx->q6_proxy_clk_vote_valid = true;
-	}
-	mutex_unlock(&ipa3_ctx->q6_proxy_clk_vote_mutex);
-}
-
-/**
- * ipa3_get_smem_restr_bytes()- Return IPA smem restricted bytes
- *
- * Return value: u16 - number of IPA smem restricted bytes
- */
-u16 ipa3_get_smem_restr_bytes(void)
-{
-	if (ipa3_ctx)
-		return ipa3_ctx->smem_restricted_bytes;
-
-	IPAERR("IPA Driver not initialized\n");
-
-	return 0;
-}
-
-/**
- * ipa3_get_modem_cfg_emb_pipe_flt()- Return ipa3_ctx->modem_cfg_emb_pipe_flt
- *
- * Return value: true if modem configures embedded pipe flt, false otherwise
- */
-bool ipa3_get_modem_cfg_emb_pipe_flt(void)
-{
-	if (ipa3_ctx)
-		return ipa3_ctx->modem_cfg_emb_pipe_flt;
-
-	IPAERR("IPA driver has not been initialized\n");
-
-	return false;
-}
-
-/**
- * ipa3_get_transport_type()
- *
- * Return value: enum ipa_transport_type
- */
-enum ipa_transport_type ipa3_get_transport_type(void)
-{
-	return IPA_TRANSPORT_TYPE_GSI;
-}
-
-u32 ipa3_get_num_pipes(void)
-{
-	return ipahal_read_reg(IPA_ENABLED_PIPES);
-}
-
-/**
- * ipa3_disable_apps_wan_cons_deaggr()-
- * set ipa_ctx->ipa_client_apps_wan_cons_agg_gro
- *
- * Return value: 0 or negative in case of failure
- */
-int ipa3_disable_apps_wan_cons_deaggr(uint32_t agg_size, uint32_t agg_count)
-{
-	int res = -1;
-
-	/* ipahal will adjust limits based on HW capabilities */
-
-	if (ipa3_ctx) {
-		ipa3_ctx->ipa_client_apps_wan_cons_agg_gro = true;
-		return 0;
-	}
-	return res;
-}
-
-static void *ipa3_get_ipc_logbuf(void)
-{
-	if (ipa3_ctx)
-		return ipa3_ctx->logbuf;
-
-	return NULL;
-}
-
-static void *ipa3_get_ipc_logbuf_low(void)
-{
-	if (ipa3_ctx)
-		return ipa3_ctx->logbuf_low;
-
-	return NULL;
-}
-
-static void ipa3_get_holb(int ep_idx, struct ipa_ep_cfg_holb *holb)
-{
-	*holb = ipa3_ctx->ep[ep_idx].holb;
-}
-
-static void ipa3_set_tag_process_before_gating(bool val)
-{
-	ipa3_ctx->tag_process_before_gating = val;
-}
-
-/**
- * ipa3_is_vlan_mode - check if a LAN driver should load in VLAN mode
- * @iface - type of vlan capable device
- * @res - query result: true for vlan mode, false for non vlan mode
- *
- * API must be called after ipa_is_ready() returns true, otherwise it will fail
- *
- * Returns: 0 on success, negative on failure
- */
-int ipa3_is_vlan_mode(enum ipa_vlan_ifaces iface, bool *res)
-{
-	if (!res) {
-		IPAERR("NULL out param\n");
-		return -EINVAL;
-	}
-
-	if (iface < 0 || iface >= IPA_VLAN_IF_MAX) {
-		IPAERR("invalid iface %d\n", iface);
-		return -EINVAL;
-	}
-
-	if (!ipa3_is_ready()) {
-		IPAERR("IPA is not ready yet\n");
-		return -ENODEV;
-	}
-
-	*res = ipa3_ctx->vlan_mode_iface[iface];
-
-	IPADBG("Driver %d vlan mode is %d\n", iface, *res);
-	return 0;
-}
-
-int ipa3_bind_api_controller(enum ipa_hw_type ipa_hw_type,
-	struct ipa_api_controller *api_ctrl)
-{
-	if (ipa_hw_type < IPA_HW_v3_0) {
-		IPAERR("Unsupported IPA HW version %d\n", ipa_hw_type);
-		WARN_ON(1);
-		return -EPERM;
-	}
-
-	api_ctrl->ipa_reset_endpoint = NULL;
-	api_ctrl->ipa_clear_endpoint_delay = ipa3_clear_endpoint_delay;
-	api_ctrl->ipa_disable_endpoint = NULL;
-	api_ctrl->ipa_cfg_ep = ipa3_cfg_ep;
-	api_ctrl->ipa_cfg_ep_nat = ipa3_cfg_ep_nat;
-	api_ctrl->ipa_cfg_ep_conn_track = ipa3_cfg_ep_conn_track;
-	api_ctrl->ipa_cfg_ep_hdr = ipa3_cfg_ep_hdr;
-	api_ctrl->ipa_cfg_ep_hdr_ext = ipa3_cfg_ep_hdr_ext;
-	api_ctrl->ipa_cfg_ep_mode = ipa3_cfg_ep_mode;
-	api_ctrl->ipa_cfg_ep_aggr = ipa3_cfg_ep_aggr;
-	api_ctrl->ipa_cfg_ep_deaggr = ipa3_cfg_ep_deaggr;
-	api_ctrl->ipa_cfg_ep_route = ipa3_cfg_ep_route;
-	api_ctrl->ipa_cfg_ep_holb = ipa3_cfg_ep_holb;
-	api_ctrl->ipa_get_holb = ipa3_get_holb;
-	api_ctrl->ipa_set_tag_process_before_gating =
-			ipa3_set_tag_process_before_gating;
-	api_ctrl->ipa_cfg_ep_cfg = ipa3_cfg_ep_cfg;
-	api_ctrl->ipa_cfg_ep_metadata_mask = ipa3_cfg_ep_metadata_mask;
-	api_ctrl->ipa_cfg_ep_holb_by_client = ipa3_cfg_ep_holb_by_client;
-	api_ctrl->ipa_cfg_ep_ctrl = ipa3_cfg_ep_ctrl;
-	api_ctrl->ipa_add_hdr = ipa3_add_hdr;
-	api_ctrl->ipa_add_hdr_usr = ipa3_add_hdr_usr;
-	api_ctrl->ipa_del_hdr = ipa3_del_hdr;
-	api_ctrl->ipa_commit_hdr = ipa3_commit_hdr;
-	api_ctrl->ipa_reset_hdr = ipa3_reset_hdr;
-	api_ctrl->ipa_get_hdr = ipa3_get_hdr;
-	api_ctrl->ipa_put_hdr = ipa3_put_hdr;
-	api_ctrl->ipa_copy_hdr = ipa3_copy_hdr;
-	api_ctrl->ipa_add_hdr_proc_ctx = ipa3_add_hdr_proc_ctx;
-	api_ctrl->ipa_del_hdr_proc_ctx = ipa3_del_hdr_proc_ctx;
-	api_ctrl->ipa_add_rt_rule = ipa3_add_rt_rule;
-	api_ctrl->ipa_add_rt_rule_v2 = ipa3_add_rt_rule_v2;
-	api_ctrl->ipa_add_rt_rule_usr = ipa3_add_rt_rule_usr;
-	api_ctrl->ipa_add_rt_rule_usr_v2 = ipa3_add_rt_rule_usr_v2;
-	api_ctrl->ipa_del_rt_rule = ipa3_del_rt_rule;
-	api_ctrl->ipa_commit_rt = ipa3_commit_rt;
-	api_ctrl->ipa_reset_rt = ipa3_reset_rt;
-	api_ctrl->ipa_get_rt_tbl = ipa3_get_rt_tbl;
-	api_ctrl->ipa_put_rt_tbl = ipa3_put_rt_tbl;
-	api_ctrl->ipa_query_rt_index = ipa3_query_rt_index;
-	api_ctrl->ipa_mdfy_rt_rule = ipa3_mdfy_rt_rule;
-	api_ctrl->ipa_mdfy_rt_rule_v2 = ipa3_mdfy_rt_rule_v2;
-	api_ctrl->ipa_add_flt_rule = ipa3_add_flt_rule;
-	api_ctrl->ipa_add_flt_rule_v2 = ipa3_add_flt_rule_v2;
-	api_ctrl->ipa_add_flt_rule_usr = ipa3_add_flt_rule_usr;
-	api_ctrl->ipa_add_flt_rule_usr_v2 = ipa3_add_flt_rule_usr_v2;
-	api_ctrl->ipa_del_flt_rule = ipa3_del_flt_rule;
-	api_ctrl->ipa_mdfy_flt_rule = ipa3_mdfy_flt_rule;
-	api_ctrl->ipa_mdfy_flt_rule_v2 = ipa3_mdfy_flt_rule_v2;
-	api_ctrl->ipa_commit_flt = ipa3_commit_flt;
-	api_ctrl->ipa_reset_flt = ipa3_reset_flt;
-	api_ctrl->ipa_allocate_nat_device = ipa3_allocate_nat_device;
-	api_ctrl->ipa_allocate_nat_table = ipa3_allocate_nat_table;
-	api_ctrl->ipa_allocate_ipv6ct_table = ipa3_allocate_ipv6ct_table;
-	api_ctrl->ipa_nat_init_cmd = ipa3_nat_init_cmd;
-	api_ctrl->ipa_ipv6ct_init_cmd = ipa3_ipv6ct_init_cmd;
-	api_ctrl->ipa_nat_dma_cmd = ipa3_nat_dma_cmd;
-	api_ctrl->ipa_table_dma_cmd = ipa3_table_dma_cmd;
-	api_ctrl->ipa_nat_del_cmd = ipa3_nat_del_cmd;
-	api_ctrl->ipa_del_nat_table = ipa3_del_nat_table;
-	api_ctrl->ipa_del_ipv6ct_table = ipa3_del_ipv6ct_table;
-	api_ctrl->ipa_nat_mdfy_pdn = ipa3_nat_mdfy_pdn;
-	api_ctrl->ipa_send_msg = ipa3_send_msg;
-	api_ctrl->ipa_register_pull_msg = ipa3_register_pull_msg;
-	api_ctrl->ipa_deregister_pull_msg = ipa3_deregister_pull_msg;
-	api_ctrl->ipa_register_intf = ipa3_register_intf;
-	api_ctrl->ipa_register_intf_ext = ipa3_register_intf_ext;
-	api_ctrl->ipa_deregister_intf = ipa3_deregister_intf;
-	api_ctrl->ipa_set_aggr_mode = ipa3_set_aggr_mode;
-	api_ctrl->ipa_set_qcncm_ndp_sig = ipa3_set_qcncm_ndp_sig;
-	api_ctrl->ipa_set_single_ndp_per_mbim = ipa3_set_single_ndp_per_mbim;
-	api_ctrl->ipa_tx_dp = ipa3_tx_dp;
-	api_ctrl->ipa_tx_dp_mul = ipa3_tx_dp_mul;
-	api_ctrl->ipa_free_skb = ipa3_free_skb;
-	api_ctrl->ipa_setup_sys_pipe = ipa3_setup_sys_pipe;
-	api_ctrl->ipa_teardown_sys_pipe = ipa3_teardown_sys_pipe;
-	api_ctrl->ipa_sys_setup = ipa3_sys_setup;
-	api_ctrl->ipa_sys_teardown = ipa3_sys_teardown;
-	api_ctrl->ipa_sys_update_gsi_hdls = ipa3_sys_update_gsi_hdls;
-	api_ctrl->ipa_connect_wdi_pipe = ipa3_connect_wdi_pipe;
-	api_ctrl->ipa_disconnect_wdi_pipe = ipa3_disconnect_wdi_pipe;
-	api_ctrl->ipa_enable_wdi_pipe = ipa3_enable_wdi_pipe;
-	api_ctrl->ipa_disable_wdi_pipe = ipa3_disable_wdi_pipe;
-	api_ctrl->ipa_resume_wdi_pipe = ipa3_resume_wdi_pipe;
-	api_ctrl->ipa_suspend_wdi_pipe = ipa3_suspend_wdi_pipe;
-	api_ctrl->ipa_get_wdi_stats = ipa3_get_wdi_stats;
-	api_ctrl->ipa_get_smem_restr_bytes = ipa3_get_smem_restr_bytes;
-	api_ctrl->ipa_broadcast_wdi_quota_reach_ind =
-			ipa3_broadcast_wdi_quota_reach_ind;
-	api_ctrl->ipa_uc_wdi_get_dbpa = ipa3_uc_wdi_get_dbpa;
-	api_ctrl->ipa_uc_reg_rdyCB = ipa3_uc_reg_rdyCB;
-	api_ctrl->ipa_uc_dereg_rdyCB = ipa3_uc_dereg_rdyCB;
-	api_ctrl->teth_bridge_init = ipa3_teth_bridge_init;
-	api_ctrl->teth_bridge_disconnect = ipa3_teth_bridge_disconnect;
-	api_ctrl->teth_bridge_connect = ipa3_teth_bridge_connect;
-	api_ctrl->ipa_set_client = ipa3_set_client;
-	api_ctrl->ipa_get_client = ipa3_get_client;
-	api_ctrl->ipa_get_client_uplink = ipa3_get_client_uplink;
-	api_ctrl->ipa_dma_init = ipa3_dma_init;
-	api_ctrl->ipa_dma_enable = ipa3_dma_enable;
-	api_ctrl->ipa_dma_disable = ipa3_dma_disable;
-	api_ctrl->ipa_dma_sync_memcpy = ipa3_dma_sync_memcpy;
-	api_ctrl->ipa_dma_async_memcpy = ipa3_dma_async_memcpy;
-	api_ctrl->ipa_dma_uc_memcpy = ipa3_dma_uc_memcpy;
-	api_ctrl->ipa_dma_destroy = ipa3_dma_destroy;
-	api_ctrl->ipa_mhi_init_engine = ipa3_mhi_init_engine;
-	api_ctrl->ipa_connect_mhi_pipe = ipa3_connect_mhi_pipe;
-	api_ctrl->ipa_disconnect_mhi_pipe = ipa3_disconnect_mhi_pipe;
-	api_ctrl->ipa_mhi_stop_gsi_channel = ipa3_mhi_stop_gsi_channel;
-	api_ctrl->ipa_uc_mhi_reset_channel = ipa3_uc_mhi_reset_channel;
-	api_ctrl->ipa_qmi_enable_force_clear_datapath_send =
-			ipa3_qmi_enable_force_clear_datapath_send;
-	api_ctrl->ipa_qmi_disable_force_clear_datapath_send =
-			ipa3_qmi_disable_force_clear_datapath_send;
-	api_ctrl->ipa_mhi_reset_channel_internal =
-			ipa3_mhi_reset_channel_internal;
-	api_ctrl->ipa_mhi_start_channel_internal =
-			ipa3_mhi_start_channel_internal;
-	api_ctrl->ipa_mhi_query_ch_info = ipa3_mhi_query_ch_info;
-	api_ctrl->ipa_mhi_resume_channels_internal =
-			ipa3_mhi_resume_channels_internal;
-	api_ctrl->ipa_has_open_aggr_frame = ipa3_has_open_aggr_frame;
-	api_ctrl->ipa_mhi_destroy_channel = ipa3_mhi_destroy_channel;
-	api_ctrl->ipa_uc_mhi_send_dl_ul_sync_info =
-			ipa3_uc_mhi_send_dl_ul_sync_info;
-	api_ctrl->ipa_uc_mhi_init = ipa3_uc_mhi_init;
-	api_ctrl->ipa_uc_mhi_suspend_channel = ipa3_uc_mhi_suspend_channel;
-	api_ctrl->ipa_uc_mhi_stop_event_update_channel =
-			ipa3_uc_mhi_stop_event_update_channel;
-	api_ctrl->ipa_uc_mhi_cleanup = ipa3_uc_mhi_cleanup;
-	api_ctrl->ipa_uc_state_check = ipa3_uc_state_check;
-	api_ctrl->ipa_write_qmap_id = ipa3_write_qmap_id;
-	api_ctrl->ipa_add_interrupt_handler = ipa3_add_interrupt_handler;
-	api_ctrl->ipa_remove_interrupt_handler = ipa3_remove_interrupt_handler;
-	api_ctrl->ipa_restore_suspend_handler = ipa3_restore_suspend_handler;
-	api_ctrl->ipa_bam_reg_dump = NULL;
-	api_ctrl->ipa_get_ep_mapping = ipa3_get_ep_mapping;
-	api_ctrl->ipa_is_ready = ipa3_is_ready;
-	api_ctrl->ipa_proxy_clk_vote = ipa3_proxy_clk_vote;
-	api_ctrl->ipa_proxy_clk_unvote = ipa3_proxy_clk_unvote;
-	api_ctrl->ipa_is_client_handle_valid = ipa3_is_client_handle_valid;
-	api_ctrl->ipa_get_client_mapping = ipa3_get_client_mapping;
-	api_ctrl->ipa_get_modem_cfg_emb_pipe_flt =
-		ipa3_get_modem_cfg_emb_pipe_flt;
-	api_ctrl->ipa_get_transport_type = ipa3_get_transport_type;
-	api_ctrl->ipa_ap_suspend = ipa3_ap_suspend;
-	api_ctrl->ipa_ap_resume = ipa3_ap_resume;
-	api_ctrl->ipa_get_smmu_domain = ipa3_get_smmu_domain;
-	api_ctrl->ipa_disable_apps_wan_cons_deaggr =
-		ipa3_disable_apps_wan_cons_deaggr;
-	api_ctrl->ipa_get_dma_dev = ipa3_get_dma_dev;
-	api_ctrl->ipa_release_wdi_mapping = ipa3_release_wdi_mapping;
-	api_ctrl->ipa_create_wdi_mapping = ipa3_create_wdi_mapping;
-	api_ctrl->ipa_get_gsi_ep_info = ipa3_get_gsi_ep_info;
-	api_ctrl->ipa_stop_gsi_channel = ipa3_stop_gsi_channel;
-	api_ctrl->ipa_start_gsi_channel = ipa3_start_gsi_channel;
-	api_ctrl->ipa_register_ipa_ready_cb = ipa3_register_ipa_ready_cb;
-	api_ctrl->ipa_inc_client_enable_clks = ipa3_inc_client_enable_clks;
-	api_ctrl->ipa_dec_client_disable_clks = ipa3_dec_client_disable_clks;
-	api_ctrl->ipa_inc_client_enable_clks_no_block =
-		ipa3_inc_client_enable_clks_no_block;
-	api_ctrl->ipa_suspend_resource_no_block =
-		ipa3_suspend_resource_no_block;
-	api_ctrl->ipa_resume_resource = ipa3_resume_resource;
-	api_ctrl->ipa_suspend_resource_sync = ipa3_suspend_resource_sync;
-	api_ctrl->ipa_set_required_perf_profile =
-		ipa3_set_required_perf_profile;
-	api_ctrl->ipa_get_ipc_logbuf = ipa3_get_ipc_logbuf;
-	api_ctrl->ipa_get_ipc_logbuf_low = ipa3_get_ipc_logbuf_low;
-	api_ctrl->ipa_rx_poll = ipa3_rx_poll;
-	api_ctrl->ipa_setup_uc_ntn_pipes = ipa3_setup_uc_ntn_pipes;
-	api_ctrl->ipa_tear_down_uc_offload_pipes =
-		ipa3_tear_down_uc_offload_pipes;
-	api_ctrl->ipa_get_pdev = ipa3_get_pdev;
-	api_ctrl->ipa_ntn_uc_reg_rdyCB = ipa3_ntn_uc_reg_rdyCB;
-	api_ctrl->ipa_ntn_uc_dereg_rdyCB = ipa3_ntn_uc_dereg_rdyCB;
-	api_ctrl->ipa_conn_wdi_pipes = ipa3_conn_wdi3_pipes;
-	api_ctrl->ipa_disconn_wdi_pipes = ipa3_disconn_wdi3_pipes;
-	api_ctrl->ipa_enable_wdi_pipes = ipa3_enable_wdi3_pipes;
-	api_ctrl->ipa_disable_wdi_pipes = ipa3_disable_wdi3_pipes;
-	api_ctrl->ipa_tz_unlock_reg = ipa3_tz_unlock_reg;
-	api_ctrl->ipa_get_smmu_params = ipa3_get_smmu_params;
-	api_ctrl->ipa_is_vlan_mode = ipa3_is_vlan_mode;
-	api_ctrl->ipa_wigig_internal_init = ipa3_wigig_internal_init;
-	api_ctrl->ipa_conn_wigig_rx_pipe_i = ipa3_conn_wigig_rx_pipe_i;
-	api_ctrl->ipa_conn_wigig_client_i = ipa3_conn_wigig_client_i;
-	api_ctrl->ipa_disconn_wigig_pipe_i = ipa3_disconn_wigig_pipe_i;
-	api_ctrl->ipa_wigig_uc_msi_init = ipa3_wigig_uc_msi_init;
-	api_ctrl->ipa_enable_wigig_pipe_i = ipa3_enable_wigig_pipe_i;
-	api_ctrl->ipa_disable_wigig_pipe_i = ipa3_disable_wigig_pipe_i;
-	api_ctrl->ipa_register_client_callback =
-		ipa3_register_client_callback;
-	api_ctrl->ipa_deregister_client_callback =
-		ipa3_deregister_client_callback;
-	return 0;
-}
-
-/**
- * ipa_is_modem_pipe()- Checks if pipe is owned by the modem
- *
- * @pipe_idx: pipe number
- * Return value: true if owned by modem, false otherwize
- */
-bool ipa_is_modem_pipe(int pipe_idx)
-{
-	int client_idx;
-
-	if (pipe_idx >= ipa3_ctx->ipa_num_pipes || pipe_idx < 0) {
-		IPAERR("Bad pipe index!\n");
-		return false;
-	}
-
-	for (client_idx = 0; client_idx < IPA_CLIENT_MAX; client_idx++) {
-		if (!IPA_CLIENT_IS_Q6_CONS(client_idx) &&
-			!IPA_CLIENT_IS_Q6_PROD(client_idx))
-			continue;
-		if (ipa3_get_ep_mapping(client_idx) == pipe_idx)
-			return true;
-	}
-
-	return false;
-}
-
-static void ipa3_write_rsrc_grp_type_reg(int group_index,
-			enum ipa_rsrc_grp_type_src n, bool src,
-			struct ipahal_reg_rsrc_grp_cfg *val)
-{
-	u8 hw_type_idx;
-
-	hw_type_idx = ipa3_get_hw_type_index();
-
-	switch (hw_type_idx) {
-	case IPA_3_0:
-		if (src) {
-			switch (group_index) {
-			case IPA_v3_0_GROUP_UL:
-			case IPA_v3_0_GROUP_DL:
-				ipahal_write_reg_n_fields(
-					IPA_SRC_RSRC_GRP_01_RSRC_TYPE_n,
-					n, val);
-				break;
-			case IPA_v3_0_GROUP_DIAG:
-			case IPA_v3_0_GROUP_DMA:
-				ipahal_write_reg_n_fields(
-					IPA_SRC_RSRC_GRP_23_RSRC_TYPE_n,
-					n, val);
-				break;
-			case IPA_v3_0_GROUP_Q6ZIP:
-			case IPA_v3_0_GROUP_UC_RX_Q:
-				ipahal_write_reg_n_fields(
-					IPA_SRC_RSRC_GRP_45_RSRC_TYPE_n,
-					n, val);
-				break;
-			default:
-				IPAERR(
-				" Invalid source resource group,index #%d\n",
-				group_index);
-				break;
-			}
-		} else {
-			switch (group_index) {
-			case IPA_v3_0_GROUP_UL:
-			case IPA_v3_0_GROUP_DL:
-				ipahal_write_reg_n_fields(
-					IPA_DST_RSRC_GRP_01_RSRC_TYPE_n,
-					n, val);
-				break;
-			case IPA_v3_0_GROUP_DIAG:
-			case IPA_v3_0_GROUP_DMA:
-				ipahal_write_reg_n_fields(
-					IPA_DST_RSRC_GRP_23_RSRC_TYPE_n,
-					n, val);
-				break;
-			case IPA_v3_0_GROUP_Q6ZIP_GENERAL:
-			case IPA_v3_0_GROUP_Q6ZIP_ENGINE:
-				ipahal_write_reg_n_fields(
-					IPA_DST_RSRC_GRP_45_RSRC_TYPE_n,
-					n, val);
-				break;
-			default:
-				IPAERR(
-				" Invalid destination resource group,index #%d\n",
-				group_index);
-				break;
-			}
-		}
-		break;
-	case IPA_3_5:
-	case IPA_3_5_MHI:
-	case IPA_3_5_1:
-		if (src) {
-			switch (group_index) {
-			case IPA_v3_5_GROUP_LWA_DL:
-			case IPA_v3_5_GROUP_UL_DL:
-				ipahal_write_reg_n_fields(
-					IPA_SRC_RSRC_GRP_01_RSRC_TYPE_n,
-					n, val);
-				break;
-			case IPA_v3_5_MHI_GROUP_DMA:
-			case IPA_v3_5_GROUP_UC_RX_Q:
-				ipahal_write_reg_n_fields(
-					IPA_SRC_RSRC_GRP_23_RSRC_TYPE_n,
-					n, val);
-				break;
-			default:
-				IPAERR(
-				" Invalid source resource group,index #%d\n",
-				group_index);
-				break;
-			}
-		} else {
-			switch (group_index) {
-			case IPA_v3_5_GROUP_LWA_DL:
-			case IPA_v3_5_GROUP_UL_DL:
-				ipahal_write_reg_n_fields(
-					IPA_DST_RSRC_GRP_01_RSRC_TYPE_n,
-					n, val);
-				break;
-			case IPA_v3_5_MHI_GROUP_DMA:
-				ipahal_write_reg_n_fields(
-					IPA_DST_RSRC_GRP_23_RSRC_TYPE_n,
-					n, val);
-				break;
-			default:
-				IPAERR(
-				" Invalid destination resource group,index #%d\n",
-				group_index);
-				break;
-			}
-		}
-		break;
-	case IPA_4_0:
-	case IPA_4_0_MHI:
-	case IPA_4_1:
-		if (src) {
-			switch (group_index) {
-			case IPA_v4_0_GROUP_LWA_DL:
-			case IPA_v4_0_GROUP_UL_DL:
-				ipahal_write_reg_n_fields(
-					IPA_SRC_RSRC_GRP_01_RSRC_TYPE_n,
-					n, val);
-				break;
-			case IPA_v4_0_MHI_GROUP_DMA:
-			case IPA_v4_0_GROUP_UC_RX_Q:
-				ipahal_write_reg_n_fields(
-					IPA_SRC_RSRC_GRP_23_RSRC_TYPE_n,
-					n, val);
-				break;
-			default:
-				IPAERR(
-				" Invalid source resource group,index #%d\n",
-				group_index);
-				break;
-			}
-		} else {
-			switch (group_index) {
-			case IPA_v4_0_GROUP_LWA_DL:
-			case IPA_v4_0_GROUP_UL_DL:
-				ipahal_write_reg_n_fields(
-					IPA_DST_RSRC_GRP_01_RSRC_TYPE_n,
-					n, val);
-				break;
-			case IPA_v4_0_MHI_GROUP_DMA:
-				ipahal_write_reg_n_fields(
-					IPA_DST_RSRC_GRP_23_RSRC_TYPE_n,
-					n, val);
-				break;
-			default:
-				IPAERR(
-				" Invalid destination resource group,index #%d\n",
-				group_index);
-				break;
-			}
-		}
-		break;
-	case IPA_4_2:
-		if (src) {
-			switch (group_index) {
-			case IPA_v4_2_GROUP_UL_DL:
-				ipahal_write_reg_n_fields(
-					IPA_SRC_RSRC_GRP_01_RSRC_TYPE_n,
-					n, val);
-				break;
-			default:
-				IPAERR(
-				" Invalid source resource group,index #%d\n",
-				group_index);
-				break;
-			}
-		} else {
-			switch (group_index) {
-			case IPA_v4_2_GROUP_UL_DL:
-				ipahal_write_reg_n_fields(
-					IPA_DST_RSRC_GRP_01_RSRC_TYPE_n,
-					n, val);
-				break;
-			default:
-				IPAERR(
-				" Invalid destination resource group,index #%d\n",
-				group_index);
-				break;
-			}
-		}
-		break;
-	case IPA_4_5:
-	case IPA_4_5_MHI:
-	case IPA_4_5_APQ:
-		if (src) {
-			switch (group_index) {
-			case IPA_v4_5_MHI_GROUP_PCIE:
-			case IPA_v4_5_GROUP_UL_DL:
-				ipahal_write_reg_n_fields(
-					IPA_SRC_RSRC_GRP_01_RSRC_TYPE_n,
-					n, val);
-				break;
-			case IPA_v4_5_MHI_GROUP_DMA:
-			case IPA_v4_5_MHI_GROUP_QDSS:
-				ipahal_write_reg_n_fields(
-					IPA_SRC_RSRC_GRP_23_RSRC_TYPE_n,
-					n, val);
-				break;
-			case IPA_v4_5_GROUP_UC_RX_Q:
-				ipahal_write_reg_n_fields(
-					IPA_SRC_RSRC_GRP_45_RSRC_TYPE_n,
-					n, val);
-				break;
-			default:
-				IPAERR(
-				" Invalid source resource group,index #%d\n",
-				group_index);
-				break;
-			}
-		} else {
-			switch (group_index) {
-			case IPA_v4_5_MHI_GROUP_PCIE:
-			case IPA_v4_5_GROUP_UL_DL:
-				ipahal_write_reg_n_fields(
-					IPA_DST_RSRC_GRP_01_RSRC_TYPE_n,
-					n, val);
-				break;
-			case IPA_v4_5_MHI_GROUP_DMA:
-			case IPA_v4_5_MHI_GROUP_QDSS:
-				ipahal_write_reg_n_fields(
-					IPA_DST_RSRC_GRP_23_RSRC_TYPE_n,
-					n, val);
-				break;
-			case IPA_v4_5_GROUP_UC_RX_Q:
-				ipahal_write_reg_n_fields(
-					IPA_DST_RSRC_GRP_45_RSRC_TYPE_n,
-					n, val);
-				break;
-			default:
-				IPAERR(
-				" Invalid destination resource group,index #%d\n",
-				group_index);
-				break;
-			}
-		}
-		break;
-	case IPA_4_7:
-		if (src) {
-			switch (group_index) {
-			case IPA_v4_7_GROUP_UL_DL:
-				ipahal_write_reg_n_fields(
-					IPA_SRC_RSRC_GRP_01_RSRC_TYPE_n,
-					n, val);
-				break;
-			default:
-				IPAERR(
-				" Invalid source resource group,index #%d\n",
-				group_index);
-				break;
-			}
-		} else {
-			switch (group_index) {
-			case IPA_v4_7_GROUP_UL_DL:
-				ipahal_write_reg_n_fields(
-					IPA_DST_RSRC_GRP_01_RSRC_TYPE_n,
-					n, val);
-				break;
-			default:
-				IPAERR(
-				" Invalid destination resource group,index #%d\n",
-				group_index);
-				break;
-			}
-		}
-		break;
-
-	default:
-		IPAERR("invalid hw type\n");
-		WARN_ON(1);
-		return;
-	}
-}
-
-static void ipa3_configure_rx_hps_clients(int depth,
-	int max_clnt_in_depth, int base_index, bool min)
-{
-	int i;
-	struct ipahal_reg_rx_hps_clients val;
-	u8 hw_type_idx;
-
-	hw_type_idx = ipa3_get_hw_type_index();
-
-	for (i = 0 ; i < max_clnt_in_depth ; i++) {
-		if (min)
-			val.client_minmax[i] =
-				ipa3_rsrc_rx_grp_config
-				[hw_type_idx]
-				[IPA_RSRC_GRP_TYPE_RX_HPS_CMDQ]
-				[i + base_index].min;
-		else
-			val.client_minmax[i] =
-				ipa3_rsrc_rx_grp_config
-				[hw_type_idx]
-				[IPA_RSRC_GRP_TYPE_RX_HPS_CMDQ]
-				[i + base_index].max;
-	}
-	if (depth) {
-		ipahal_write_reg_fields(min ? IPA_RX_HPS_CLIENTS_MIN_DEPTH_1 :
-					IPA_RX_HPS_CLIENTS_MAX_DEPTH_1,
-					&val);
-	} else {
-		ipahal_write_reg_fields(min ? IPA_RX_HPS_CLIENTS_MIN_DEPTH_0 :
-					IPA_RX_HPS_CLIENTS_MAX_DEPTH_0,
-					&val);
-	}
-}
-
-static void ipa3_configure_rx_hps_weight(void)
-{
-	struct ipahal_reg_rx_hps_weights val;
-	u8 hw_type_idx;
-
-	hw_type_idx = ipa3_get_hw_type_index();
-
-	val.hps_queue_weight_0 =
-			ipa3_rsrc_rx_grp_hps_weight_config
-			[hw_type_idx][IPA_RSRC_GRP_TYPE_RX_HPS_WEIGHT_CONFIG]
-			[0];
-	val.hps_queue_weight_1 =
-			ipa3_rsrc_rx_grp_hps_weight_config
-			[hw_type_idx][IPA_RSRC_GRP_TYPE_RX_HPS_WEIGHT_CONFIG]
-			[1];
-	val.hps_queue_weight_2 =
-			ipa3_rsrc_rx_grp_hps_weight_config
-			[hw_type_idx][IPA_RSRC_GRP_TYPE_RX_HPS_WEIGHT_CONFIG]
-			[2];
-	val.hps_queue_weight_3 =
-			ipa3_rsrc_rx_grp_hps_weight_config
-			[hw_type_idx][IPA_RSRC_GRP_TYPE_RX_HPS_WEIGHT_CONFIG]
-			[3];
-
-	ipahal_write_reg_fields(IPA_HPS_FTCH_ARB_QUEUE_WEIGHT, &val);
-}
-
-static void ipa3_configure_rx_hps(void)
-{
-	int rx_hps_max_clnt_in_depth0;
-
-	IPADBG("Assign RX_HPS CMDQ rsrc groups min-max limits\n");
-
-	/* Starting IPA4.5 we have 5 RX_HPS_CMDQ */
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_5)
-		rx_hps_max_clnt_in_depth0 = 4;
-	else
-		rx_hps_max_clnt_in_depth0 = 5;
-
-	ipa3_configure_rx_hps_clients(0, rx_hps_max_clnt_in_depth0, 0, true);
-	ipa3_configure_rx_hps_clients(0, rx_hps_max_clnt_in_depth0, 0, false);
-
-	/*
-	 * IPA 3.0/3.1 uses 6 RX_HPS_CMDQ and needs depths1 for that
-	 * which has two clients
-	 */
-	if (ipa3_ctx->ipa_hw_type <= IPA_HW_v3_1) {
-		ipa3_configure_rx_hps_clients(1, 2, rx_hps_max_clnt_in_depth0,
-			true);
-		ipa3_configure_rx_hps_clients(1, 2, rx_hps_max_clnt_in_depth0,
-			false);
-	}
-
-	/* Starting IPA4.2 no support to HPS weight config */
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v3_5 &&
-		(ipa3_ctx->ipa_hw_type < IPA_HW_v4_2))
-		ipa3_configure_rx_hps_weight();
-}
-
-void ipa3_set_resorce_groups_min_max_limits(void)
-{
-	int i;
-	int j;
-	int src_rsrc_type_max;
-	int dst_rsrc_type_max;
-	int src_grp_idx_max;
-	int dst_grp_idx_max;
-	struct ipahal_reg_rsrc_grp_cfg val;
-	u8 hw_type_idx;
-
-	IPADBG("ENTER\n");
-
-	hw_type_idx = ipa3_get_hw_type_index();
-	switch (hw_type_idx) {
-	case IPA_3_0:
-		src_rsrc_type_max = IPA_v3_0_RSRC_GRP_TYPE_SRC_MAX;
-		dst_rsrc_type_max = IPA_v3_0_RSRC_GRP_TYPE_DST_MAX;
-		src_grp_idx_max = IPA_v3_0_GROUP_MAX;
-		dst_grp_idx_max = IPA_v3_0_GROUP_MAX;
-		break;
-	case IPA_3_5:
-	case IPA_3_5_MHI:
-	case IPA_3_5_1:
-		src_rsrc_type_max = IPA_v3_5_RSRC_GRP_TYPE_SRC_MAX;
-		dst_rsrc_type_max = IPA_v3_5_RSRC_GRP_TYPE_DST_MAX;
-		src_grp_idx_max = IPA_v3_5_SRC_GROUP_MAX;
-		dst_grp_idx_max = IPA_v3_5_DST_GROUP_MAX;
-		break;
-	case IPA_4_0:
-	case IPA_4_0_MHI:
-	case IPA_4_1:
-		src_rsrc_type_max = IPA_v4_0_RSRC_GRP_TYPE_SRC_MAX;
-		dst_rsrc_type_max = IPA_v4_0_RSRC_GRP_TYPE_DST_MAX;
-		src_grp_idx_max = IPA_v4_0_SRC_GROUP_MAX;
-		dst_grp_idx_max = IPA_v4_0_DST_GROUP_MAX;
-		break;
-	case IPA_4_2:
-		src_rsrc_type_max = IPA_v4_0_RSRC_GRP_TYPE_SRC_MAX;
-		dst_rsrc_type_max = IPA_v4_0_RSRC_GRP_TYPE_DST_MAX;
-		src_grp_idx_max = IPA_v4_2_SRC_GROUP_MAX;
-		dst_grp_idx_max = IPA_v4_2_DST_GROUP_MAX;
-		break;
-	case IPA_4_5:
-	case IPA_4_5_MHI:
-	case IPA_4_5_APQ:
-		src_rsrc_type_max = IPA_v4_0_RSRC_GRP_TYPE_SRC_MAX;
-		dst_rsrc_type_max = IPA_v4_0_RSRC_GRP_TYPE_DST_MAX;
-		src_grp_idx_max = IPA_v4_5_SRC_GROUP_MAX;
-		dst_grp_idx_max = IPA_v4_5_DST_GROUP_MAX;
-		break;
-	case IPA_4_7:
-		src_rsrc_type_max = IPA_v4_0_RSRC_GRP_TYPE_SRC_MAX;
-		dst_rsrc_type_max = IPA_v4_0_RSRC_GRP_TYPE_DST_MAX;
-		src_grp_idx_max = IPA_v4_7_SRC_GROUP_MAX;
-		dst_grp_idx_max = IPA_v4_7_DST_GROUP_MAX;
-		break;
-default:
-		IPAERR("invalid hw type index\n");
-		WARN_ON(1);
-		return;
-	}
-
-	IPADBG("Assign source rsrc groups min-max limits\n");
-	for (i = 0; i < src_rsrc_type_max; i++) {
-		for (j = 0; j < src_grp_idx_max; j = j + 2) {
-			val.x_min =
-			ipa3_rsrc_src_grp_config[hw_type_idx][i][j].min;
-			val.x_max =
-			ipa3_rsrc_src_grp_config[hw_type_idx][i][j].max;
-			val.y_min =
-			ipa3_rsrc_src_grp_config[hw_type_idx][i][j + 1].min;
-			val.y_max =
-			ipa3_rsrc_src_grp_config[hw_type_idx][i][j + 1].max;
-			ipa3_write_rsrc_grp_type_reg(j, i, true, &val);
-		}
-	}
-
-	IPADBG("Assign destination rsrc groups min-max limits\n");
-	for (i = 0; i < dst_rsrc_type_max; i++) {
-		for (j = 0; j < dst_grp_idx_max; j = j + 2) {
-			val.x_min =
-			ipa3_rsrc_dst_grp_config[hw_type_idx][i][j].min;
-			val.x_max =
-			ipa3_rsrc_dst_grp_config[hw_type_idx][i][j].max;
-			val.y_min =
-			ipa3_rsrc_dst_grp_config[hw_type_idx][i][j + 1].min;
-			val.y_max =
-			ipa3_rsrc_dst_grp_config[hw_type_idx][i][j + 1].max;
-			ipa3_write_rsrc_grp_type_reg(j, i, false, &val);
-		}
-	}
-
-	/* move rx_hps resource group configuration from HLOS to TZ
-	 * on real platform with IPA 3.1 or later
-	 */
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v3_1 ||
-		ipa3_ctx->ipa3_hw_mode == IPA_HW_MODE_VIRTUAL ||
-		ipa3_ctx->ipa3_hw_mode == IPA_HW_MODE_EMULATION) {
-		ipa3_configure_rx_hps();
-	}
-
-	IPADBG("EXIT\n");
-}
-
-static void ipa3_gsi_poll_after_suspend(struct ipa3_ep_context *ep)
-{
-	bool empty;
-
-	IPADBG("switch ch %ld to poll\n", ep->gsi_chan_hdl);
-	gsi_config_channel_mode(ep->gsi_chan_hdl, GSI_CHAN_MODE_POLL);
-	gsi_is_channel_empty(ep->gsi_chan_hdl, &empty);
-	if (!empty) {
-		IPADBG("ch %ld not empty\n", ep->gsi_chan_hdl);
-		/* queue a work to start polling if don't have one */
-		atomic_set(&ipa3_ctx->transport_pm.eot_activity, 1);
-		if (!atomic_read(&ep->sys->curr_polling_state))
-			__ipa_gsi_irq_rx_scedule_poll(ep->sys);
-	}
-}
-
-static int __ipa3_stop_gsi_channel(u32 clnt_hdl)
-{
-	struct ipa_mem_buffer mem;
-	int res = 0;
-	int i;
-	struct ipa3_ep_context *ep;
-	enum ipa_client_type client_type;
-	struct IpaHwOffloadStatsAllocCmdData_t *gsi_info;
-
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-		ipa3_ctx->ep[clnt_hdl].valid == 0) {
-		IPAERR("bad parm.\n");
-		return -EINVAL;
-	}
-
-	ep = &ipa3_ctx->ep[clnt_hdl];
-	client_type = ipa3_get_client_mapping(clnt_hdl);
-	memset(&mem, 0, sizeof(mem));
-
-	/* stop uC gsi dbg stats monitor */
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_5 &&
-		ipa3_ctx->ipa_hw_type != IPA_HW_v4_7) {
-		switch (client_type) {
-		case IPA_CLIENT_MHI_PRIME_TETH_PROD:
-			gsi_info = &ipa3_ctx->gsi_info[IPA_HW_PROTOCOL_MHIP];
-			gsi_info->ch_id_info[0].ch_id = 0xff;
-			gsi_info->ch_id_info[0].dir = DIR_PRODUCER;
-			ipa3_uc_debug_stats_alloc(*gsi_info);
-			break;
-		case IPA_CLIENT_MHI_PRIME_TETH_CONS:
-			gsi_info = &ipa3_ctx->gsi_info[IPA_HW_PROTOCOL_MHIP];
-			gsi_info->ch_id_info[1].ch_id = 0xff;
-			gsi_info->ch_id_info[1].dir = DIR_CONSUMER;
-			ipa3_uc_debug_stats_alloc(*gsi_info);
-			break;
-		case IPA_CLIENT_MHI_PRIME_RMNET_PROD:
-			gsi_info = &ipa3_ctx->gsi_info[IPA_HW_PROTOCOL_MHIP];
-			gsi_info->ch_id_info[2].ch_id = 0xff;
-			gsi_info->ch_id_info[2].dir = DIR_PRODUCER;
-			ipa3_uc_debug_stats_alloc(*gsi_info);
-			break;
-		case IPA_CLIENT_MHI_PRIME_RMNET_CONS:
-			gsi_info = &ipa3_ctx->gsi_info[IPA_HW_PROTOCOL_MHIP];
-			gsi_info->ch_id_info[3].ch_id = 0xff;
-			gsi_info->ch_id_info[3].dir = DIR_CONSUMER;
-			ipa3_uc_debug_stats_alloc(*gsi_info);
-			break;
-		case IPA_CLIENT_USB_PROD:
-			gsi_info = &ipa3_ctx->gsi_info[IPA_HW_PROTOCOL_USB];
-			gsi_info->ch_id_info[0].ch_id = 0xff;
-			gsi_info->ch_id_info[0].dir = DIR_PRODUCER;
-			ipa3_uc_debug_stats_alloc(*gsi_info);
-			break;
-		case IPA_CLIENT_USB_CONS:
-			gsi_info = &ipa3_ctx->gsi_info[IPA_HW_PROTOCOL_USB];
-			gsi_info->ch_id_info[1].ch_id = 0xff;
-			gsi_info->ch_id_info[1].dir = DIR_CONSUMER;
-			ipa3_uc_debug_stats_alloc(*gsi_info);
-			break;
-		default:
-			IPADBG("client_type %d not supported\n",
-				client_type);
-		}
-	}
-	if (IPA_CLIENT_IS_PROD(ep->client)) {
-		IPADBG("Calling gsi_stop_channel ch:%lu\n",
-			ep->gsi_chan_hdl);
-		res = gsi_stop_channel(ep->gsi_chan_hdl);
-		IPADBG("gsi_stop_channel ch: %lu returned %d\n",
-			ep->gsi_chan_hdl, res);
-		return res;
-	}
-
-	for (i = 0; i < IPA_GSI_CHANNEL_STOP_MAX_RETRY; i++) {
-		IPADBG("Calling gsi_stop_channel ch:%lu\n",
-			ep->gsi_chan_hdl);
-		res = gsi_stop_channel(ep->gsi_chan_hdl);
-		IPADBG("gsi_stop_channel ch: %lu returned %d\n",
-			ep->gsi_chan_hdl, res);
-		if (res != -GSI_STATUS_AGAIN && res != -GSI_STATUS_TIMED_OUT)
-			return res;
-		/*
-		 * From >=IPA4.0 version not required to send dma send command,
-		 * this issue was fixed in latest versions.
-		 */
-		if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_0) {
-			IPADBG("Inject a DMA_TASK with 1B packet to IPA\n");
-			/* Send a 1B packet DMA_TASK to IPA and try again */
-			res = ipa3_inject_dma_task_for_gsi();
-			if (res) {
-				IPAERR("Failed to inject DMA TASk for GSI\n");
-				return res;
-			}
-		}
-		/* sleep for short period to flush IPA */
-		usleep_range(IPA_GSI_CHANNEL_STOP_SLEEP_MIN_USEC,
-			IPA_GSI_CHANNEL_STOP_SLEEP_MAX_USEC);
-	}
-
-	IPAERR("Failed  to stop GSI channel with retries\n");
-	return -EFAULT;
-}
-
-/**
- * ipa3_stop_gsi_channel()- Stops a GSI channel in IPA
- * @chan_hdl: GSI channel handle
- *
- * This function implements the sequence to stop a GSI channel
- * in IPA. This function returns when the channel is in STOP state.
- *
- * Return value: 0 on success, negative otherwise
- */
-int ipa3_stop_gsi_channel(u32 clnt_hdl)
-{
-	int res;
-
-	IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-	res = __ipa3_stop_gsi_channel(clnt_hdl);
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-
-	return res;
-}
-
-void ipa3_suspend_apps_pipes(bool suspend)
-{
-	struct ipa_ep_cfg_ctrl cfg;
-	int ipa_ep_idx;
-	struct ipa3_ep_context *ep;
-	int res;
-
-	memset(&cfg, 0, sizeof(cfg));
-	cfg.ipa_ep_suspend = suspend;
-
-	ipa_ep_idx = ipa3_get_ep_mapping(IPA_CLIENT_APPS_LAN_CONS);
-	if (ipa_ep_idx < 0) {
-		IPAERR("IPA client mapping failed\n");
-		ipa_assert();
-		return;
-	}
-	ep = &ipa3_ctx->ep[ipa_ep_idx];
-	if (ep->valid) {
-		IPADBG("%s pipe %d\n", suspend ? "suspend" : "unsuspend",
-			ipa_ep_idx);
-		/*
-		 * move the channel to callback mode.
-		 * This needs to happen before starting the channel to make
-		 * sure we don't loose any interrupt
-		 */
-		if (!suspend && !atomic_read(&ep->sys->curr_polling_state))
-			gsi_config_channel_mode(ep->gsi_chan_hdl,
-				GSI_CHAN_MODE_CALLBACK);
-
-		if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0) {
-			if (suspend) {
-				res = __ipa3_stop_gsi_channel(ipa_ep_idx);
-				if (res) {
-					IPAERR("failed to stop LAN channel\n");
-					ipa_assert();
-				}
-			} else {
-				res = gsi_start_channel(ep->gsi_chan_hdl);
-				if (res) {
-					IPAERR("failed to start LAN channel\n");
-					ipa_assert();
-				}
-			}
-		} else {
-			ipa3_cfg_ep_ctrl(ipa_ep_idx, &cfg);
-		}
-		if (suspend)
-			ipa3_gsi_poll_after_suspend(ep);
-	}
-
-	ipa_ep_idx = ipa_get_ep_mapping(IPA_CLIENT_APPS_WAN_CONS);
-	/* Considering the case for SSR. */
-	if (ipa_ep_idx == -1) {
-		IPADBG("Invalid mapping for IPA_CLIENT_APPS_WAN_CONS\n");
-		return;
-	}
-	ep = &ipa3_ctx->ep[ipa_ep_idx];
-	if (ep->valid) {
-		IPADBG("%s pipe %d\n", suspend ? "suspend" : "unsuspend",
-			ipa_ep_idx);
-		/*
-		 * move the channel to callback mode.
-		 * This needs to happen before starting the channel to make
-		 * sure we don't loose any interrupt
-		 */
-		if (!suspend && !atomic_read(&ep->sys->curr_polling_state))
-			gsi_config_channel_mode(ep->gsi_chan_hdl,
-				GSI_CHAN_MODE_CALLBACK);
-		if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0) {
-			if (suspend) {
-				res = __ipa3_stop_gsi_channel(ipa_ep_idx);
-				if (res) {
-					IPAERR("failed to stop WAN channel\n");
-					ipa_assert();
-				}
-			} else if (!atomic_read(&ipa3_ctx->is_ssr)) {
-				/* If SSR was alreday started not required to
-				 * start WAN channel,Because in SSR will stop
-				 * channel and reset the channel.
-				 */
-				res = gsi_start_channel(ep->gsi_chan_hdl);
-				if (res) {
-					IPAERR("failed to start WAN channel\n");
-					ipa_assert();
-				}
-			}
-		} else {
-			ipa3_cfg_ep_ctrl(ipa_ep_idx, &cfg);
-		}
-		if (suspend)
-			ipa3_gsi_poll_after_suspend(ep);
-	}
-
-	ipa_ep_idx = ipa_get_ep_mapping(IPA_CLIENT_ODL_DPL_CONS);
-	/* Considering the case for SSR. */
-	if (ipa_ep_idx == -1) {
-		IPADBG("Invalid mapping for IPA_CLIENT_ODL_DPL_CONS\n");
-		return;
-	}
-	ep = &ipa3_ctx->ep[ipa_ep_idx];
-	if (ep->valid) {
-		IPADBG("%s pipe %d\n", suspend ? "suspend" : "unsuspend",
-			ipa_ep_idx);
-		/*
-		 * move the channel to callback mode.
-		 * This needs to happen before starting the channel to make
-		 * sure we don't loose any interrupt
-		 */
-		if (!suspend && !atomic_read(&ep->sys->curr_polling_state))
-			gsi_config_channel_mode(ep->gsi_chan_hdl,
-			GSI_CHAN_MODE_CALLBACK);
-		if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_0) {
-			if (suspend) {
-				res = __ipa3_stop_gsi_channel(ipa_ep_idx);
-				if (res) {
-					IPAERR("failed to stop ODL channel\n");
-					ipa_assert();
-				}
-			} else if (!atomic_read(&ipa3_ctx->is_ssr)) {
-				/* If SSR was alreday started not required to
-				 * start WAN channel,Because in SSR will stop
-				 * channel and reset the channel.
-				 */
-				res = gsi_start_channel(ep->gsi_chan_hdl);
-				if (res) {
-					IPAERR("failed to start ODL channel\n");
-					ipa_assert();
-				}
-			}
-		} else {
-			ipa3_cfg_ep_ctrl(ipa_ep_idx, &cfg);
-		}
-		if (suspend)
-			ipa3_gsi_poll_after_suspend(ep);
-	}
-}
-
-int ipa3_allocate_dma_task_for_gsi(void)
-{
-	struct ipahal_imm_cmd_dma_task_32b_addr cmd = { 0 };
-
-	IPADBG("Allocate mem\n");
-	ipa3_ctx->dma_task_info.mem.size = IPA_GSI_CHANNEL_STOP_PKT_SIZE;
-	ipa3_ctx->dma_task_info.mem.base = dma_alloc_coherent(ipa3_ctx->pdev,
-		ipa3_ctx->dma_task_info.mem.size,
-		&ipa3_ctx->dma_task_info.mem.phys_base,
-		GFP_KERNEL);
-	if (!ipa3_ctx->dma_task_info.mem.base) {
-		IPAERR("no mem\n");
-		return -EFAULT;
-	}
-
-	cmd.flsh = true;
-	cmd.size1 = ipa3_ctx->dma_task_info.mem.size;
-	cmd.addr1 = ipa3_ctx->dma_task_info.mem.phys_base;
-	cmd.packet_size = ipa3_ctx->dma_task_info.mem.size;
-	ipa3_ctx->dma_task_info.cmd_pyld = ipahal_construct_imm_cmd(
-			IPA_IMM_CMD_DMA_TASK_32B_ADDR, &cmd, false);
-	if (!ipa3_ctx->dma_task_info.cmd_pyld) {
-		IPAERR("failed to construct dma_task_32b_addr cmd\n");
-		dma_free_coherent(ipa3_ctx->pdev,
-			ipa3_ctx->dma_task_info.mem.size,
-			ipa3_ctx->dma_task_info.mem.base,
-			ipa3_ctx->dma_task_info.mem.phys_base);
-		memset(&ipa3_ctx->dma_task_info, 0,
-			sizeof(ipa3_ctx->dma_task_info));
-		return -EFAULT;
-	}
-
-	return 0;
-}
-
-void ipa3_free_dma_task_for_gsi(void)
-{
-	dma_free_coherent(ipa3_ctx->pdev,
-		ipa3_ctx->dma_task_info.mem.size,
-		ipa3_ctx->dma_task_info.mem.base,
-		ipa3_ctx->dma_task_info.mem.phys_base);
-	ipahal_destroy_imm_cmd(ipa3_ctx->dma_task_info.cmd_pyld);
-	memset(&ipa3_ctx->dma_task_info, 0, sizeof(ipa3_ctx->dma_task_info));
-}
-
-/**
- * ipa3_inject_dma_task_for_gsi()- Send DMA_TASK to IPA for GSI stop channel
- *
- * Send a DMA_TASK of 1B to IPA to unblock GSI channel in STOP_IN_PROG.
- * Return value: 0 on success, negative otherwise
- */
-int ipa3_inject_dma_task_for_gsi(void)
-{
-	struct ipa3_desc desc;
-
-	ipa3_init_imm_cmd_desc(&desc, ipa3_ctx->dma_task_info.cmd_pyld);
-
-	IPADBG("sending 1B packet to IPA\n");
-	if (ipa3_send_cmd_timeout(1, &desc,
-		IPA_DMA_TASK_FOR_GSI_TIMEOUT_MSEC)) {
-		IPAERR("ipa3_send_cmd failed\n");
-		return -EFAULT;
-	}
-
-	return 0;
-}
-
-static int ipa3_load_single_fw(const struct firmware *firmware,
-	const struct elf32_phdr *phdr)
-{
-	uint32_t *fw_mem_base;
-	int index;
-	const uint32_t *elf_data_ptr;
-
-	if (phdr->p_offset > firmware->size) {
-		IPAERR("Invalid ELF: offset=%u is beyond elf_size=%zu\n",
-			phdr->p_offset, firmware->size);
-		return -EINVAL;
-	}
-	if ((firmware->size - phdr->p_offset) < phdr->p_filesz) {
-		IPAERR("Invalid ELF: offset=%u filesz=%u elf_size=%zu\n",
-			phdr->p_offset, phdr->p_filesz, firmware->size);
-		return -EINVAL;
-	}
-
-	if (phdr->p_memsz % sizeof(uint32_t)) {
-		IPAERR("FW mem size %u doesn't align to 32bit\n",
-			phdr->p_memsz);
-		return -EFAULT;
-	}
-
-	if (phdr->p_filesz > phdr->p_memsz) {
-		IPAERR("FW image too big src_size=%u dst_size=%u\n",
-			phdr->p_filesz, phdr->p_memsz);
-		return -EFAULT;
-	}
-
-	fw_mem_base = ioremap(phdr->p_vaddr, phdr->p_memsz);
-	if (!fw_mem_base) {
-		IPAERR("Failed to map 0x%x for the size of %u\n",
-			phdr->p_vaddr, phdr->p_memsz);
-		return -ENOMEM;
-	}
-
-	/* Set the entire region to 0s */
-	memset(fw_mem_base, 0, phdr->p_memsz);
-
-	elf_data_ptr = (uint32_t *)(firmware->data + phdr->p_offset);
-
-	/* Write the FW */
-	for (index = 0; index < phdr->p_filesz/sizeof(uint32_t); index++) {
-		writel_relaxed(*elf_data_ptr, &fw_mem_base[index]);
-		elf_data_ptr++;
-	}
-
-	iounmap(fw_mem_base);
-
-	return 0;
-}
-
-struct ipa3_hps_dps_areas_info {
-	u32 dps_abs_addr;
-	u32 dps_sz;
-	u32 hps_abs_addr;
-	u32 hps_sz;
-};
-
-static void ipa3_get_hps_dps_areas_absolute_addr_and_sz(
-	struct ipa3_hps_dps_areas_info *info)
-{
-	u32 dps_area_start;
-	u32 dps_area_end;
-	u32 hps_area_start;
-	u32 hps_area_end;
-
-	if (ipa3_ctx->ipa_hw_type < IPA_HW_v4_5) {
-		dps_area_start = ipahal_get_reg_ofst(IPA_DPS_SEQUENCER_FIRST);
-		dps_area_end = ipahal_get_reg_ofst(IPA_DPS_SEQUENCER_LAST);
-		hps_area_start = ipahal_get_reg_ofst(IPA_HPS_SEQUENCER_FIRST);
-		hps_area_end = ipahal_get_reg_ofst(IPA_HPS_SEQUENCER_LAST);
-
-		info->dps_abs_addr = ipa3_ctx->ipa_wrapper_base +
-			ipahal_get_reg_base() + dps_area_start;
-		info->hps_abs_addr = ipa3_ctx->ipa_wrapper_base +
-			ipahal_get_reg_base() + hps_area_start;
-	} else {
-		dps_area_start = ipahal_read_reg(IPA_DPS_SEQUENCER_FIRST);
-		dps_area_end = ipahal_read_reg(IPA_DPS_SEQUENCER_LAST);
-		hps_area_start = ipahal_read_reg(IPA_HPS_SEQUENCER_FIRST);
-		hps_area_end = ipahal_read_reg(IPA_HPS_SEQUENCER_LAST);
-
-		info->dps_abs_addr = ipa3_ctx->ipa_wrapper_base +
-			dps_area_start;
-		info->hps_abs_addr = ipa3_ctx->ipa_wrapper_base +
-			hps_area_start;
-	}
-
-	info->dps_sz = dps_area_end - dps_area_start + sizeof(u32);
-	info->hps_sz = hps_area_end - hps_area_start + sizeof(u32);
-
-	IPADBG("dps area: start offset=0x%x end offset=0x%x\n",
-		dps_area_start, dps_area_end);
-	IPADBG("hps area: start offset=0x%x end offset=0x%x\n",
-		hps_area_start, hps_area_end);
-}
-
-/**
- * emulator_load_single_fw() - load firmware into emulator's memory
- *
- * @firmware: Structure which contains the FW data from the user space.
- * @phdr: ELF program header
- * @loc_to_map: physical location to map into virtual space
- * @size_to_map: the size of memory to map into virtual space
- *
- * Return value: 0 on success, negative otherwise
- */
-static int emulator_load_single_fw(
-	const struct firmware   *firmware,
-	const struct elf32_phdr *phdr,
-	u32                      loc_to_map,
-	u32                      size_to_map)
-{
-	int index;
-	uint32_t ofb;
-	const uint32_t *elf_data_ptr;
-	void __iomem *fw_base;
-
-	IPADBG("firmware(%pK) phdr(%pK) loc_to_map(0x%X) size_to_map(%u)\n",
-	       firmware, phdr, loc_to_map, size_to_map);
-
-	if (phdr->p_offset > firmware->size) {
-		IPAERR("Invalid ELF: offset=%u is beyond elf_size=%zu\n",
-			phdr->p_offset, firmware->size);
-		return -EINVAL;
-	}
-	if ((firmware->size - phdr->p_offset) < phdr->p_filesz) {
-		IPAERR("Invalid ELF: offset=%u filesz=%u elf_size=%zu\n",
-			phdr->p_offset, phdr->p_filesz, firmware->size);
-		return -EINVAL;
-	}
-
-	if (phdr->p_memsz % sizeof(uint32_t)) {
-		IPAERR("FW mem size %u doesn't align to 32bit\n",
-			phdr->p_memsz);
-		return -EFAULT;
-	}
-
-	if (phdr->p_filesz > phdr->p_memsz) {
-		IPAERR("FW image too big src_size=%u dst_size=%u\n",
-			phdr->p_filesz, phdr->p_memsz);
-		return -EFAULT;
-	}
-
-	IPADBG("ELF: p_memsz(0x%x) p_filesz(0x%x) p_filesz/4(0x%x)\n",
-	       (uint32_t) phdr->p_memsz,
-	       (uint32_t) phdr->p_filesz,
-	       (uint32_t) (phdr->p_filesz/sizeof(uint32_t)));
-
-	fw_base = ioremap(loc_to_map, size_to_map);
-	if (!fw_base) {
-		IPAERR("Failed to map 0x%X for the size of %u\n",
-		       loc_to_map, size_to_map);
-		return -ENOMEM;
-	}
-
-	IPADBG("Physical base(0x%X) mapped to virtual (%pK) with len (%u)\n",
-	       loc_to_map,
-	       fw_base,
-	       size_to_map);
-
-	/* Set the entire region to 0s */
-	ofb = 0;
-	for (index = 0; index < phdr->p_memsz/sizeof(uint32_t); index++) {
-		writel_relaxed(0, fw_base + ofb);
-		ofb += sizeof(uint32_t);
-	}
-
-	elf_data_ptr = (uint32_t *)(firmware->data + phdr->p_offset);
-
-	/* Write the FW */
-	ofb = 0;
-	for (index = 0; index < phdr->p_filesz/sizeof(uint32_t); index++) {
-		writel_relaxed(*elf_data_ptr, fw_base + ofb);
-		elf_data_ptr++;
-		ofb += sizeof(uint32_t);
-	}
-
-	iounmap(fw_base);
-
-	return 0;
-}
-
-/**
- * ipa3_load_fws() - Load the IPAv3 FWs into IPA&GSI SRAM.
- *
- * @firmware: Structure which contains the FW data from the user space.
- * @gsi_mem_base: GSI base address
- * @gsi_ver: GSI Version
- *
- * Return value: 0 on success, negative otherwise
- *
- */
-int ipa3_load_fws(const struct firmware *firmware, phys_addr_t gsi_mem_base,
-	enum gsi_ver gsi_ver)
-{
-	const struct elf32_hdr *ehdr;
-	const struct elf32_phdr *phdr;
-	unsigned long gsi_iram_ofst;
-	unsigned long gsi_iram_size;
-	int rc;
-	struct ipa3_hps_dps_areas_info dps_hps_info;
-
-	if (gsi_ver == GSI_VER_ERR) {
-		IPAERR("Invalid GSI Version\n");
-		return -EINVAL;
-	}
-
-	if (!gsi_mem_base) {
-		IPAERR("Invalid GSI base address\n");
-		return -EINVAL;
-	}
-
-	ipa_assert_on(!firmware);
-	/* One program header per FW image: GSI, DPS and HPS */
-	if (firmware->size < (sizeof(*ehdr) + 3 * sizeof(*phdr))) {
-		IPAERR("Missing ELF and Program headers firmware size=%zu\n",
-			firmware->size);
-		return -EINVAL;
-	}
-
-	ehdr = (struct elf32_hdr *) firmware->data;
-	ipa_assert_on(!ehdr);
-	if (ehdr->e_phnum != 3) {
-		IPAERR("Unexpected number of ELF program headers\n");
-		return -EINVAL;
-	}
-	phdr = (struct elf32_phdr *)(firmware->data + sizeof(*ehdr));
-
-	/*
-	 * Each ELF program header represents a FW image and contains:
-	 *  p_vaddr : The starting address to which the FW needs to loaded.
-	 *  p_memsz : The size of the IRAM (where the image loaded)
-	 *  p_filesz: The size of the FW image embedded inside the ELF
-	 *  p_offset: Absolute offset to the image from the head of the ELF
-	 */
-
-	/* Load GSI FW image */
-	gsi_get_inst_ram_offset_and_size(&gsi_iram_ofst, &gsi_iram_size,
-		gsi_ver);
-	if (phdr->p_vaddr != (gsi_mem_base + gsi_iram_ofst)) {
-		IPAERR(
-			"Invalid GSI FW img load addr vaddr=0x%x gsi_mem_base=%pa gsi_iram_ofst=0x%lx\n"
-			, phdr->p_vaddr, &gsi_mem_base, gsi_iram_ofst);
-		return -EINVAL;
-	}
-	if (phdr->p_memsz > gsi_iram_size) {
-		IPAERR("Invalid GSI FW img size memsz=%d gsi_iram_size=%lu\n",
-			phdr->p_memsz, gsi_iram_size);
-		return -EINVAL;
-	}
-	rc = ipa3_load_single_fw(firmware, phdr);
-	if (rc)
-		return rc;
-
-	phdr++;
-	ipa3_get_hps_dps_areas_absolute_addr_and_sz(&dps_hps_info);
-
-	/* Load IPA DPS FW image */
-	if (phdr->p_vaddr != dps_hps_info.dps_abs_addr) {
-		IPAERR(
-			"Invalid IPA DPS img load addr vaddr=0x%x dps_abs_addr=0x%x\n"
-			, phdr->p_vaddr, dps_hps_info.dps_abs_addr);
-		return -EINVAL;
-	}
-	if (phdr->p_memsz > dps_hps_info.dps_sz) {
-		IPAERR("Invalid IPA DPS img size memsz=%d dps_area_size=%u\n",
-			phdr->p_memsz, dps_hps_info.dps_sz);
-		return -EINVAL;
-	}
-	rc = ipa3_load_single_fw(firmware, phdr);
-	if (rc)
-		return rc;
-
-	phdr++;
-
-	/* Load IPA HPS FW image */
-	if (phdr->p_vaddr != dps_hps_info.hps_abs_addr) {
-		IPAERR(
-			"Invalid IPA HPS img load addr vaddr=0x%x hps_abs_addr=0x%x\n"
-			, phdr->p_vaddr, dps_hps_info.hps_abs_addr);
-		return -EINVAL;
-	}
-	if (phdr->p_memsz > dps_hps_info.hps_sz) {
-		IPAERR("Invalid IPA HPS img size memsz=%d hps_area_size=%u\n",
-			phdr->p_memsz, dps_hps_info.hps_sz);
-		return -EINVAL;
-	}
-	rc = ipa3_load_single_fw(firmware, phdr);
-	if (rc)
-		return rc;
-
-	IPADBG("IPA FWs (GSI FW, DPS and HPS) loaded successfully\n");
-	return 0;
-}
-
-/*
- * The following needed for the EMULATION system. On a non-emulation
- * system (ie. the real UE), this functionality is done in the
- * TZ...
- */
-
-static void ipa_gsi_setup_reg(void)
-{
-	u32 reg_val, start;
-	int i;
-	const struct ipa_gsi_ep_config *gsi_ep_info_cfg;
-	enum ipa_client_type type;
-
-	IPADBG("Setting up registers in preparation for firmware download\n");
-
-	/* setup IPA_ENDP_GSI_CFG_TLV_n reg */
-	start = 0;
-	ipa3_ctx->ipa_num_pipes = ipa3_get_num_pipes();
-	IPADBG("ipa_num_pipes=%u\n", ipa3_ctx->ipa_num_pipes);
-
-	for (i = 0; i < ipa3_ctx->ipa_num_pipes; i++) {
-		type = ipa3_get_client_by_pipe(i);
-		gsi_ep_info_cfg = ipa3_get_gsi_ep_info(type);
-		IPAERR("for ep %d client is %d gsi_ep_info_cfg=%pK\n",
-			i, type, gsi_ep_info_cfg);
-		if (!gsi_ep_info_cfg)
-			continue;
-		reg_val = ((gsi_ep_info_cfg->ipa_if_tlv << 16) & 0x00FF0000);
-		reg_val += (start & 0xFFFF);
-		start += gsi_ep_info_cfg->ipa_if_tlv;
-		ipahal_write_reg_n(IPA_ENDP_GSI_CFG_TLV_n, i, reg_val);
-	}
-
-	/* setup IPA_ENDP_GSI_CFG_AOS_n reg */
-	for (i = 0; i < ipa3_ctx->ipa_num_pipes; i++) {
-		type = ipa3_get_client_by_pipe(i);
-		gsi_ep_info_cfg = ipa3_get_gsi_ep_info(type);
-		if (!gsi_ep_info_cfg)
-			continue;
-		reg_val = ((gsi_ep_info_cfg->ipa_if_aos << 16) & 0x00FF0000);
-		reg_val += (start & 0xFFFF);
-		start += gsi_ep_info_cfg->ipa_if_aos;
-		ipahal_write_reg_n(IPA_ENDP_GSI_CFG_AOS_n, i, reg_val);
-	}
-
-	/* setup GSI_MAP_EE_n_CH_k_VP_TABLE reg */
-	for (i = 0; i < ipa3_ctx->ipa_num_pipes; i++) {
-		type = ipa3_get_client_by_pipe(i);
-		gsi_ep_info_cfg = ipa3_get_gsi_ep_info(type);
-		if (!gsi_ep_info_cfg)
-			continue;
-		reg_val = i & 0x1F;
-		gsi_map_virtual_ch_to_per_ep(
-			gsi_ep_info_cfg->ee,
-			gsi_ep_info_cfg->ipa_gsi_chan_num,
-			reg_val);
-	}
-
-	/* setup IPA_ENDP_GSI_CFG1_n reg */
-	for (i = 0; i < ipa3_ctx->ipa_num_pipes; i++) {
-		type = ipa3_get_client_by_pipe(i);
-		gsi_ep_info_cfg = ipa3_get_gsi_ep_info(type);
-		if (!gsi_ep_info_cfg)
-			continue;
-		reg_val = (1 << 31) + (1 << 16);
-		ipahal_write_reg_n(IPA_ENDP_GSI_CFG1_n, i, 1<<16);
-		ipahal_write_reg_n(IPA_ENDP_GSI_CFG1_n, i, reg_val);
-		ipahal_write_reg_n(IPA_ENDP_GSI_CFG1_n, i, 1<<16);
-	}
-}
-
-/**
- * emulator_load_fws() - Load the IPAv3 FWs into IPA&GSI SRAM.
- *
- * @firmware: Structure which contains the FW data from the user space.
- * @transport_mem_base: Where to load
- * @transport_mem_size: Space available to load into
- * @gsi_ver: Version of the gsi
- *
- * Return value: 0 on success, negative otherwise
- */
-int emulator_load_fws(
-	const struct firmware *firmware,
-	u32 transport_mem_base,
-	u32 transport_mem_size,
-	enum gsi_ver gsi_ver)
-{
-	const struct elf32_hdr *ehdr;
-	const struct elf32_phdr *phdr;
-	unsigned long gsi_offset, gsi_ram_size;
-	struct ipa3_hps_dps_areas_info dps_hps_info;
-	int rc;
-
-	IPADBG("Loading firmware(%pK)\n", firmware);
-
-	if (!firmware) {
-		IPAERR("firmware pointer passed to function is NULL\n");
-		return -EINVAL;
-	}
-
-	/* One program header per FW image: GSI, DPS and HPS */
-	if (firmware->size < (sizeof(*ehdr) + 3 * sizeof(*phdr))) {
-		IPAERR(
-		    "Missing ELF and Program headers firmware size=%zu\n",
-		    firmware->size);
-		return -EINVAL;
-	}
-
-	ehdr = (struct elf32_hdr *) firmware->data;
-
-	ipa_assert_on(!ehdr);
-
-	if (ehdr->e_phnum != 3) {
-		IPAERR("Unexpected number of ELF program headers\n");
-		return -EINVAL;
-	}
-
-	ipa3_get_hps_dps_areas_absolute_addr_and_sz(&dps_hps_info);
-
-	/*
-	 * Each ELF program header represents a FW image and contains:
-	 *  p_vaddr : The starting address to which the FW needs to loaded.
-	 *  p_memsz : The size of the IRAM (where the image loaded)
-	 *  p_filesz: The size of the FW image embedded inside the ELF
-	 *  p_offset: Absolute offset to the image from the head of the ELF
-	 *
-	 * NOTE WELL: On the emulation platform, the p_vaddr address
-	 *            is not relevant and is unused.  This is because
-	 *            on the emulation platform, the registers'
-	 *            address location is mutable, since it's mapped
-	 *            in via a PCIe probe.  Given this, it is the
-	 *            mapped address info that's used while p_vaddr is
-	 *            ignored.
-	 */
-	phdr = (struct elf32_phdr *)(firmware->data + sizeof(*ehdr));
-
-	phdr += 2;
-
-	/*
-	 * Attempt to load IPA HPS FW image
-	 */
-	if (phdr->p_memsz > dps_hps_info.hps_sz) {
-		IPAERR("Invalid IPA HPS img size memsz=%d hps_size=%u\n",
-		       phdr->p_memsz, dps_hps_info.hps_sz);
-		return -EINVAL;
-	}
-	IPADBG("Loading HPS FW\n");
-	rc = emulator_load_single_fw(
-		firmware, phdr,
-		dps_hps_info.hps_abs_addr, dps_hps_info.hps_sz);
-	if (rc)
-		return rc;
-	IPADBG("Loading HPS FW complete\n");
-
-	--phdr;
-
-	/*
-	 * Attempt to load IPA DPS FW image
-	 */
-	if (phdr->p_memsz > dps_hps_info.dps_sz) {
-		IPAERR("Invalid IPA DPS img size memsz=%d dps_size=%u\n",
-		       phdr->p_memsz, dps_hps_info.dps_sz);
-		return -EINVAL;
-	}
-	IPADBG("Loading DPS FW\n");
-	rc = emulator_load_single_fw(
-		firmware, phdr,
-		dps_hps_info.dps_abs_addr, dps_hps_info.dps_sz);
-	if (rc)
-		return rc;
-	IPADBG("Loading DPS FW complete\n");
-
-	/*
-	 * Run gsi register setup which is normally done in TZ on
-	 * non-EMULATION systems...
-	 */
-	ipa_gsi_setup_reg();
-
-	--phdr;
-
-	gsi_get_inst_ram_offset_and_size(&gsi_offset, &gsi_ram_size, gsi_ver);
-
-	/*
-	 * Attempt to load GSI FW image
-	 */
-	if (phdr->p_memsz > gsi_ram_size) {
-		IPAERR(
-		    "Invalid GSI FW img size memsz=%d gsi_ram_size=%lu\n",
-		    phdr->p_memsz, gsi_ram_size);
-		return -EINVAL;
-	}
-	IPADBG("Loading GSI FW\n");
-	rc = emulator_load_single_fw(
-		firmware, phdr,
-		transport_mem_base + (u32) gsi_offset, gsi_ram_size);
-	if (rc)
-		return rc;
-	IPADBG("Loading GSI FW complete\n");
-
-	IPADBG("IPA FWs (GSI FW, DPS and HPS) loaded successfully\n");
-
-	return 0;
-}
-
-/**
- * ipa3_is_apq() - indicate apq platform or not
- *
- * Return value: true if apq, false if not apq platform
- *
- */
-bool ipa3_is_apq(void)
-{
-	if (ipa3_ctx->platform_type == IPA_PLAT_TYPE_APQ)
-		return true;
-	else
-		return false;
-}
-
-/**
- * ipa3_disable_prefetch() - disable\enable tx prefetch
- *
- * @client: the client which is related to the TX where prefetch will be
- *          disabled
- *
- * Return value: Non applicable
- *
- */
-void ipa3_disable_prefetch(enum ipa_client_type client)
-{
-	struct ipahal_reg_tx_cfg cfg;
-	u8 qmb;
-
-	qmb = ipa3_get_qmb_master_sel(client);
-
-	IPADBG("disabling prefetch for qmb %d\n", (int)qmb);
-
-	ipahal_read_reg_fields(IPA_TX_CFG, &cfg);
-	/* QMB0 (DDR) correlates with TX0, QMB1(PCIE) correlates with TX1 */
-	if (qmb == QMB_MASTER_SELECT_DDR)
-		cfg.tx0_prefetch_disable = true;
-	else
-		cfg.tx1_prefetch_disable = true;
-	ipahal_write_reg_fields(IPA_TX_CFG, &cfg);
-}
-
-/**
- * ipa3_get_pdev() - return a pointer to IPA dev struct
- *
- * Return value: a pointer to IPA dev struct
- *
- */
-struct device *ipa3_get_pdev(void)
-{
-	if (!ipa3_ctx)
-		return NULL;
-
-	return ipa3_ctx->pdev;
-}
-
-/**
- * ipa3_enable_dcd() - enable dynamic clock division on IPA
- *
- * Return value: Non applicable
- *
- */
-void ipa3_enable_dcd(void)
-{
-	struct ipahal_reg_idle_indication_cfg idle_indication_cfg;
-
-	/* recommended values for IPA 3.5 according to IPA HPG */
-	idle_indication_cfg.const_non_idle_enable = false;
-	idle_indication_cfg.enter_idle_debounce_thresh = 256;
-
-	ipahal_write_reg_fields(IPA_IDLE_INDICATION_CFG,
-			&idle_indication_cfg);
-}
-
-void ipa3_init_imm_cmd_desc(struct ipa3_desc *desc,
-	struct ipahal_imm_cmd_pyld *cmd_pyld)
-{
-	memset(desc, 0, sizeof(*desc));
-	desc->opcode = cmd_pyld->opcode;
-	desc->pyld = cmd_pyld->data;
-	desc->len = cmd_pyld->len;
-	desc->type = IPA_IMM_CMD_DESC;
-}
-
-u32 ipa3_get_r_rev_version(void)
-{
-	static u32 r_rev;
-
-	if (r_rev != 0)
-		return r_rev;
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-	r_rev = ipahal_read_reg(IPA_VERSION);
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-
-	return r_rev;
-}

ipa/ipa_v3/ipa_wdi3_i.c

@@ -1,962 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2018 - 2019, The Linux Foundation. All rights reserved.
- */
-
-#include "ipa_i.h"
-#include <linux/ipa_wdi3.h>
-
-#define UPDATE_RP_MODERATION_CONFIG 1
-#define UPDATE_RP_MODERATION_THRESHOLD 8
-
-#define IPA_WLAN_AGGR_PKT_LIMIT 1
-#define IPA_WLAN_AGGR_BYTE_LIMIT 2 /*2 Kbytes Agger hard byte limit*/
-
-#define IPA_WDI3_GSI_EVT_RING_INT_MODT 32
-
-static void ipa3_wdi3_gsi_evt_ring_err_cb(struct gsi_evt_err_notify *notify)
-{
-	switch (notify->evt_id) {
-	case GSI_EVT_OUT_OF_BUFFERS_ERR:
-		IPAERR("Got GSI_EVT_OUT_OF_BUFFERS_ERR\n");
-		break;
-	case GSI_EVT_OUT_OF_RESOURCES_ERR:
-		IPAERR("Got GSI_EVT_OUT_OF_RESOURCES_ERR\n");
-		break;
-	case GSI_EVT_UNSUPPORTED_INTER_EE_OP_ERR:
-		IPAERR("Got GSI_EVT_UNSUPPORTED_INTER_EE_OP_ERR\n");
-		break;
-	case GSI_EVT_EVT_RING_EMPTY_ERR:
-		IPAERR("Got GSI_EVT_EVT_RING_EMPTY_ERR\n");
-		break;
-	default:
-		IPAERR("Unexpected err evt: %d\n", notify->evt_id);
-	}
-	ipa_assert();
-}
-
-static void ipa3_wdi3_gsi_chan_err_cb(struct gsi_chan_err_notify *notify)
-{
-	switch (notify->evt_id) {
-	case GSI_CHAN_INVALID_TRE_ERR:
-		IPAERR("Got GSI_CHAN_INVALID_TRE_ERR\n");
-		break;
-	case GSI_CHAN_NON_ALLOCATED_EVT_ACCESS_ERR:
-		IPAERR("Got GSI_CHAN_NON_ALLOCATED_EVT_ACCESS_ERR\n");
-		break;
-	case GSI_CHAN_OUT_OF_BUFFERS_ERR:
-		IPAERR("Got GSI_CHAN_OUT_OF_BUFFERS_ERR\n");
-		break;
-	case GSI_CHAN_OUT_OF_RESOURCES_ERR:
-		IPAERR("Got GSI_CHAN_OUT_OF_RESOURCES_ERR\n");
-		break;
-	case GSI_CHAN_UNSUPPORTED_INTER_EE_OP_ERR:
-		IPAERR("Got GSI_CHAN_UNSUPPORTED_INTER_EE_OP_ERR\n");
-		break;
-	case GSI_CHAN_HWO_1_ERR:
-		IPAERR("Got GSI_CHAN_HWO_1_ERR\n");
-		break;
-	default:
-		IPAERR("Unexpected err evt: %d\n", notify->evt_id);
-	}
-	ipa_assert();
-}
-
-static int ipa3_setup_wdi3_gsi_channel(u8 is_smmu_enabled,
-	struct ipa_wdi_pipe_setup_info *info,
-	struct ipa_wdi_pipe_setup_info_smmu *info_smmu, u8 dir,
-	struct ipa3_ep_context *ep)
-{
-	struct gsi_evt_ring_props gsi_evt_ring_props;
-	struct gsi_chan_props gsi_channel_props;
-	union __packed gsi_channel_scratch ch_scratch;
-	union __packed gsi_evt_scratch evt_scratch;
-	const struct ipa_gsi_ep_config *gsi_ep_info;
-	int result, len;
-	unsigned long va;
-	uint32_t addr_low, addr_high;
-
-	if (!info || !info_smmu || !ep) {
-		IPAERR("invalid input\n");
-		return -EINVAL;
-	}
-	/* setup event ring */
-	memset(&gsi_evt_ring_props, 0, sizeof(gsi_evt_ring_props));
-	gsi_evt_ring_props.intf = GSI_EVT_CHTYPE_WDI3_EV;
-	gsi_evt_ring_props.intr = GSI_INTR_IRQ;
-	/* 16 (for Tx) and 8 (for Rx) */
-	if (dir == IPA_WDI3_TX_DIR)
-		gsi_evt_ring_props.re_size = GSI_EVT_RING_RE_SIZE_16B;
-	else
-		gsi_evt_ring_props.re_size = GSI_EVT_RING_RE_SIZE_8B;
-	if (!is_smmu_enabled) {
-		gsi_evt_ring_props.ring_len = info->event_ring_size;
-		gsi_evt_ring_props.ring_base_addr =
-			(u64)info->event_ring_base_pa;
-	} else {
-		len = info_smmu->event_ring_size;
-		if (dir == IPA_WDI3_TX_DIR) {
-			if (ipa_create_gsi_smmu_mapping(IPA_WDI_CE_RING_RES,
-				true, info->event_ring_base_pa,
-				&info_smmu->event_ring_base, len,
-				false, &va)) {
-				IPAERR("failed to get smmu mapping\n");
-				return -EFAULT;
-			}
-		} else {
-			if (ipa_create_gsi_smmu_mapping(
-				IPA_WDI_RX_COMP_RING_RES, true,
-				info->event_ring_base_pa,
-				&info_smmu->event_ring_base, len,
-				false, &va)) {
-				IPAERR("failed to get smmu mapping\n");
-				return -EFAULT;
-			}
-		}
-		gsi_evt_ring_props.ring_len = len;
-		gsi_evt_ring_props.ring_base_addr = (u64)va;
-	}
-	gsi_evt_ring_props.int_modt = IPA_WDI3_GSI_EVT_RING_INT_MODT;
-	gsi_evt_ring_props.int_modc = 1;
-	gsi_evt_ring_props.exclusive = true;
-	gsi_evt_ring_props.err_cb = ipa3_wdi3_gsi_evt_ring_err_cb;
-	gsi_evt_ring_props.user_data = NULL;
-
-	result = gsi_alloc_evt_ring(&gsi_evt_ring_props, ipa3_ctx->gsi_dev_hdl,
-		&ep->gsi_evt_ring_hdl);
-	if (result != GSI_STATUS_SUCCESS) {
-		IPAERR("fail to alloc RX event ring\n");
-		result = -EFAULT;
-		goto fail_smmu_mapping;
-	}
-
-	ep->gsi_mem_info.evt_ring_len = gsi_evt_ring_props.ring_len;
-	ep->gsi_mem_info.evt_ring_base_addr =
-		gsi_evt_ring_props.ring_base_addr;
-
-	/* setup channel ring */
-	memset(&gsi_channel_props, 0, sizeof(gsi_channel_props));
-	gsi_channel_props.prot = GSI_CHAN_PROT_WDI3;
-	if (dir == IPA_WDI3_TX_DIR)
-		gsi_channel_props.dir = GSI_CHAN_DIR_FROM_GSI;
-	else
-		gsi_channel_props.dir = GSI_CHAN_DIR_TO_GSI;
-
-	gsi_ep_info = ipa3_get_gsi_ep_info(ep->client);
-	if (!gsi_ep_info) {
-		IPAERR("Failed getting GSI EP info for client=%d\n",
-		       ep->client);
-		result = -EINVAL;
-		goto fail_get_gsi_ep_info;
-	} else
-		gsi_channel_props.ch_id = gsi_ep_info->ipa_gsi_chan_num;
-
-	gsi_channel_props.evt_ring_hdl = ep->gsi_evt_ring_hdl;
-	gsi_channel_props.re_size = GSI_CHAN_RE_SIZE_16B;
-	gsi_channel_props.use_db_eng = GSI_CHAN_DB_MODE;
-	gsi_channel_props.max_prefetch = GSI_ONE_PREFETCH_SEG;
-	gsi_channel_props.prefetch_mode =
-		gsi_ep_info->prefetch_mode;
-	gsi_channel_props.empty_lvl_threshold =
-		gsi_ep_info->prefetch_threshold;
-	gsi_channel_props.low_weight = 1;
-	gsi_channel_props.err_cb = ipa3_wdi3_gsi_chan_err_cb;
-
-	if (!is_smmu_enabled) {
-		gsi_channel_props.ring_len = (u16)info->transfer_ring_size;
-		gsi_channel_props.ring_base_addr =
-			(u64)info->transfer_ring_base_pa;
-	} else {
-		len = info_smmu->transfer_ring_size;
-		if (dir == IPA_WDI3_TX_DIR) {
-			if (ipa_create_gsi_smmu_mapping(IPA_WDI_TX_RING_RES,
-				true, info->transfer_ring_base_pa,
-				&info_smmu->transfer_ring_base, len,
-				false, &va)) {
-				IPAERR("failed to get smmu mapping\n");
-				result = -EFAULT;
-				goto fail_get_gsi_ep_info;
-			}
-		} else {
-			if (ipa_create_gsi_smmu_mapping(
-				IPA_WDI_RX_RING_RES, true,
-				info->transfer_ring_base_pa,
-				&info_smmu->transfer_ring_base, len,
-				false, &va)) {
-				IPAERR("failed to get smmu mapping\n");
-				result = -EFAULT;
-				goto fail_get_gsi_ep_info;
-			}
-		}
-		gsi_channel_props.ring_len = len;
-		gsi_channel_props.ring_base_addr = (u64)va;
-	}
-
-	result = gsi_alloc_channel(&gsi_channel_props, ipa3_ctx->gsi_dev_hdl,
-		&ep->gsi_chan_hdl);
-	if (result != GSI_STATUS_SUCCESS) {
-		goto fail_get_gsi_ep_info;
-	}
-
-	ep->gsi_mem_info.chan_ring_len = gsi_channel_props.ring_len;
-	ep->gsi_mem_info.chan_ring_base_addr =
-		gsi_channel_props.ring_base_addr;
-
-	/* write event scratch */
-	memset(&evt_scratch, 0, sizeof(evt_scratch));
-	evt_scratch.wdi3.update_rp_moderation_config =
-		UPDATE_RP_MODERATION_CONFIG;
-	result = gsi_write_evt_ring_scratch(ep->gsi_evt_ring_hdl, evt_scratch);
-	if (result != GSI_STATUS_SUCCESS) {
-		IPAERR("failed to write evt ring scratch\n");
-		goto fail_write_scratch;
-	}
-
-	if (!is_smmu_enabled) {
-		IPADBG("smmu disabled\n");
-		if (info->is_evt_rn_db_pcie_addr == true)
-			IPADBG_LOW("is_evt_rn_db_pcie_addr is PCIE addr\n");
-		else
-			IPADBG_LOW("is_evt_rn_db_pcie_addr is DDR addr\n");
-		IPADBG_LOW("LSB 0x%x\n",
-			(u32)info->event_ring_doorbell_pa);
-		IPADBG_LOW("MSB 0x%x\n",
-			(u32)((u64)info->event_ring_doorbell_pa >> 32));
-	} else {
-		IPADBG("smmu enabled\n");
-		if (info_smmu->is_evt_rn_db_pcie_addr == true)
-			IPADBG_LOW("is_evt_rn_db_pcie_addr is PCIE addr\n");
-		else
-			IPADBG_LOW("is_evt_rn_db_pcie_addr is DDR addr\n");
-		IPADBG_LOW("LSB 0x%x\n",
-			(u32)info_smmu->event_ring_doorbell_pa);
-		IPADBG_LOW("MSB 0x%x\n",
-			(u32)((u64)info_smmu->event_ring_doorbell_pa >> 32));
-	}
-
-	if (!is_smmu_enabled) {
-		addr_low = (u32)info->event_ring_doorbell_pa;
-		addr_high = (u32)((u64)info->event_ring_doorbell_pa >> 32);
-	} else {
-		if (dir == IPA_WDI3_TX_DIR) {
-			if (ipa_create_gsi_smmu_mapping(IPA_WDI_CE_DB_RES,
-				true, info_smmu->event_ring_doorbell_pa,
-				NULL, 4, true, &va)) {
-				IPAERR("failed to get smmu mapping\n");
-				result = -EFAULT;
-				goto fail_write_scratch;
-			}
-		} else {
-			if (ipa_create_gsi_smmu_mapping(
-				IPA_WDI_RX_COMP_RING_WP_RES,
-				true, info_smmu->event_ring_doorbell_pa,
-				NULL, 4, true, &va)) {
-				IPAERR("failed to get smmu mapping\n");
-				result = -EFAULT;
-				goto fail_write_scratch;
-			}
-		}
-		addr_low = (u32)va;
-		addr_high = (u32)((u64)va >> 32);
-	}
-
-	/*
-	 * Arch specific:
-	 * pcie addr which are not via smmu, use pa directly!
-	 * pcie and DDR via 2 different port
-	 * assert bit 40 to indicate it is pcie addr
-	 * WDI-3.0, MSM --> pcie via smmu
-	 * WDI-3.0, MDM --> pcie not via smmu + dual port
-	 * assert bit 40 in case
-	 */
-	if ((ipa3_ctx->platform_type == IPA_PLAT_TYPE_MDM) &&
-		is_smmu_enabled) {
-		/*
-		 * Ir-respective of smmu enabled don't use IOVA addr
-		 * since pcie not via smmu in MDM's
-		 */
-		if (info_smmu->is_evt_rn_db_pcie_addr == true) {
-			addr_low = (u32)info_smmu->event_ring_doorbell_pa;
-			addr_high =
-				(u32)((u64)info_smmu->event_ring_doorbell_pa
-				>> 32);
-		}
-	}
-
-	/*
-	 * GSI recomendation to set bit-40 for (mdm targets && pcie addr)
-	 * from wdi-3.0 interface document
-	 */
-	if (!is_smmu_enabled) {
-		if ((ipa3_ctx->platform_type == IPA_PLAT_TYPE_MDM) &&
-			info->is_evt_rn_db_pcie_addr)
-			addr_high |= (1 << 8);
-	} else {
-		if ((ipa3_ctx->platform_type == IPA_PLAT_TYPE_MDM) &&
-			info_smmu->is_evt_rn_db_pcie_addr)
-			addr_high |= (1 << 8);
-	}
-
-	gsi_wdi3_write_evt_ring_db(ep->gsi_evt_ring_hdl,
-			addr_low,
-			addr_high);
-
-	/* write channel scratch */
-	memset(&ch_scratch, 0, sizeof(ch_scratch));
-	ch_scratch.wdi3.update_rp_moderation_threshold =
-		UPDATE_RP_MODERATION_THRESHOLD;
-	if (dir == IPA_WDI3_RX_DIR) {
-		if (!is_smmu_enabled)
-			ch_scratch.wdi3.rx_pkt_offset = info->pkt_offset;
-		else
-			ch_scratch.wdi3.rx_pkt_offset = info_smmu->pkt_offset;
-		/* this metadata reg offset need to be in words */
-		ch_scratch.wdi3.endp_metadata_reg_offset =
-			ipahal_get_reg_mn_ofst(IPA_ENDP_INIT_HDR_METADATA_n, 0,
-				gsi_ep_info->ipa_ep_num) / 4;
-	}
-
-	if (!is_smmu_enabled) {
-		IPADBG_LOW("smmu disabled\n");
-		if (info->is_txr_rn_db_pcie_addr == true)
-			IPADBG_LOW("is_txr_rn_db_pcie_addr is PCIE addr\n");
-		else
-			IPADBG_LOW("is_txr_rn_db_pcie_addr is DDR addr\n");
-		IPADBG_LOW("LSB 0x%x\n",
-			(u32)info->transfer_ring_doorbell_pa);
-		IPADBG_LOW("MSB 0x%x\n",
-			(u32)((u64)info->transfer_ring_doorbell_pa >> 32));
-	} else {
-		IPADBG_LOW("smmu eabled\n");
-		if (info_smmu->is_txr_rn_db_pcie_addr == true)
-			IPADBG_LOW("is_txr_rn_db_pcie_addr is PCIE addr\n");
-		else
-			IPADBG_LOW("is_txr_rn_db_pcie_addr is DDR addr\n");
-		IPADBG_LOW("LSB 0x%x\n",
-			(u32)info_smmu->transfer_ring_doorbell_pa);
-		IPADBG_LOW("MSB 0x%x\n",
-			(u32)((u64)info_smmu->transfer_ring_doorbell_pa >> 32));
-	}
-
-	if (!is_smmu_enabled) {
-		ch_scratch.wdi3.wifi_rp_address_low =
-			(u32)info->transfer_ring_doorbell_pa;
-		ch_scratch.wdi3.wifi_rp_address_high =
-			(u32)((u64)info->transfer_ring_doorbell_pa >> 32);
-	} else {
-		if (dir == IPA_WDI3_TX_DIR) {
-			if (ipa_create_gsi_smmu_mapping(IPA_WDI_TX_DB_RES,
-				true, info_smmu->transfer_ring_doorbell_pa,
-				NULL, 4, true, &va)) {
-				IPAERR("failed to get smmu mapping\n");
-				result = -EFAULT;
-				goto fail_write_scratch;
-			}
-			ch_scratch.wdi3.wifi_rp_address_low = (u32)va;
-			ch_scratch.wdi3.wifi_rp_address_high =
-				(u32)((u64)va >> 32);
-		} else {
-			if (ipa_create_gsi_smmu_mapping(IPA_WDI_RX_RING_RP_RES,
-				true, info_smmu->transfer_ring_doorbell_pa,
-				NULL, 4, true, &va)) {
-				IPAERR("failed to get smmu mapping\n");
-				result = -EFAULT;
-				goto fail_write_scratch;
-			}
-			ch_scratch.wdi3.wifi_rp_address_low = (u32)va;
-			ch_scratch.wdi3.wifi_rp_address_high =
-				(u32)((u64)va >> 32);
-		}
-	}
-
-	/*
-	 * Arch specific:
-	 * pcie addr which are not via smmu, use pa directly!
-	 * pcie and DDR via 2 different port
-	 * assert bit 40 to indicate it is pcie addr
-	 * WDI-3.0, MSM --> pcie via smmu
-	 * WDI-3.0, MDM --> pcie not via smmu + dual port
-	 * assert bit 40 in case
-	 */
-	if ((ipa3_ctx->platform_type == IPA_PLAT_TYPE_MDM) &&
-		is_smmu_enabled) {
-		/*
-		 * Ir-respective of smmu enabled don't use IOVA addr
-		 * since pcie not via smmu in MDM's
-		 */
-		if (info_smmu->is_txr_rn_db_pcie_addr == true) {
-			ch_scratch.wdi3.wifi_rp_address_low =
-				(u32)info_smmu->transfer_ring_doorbell_pa;
-			ch_scratch.wdi3.wifi_rp_address_high =
-				(u32)((u64)info_smmu->transfer_ring_doorbell_pa
-				>> 32);
-		}
-	}
-
-	/*
-	 * GSI recomendation to set bit-40 for (mdm targets && pcie addr)
-	 * from wdi-3.0 interface document
-	 */
-	if (!is_smmu_enabled) {
-		if ((ipa3_ctx->platform_type == IPA_PLAT_TYPE_MDM) &&
-			info->is_txr_rn_db_pcie_addr)
-			ch_scratch.wdi3.wifi_rp_address_high =
-			(u32)((u32)ch_scratch.wdi3.wifi_rp_address_high |
-			(1 << 8));
-	} else {
-		if ((ipa3_ctx->platform_type == IPA_PLAT_TYPE_MDM) &&
-			info_smmu->is_txr_rn_db_pcie_addr)
-			ch_scratch.wdi3.wifi_rp_address_high =
-			(u32)((u32)ch_scratch.wdi3.wifi_rp_address_high |
-			(1 << 8));
-	}
-
-	result = gsi_write_channel_scratch(ep->gsi_chan_hdl, ch_scratch);
-	if (result != GSI_STATUS_SUCCESS) {
-		IPAERR("failed to write evt ring scratch\n");
-		goto fail_write_scratch;
-	}
-	return 0;
-
-fail_write_scratch:
-	gsi_dealloc_channel(ep->gsi_chan_hdl);
-	ep->gsi_chan_hdl = ~0;
-fail_get_gsi_ep_info:
-	gsi_dealloc_evt_ring(ep->gsi_evt_ring_hdl);
-	ep->gsi_evt_ring_hdl = ~0;
-fail_smmu_mapping:
-	ipa3_release_wdi3_gsi_smmu_mappings(dir);
-	return result;
-}
-
-int ipa3_conn_wdi3_pipes(struct ipa_wdi_conn_in_params *in,
-	struct ipa_wdi_conn_out_params *out,
-	ipa_wdi_meter_notifier_cb wdi_notify)
-{
-	enum ipa_client_type rx_client;
-	enum ipa_client_type tx_client;
-	struct ipa3_ep_context *ep_rx;
-	struct ipa3_ep_context *ep_tx;
-	int ipa_ep_idx_rx;
-	int ipa_ep_idx_tx;
-	int result = 0;
-	u32 gsi_db_addr_low, gsi_db_addr_high;
-	void __iomem *db_addr;
-	u32 evt_ring_db_addr_low, evt_ring_db_addr_high;
-
-	/* wdi3 only support over gsi */
-	if (!ipa3_ctx->ipa_wdi3_over_gsi) {
-		IPAERR("wdi3 over uc offload not supported");
-		WARN_ON(1);
-		return -EFAULT;
-	}
-
-	if (in == NULL || out == NULL) {
-		IPAERR("invalid input\n");
-		return -EINVAL;
-	}
-
-	if (in->is_smmu_enabled == false) {
-		rx_client = in->u_rx.rx.client;
-		tx_client = in->u_tx.tx.client;
-	} else {
-		rx_client = in->u_rx.rx_smmu.client;
-		tx_client = in->u_tx.tx_smmu.client;
-	}
-
-	ipa_ep_idx_rx = ipa_get_ep_mapping(rx_client);
-	ipa_ep_idx_tx = ipa_get_ep_mapping(tx_client);
-
-	if (ipa_ep_idx_rx == -1 || ipa_ep_idx_tx == -1) {
-		IPAERR("fail to alloc EP.\n");
-		return -EFAULT;
-	}
-	if (ipa_ep_idx_rx >= IPA3_MAX_NUM_PIPES ||
-		ipa_ep_idx_tx >= IPA3_MAX_NUM_PIPES) {
-		IPAERR("ep out of range.\n");
-		return -EFAULT;
-	}
-
-	ep_rx = &ipa3_ctx->ep[ipa_ep_idx_rx];
-	ep_tx = &ipa3_ctx->ep[ipa_ep_idx_tx];
-
-	if (ep_rx->valid || ep_tx->valid) {
-		IPAERR("EP already allocated.\n");
-		return -EFAULT;
-	}
-
-	memset(ep_rx, 0, offsetof(struct ipa3_ep_context, sys));
-	memset(ep_tx, 0, offsetof(struct ipa3_ep_context, sys));
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-#ifdef IPA_WAN_MSG_IPv6_ADDR_GW_LEN
-	if (wdi_notify)
-		ipa3_ctx->uc_wdi_ctx.stats_notify = wdi_notify;
-	else
-		IPADBG("wdi_notify is null\n");
-#endif
-
-	/* setup rx ep cfg */
-	ep_rx->valid = 1;
-	ep_rx->client = rx_client;
-	result = ipa3_disable_data_path(ipa_ep_idx_rx);
-	if (result) {
-		IPAERR("disable data path failed res=%d clnt=%d.\n", result,
-			ipa_ep_idx_rx);
-		IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-		return -EFAULT;
-	}
-	ep_rx->client_notify = in->notify;
-	ep_rx->priv = in->priv;
-
-	if (in->is_smmu_enabled == false)
-		memcpy(&ep_rx->cfg, &in->u_rx.rx.ipa_ep_cfg,
-			sizeof(ep_rx->cfg));
-	else
-		memcpy(&ep_rx->cfg, &in->u_rx.rx_smmu.ipa_ep_cfg,
-			sizeof(ep_rx->cfg));
-
-	if (ipa3_cfg_ep(ipa_ep_idx_rx, &ep_rx->cfg)) {
-		IPAERR("fail to setup rx pipe cfg\n");
-		result = -EFAULT;
-		goto fail;
-	}
-
-	IPADBG("ipa3_ctx->ipa_wdi3_over_gsi %d\n",
-		   ipa3_ctx->ipa_wdi3_over_gsi);
-	/* setup RX gsi channel */
-	if (ipa3_setup_wdi3_gsi_channel(in->is_smmu_enabled,
-		&in->u_rx.rx, &in->u_rx.rx_smmu, IPA_WDI3_RX_DIR,
-		ep_rx)) {
-		IPAERR("fail to setup wdi3 gsi rx channel\n");
-		result = -EFAULT;
-		goto fail;
-	}
-	if (gsi_query_channel_db_addr(ep_rx->gsi_chan_hdl,
-		&gsi_db_addr_low, &gsi_db_addr_high)) {
-		IPAERR("failed to query gsi rx db addr\n");
-		result = -EFAULT;
-		goto fail;
-	}
-	/* only 32 bit lsb is used */
-	out->rx_uc_db_pa = (phys_addr_t)(gsi_db_addr_low);
-	IPADBG("out->rx_uc_db_pa %llu\n", out->rx_uc_db_pa);
-
-	ipa3_install_dflt_flt_rules(ipa_ep_idx_rx);
-	IPADBG("client %d (ep: %d) connected\n", rx_client,
-		ipa_ep_idx_rx);
-
-	/* setup tx ep cfg */
-	ep_tx->valid = 1;
-	ep_tx->client = tx_client;
-	result = ipa3_disable_data_path(ipa_ep_idx_tx);
-	if (result) {
-		IPAERR("disable data path failed res=%d ep=%d.\n", result,
-			ipa_ep_idx_tx);
-		result = -EFAULT;
-		goto fail;
-	}
-
-	if (in->is_smmu_enabled == false)
-		memcpy(&ep_tx->cfg, &in->u_tx.tx.ipa_ep_cfg,
-			sizeof(ep_tx->cfg));
-	else
-		memcpy(&ep_tx->cfg, &in->u_tx.tx_smmu.ipa_ep_cfg,
-			sizeof(ep_tx->cfg));
-
-	ep_tx->cfg.aggr.aggr_en = IPA_ENABLE_AGGR;
-	ep_tx->cfg.aggr.aggr = IPA_GENERIC;
-	ep_tx->cfg.aggr.aggr_byte_limit = IPA_WLAN_AGGR_BYTE_LIMIT;
-	ep_tx->cfg.aggr.aggr_pkt_limit = IPA_WLAN_AGGR_PKT_LIMIT;
-	ep_tx->cfg.aggr.aggr_hard_byte_limit_en = IPA_ENABLE_AGGR;
-	if (ipa3_cfg_ep(ipa_ep_idx_tx, &ep_tx->cfg)) {
-		IPAERR("fail to setup tx pipe cfg\n");
-		result = -EFAULT;
-		goto fail;
-	}
-
-	/* setup TX gsi channel */
-	if (ipa3_setup_wdi3_gsi_channel(in->is_smmu_enabled,
-		&in->u_tx.tx, &in->u_tx.tx_smmu, IPA_WDI3_TX_DIR,
-		ep_tx)) {
-		IPAERR("fail to setup wdi3 gsi tx channel\n");
-		result = -EFAULT;
-		goto fail;
-	}
-	if (gsi_query_channel_db_addr(ep_tx->gsi_chan_hdl,
-		&gsi_db_addr_low, &gsi_db_addr_high)) {
-		IPAERR("failed to query gsi tx db addr\n");
-		result = -EFAULT;
-		goto fail;
-	}
-	/* only 32 bit lsb is used */
-	out->tx_uc_db_pa = (phys_addr_t)(gsi_db_addr_low);
-	IPADBG("out->tx_uc_db_pa %llu\n", out->tx_uc_db_pa);
-	IPADBG("client %d (ep: %d) connected\n", tx_client,
-		ipa_ep_idx_tx);
-
-	/* ring initial event ring dbs */
-	gsi_query_evt_ring_db_addr(ep_rx->gsi_evt_ring_hdl,
-		&evt_ring_db_addr_low, &evt_ring_db_addr_high);
-	IPADBG("evt_ring_hdl %lu, db_addr_low %u db_addr_high %u\n",
-		ep_rx->gsi_evt_ring_hdl, evt_ring_db_addr_low,
-		evt_ring_db_addr_high);
-
-	/* only 32 bit lsb is used */
-	db_addr = ioremap((phys_addr_t)(evt_ring_db_addr_low), 4);
-	/*
-	 * IPA/GSI driver should ring the event DB once after
-	 * initialization of the event, with a value that is
-	 * outside of the ring range. Eg: ring base = 0x1000,
-	 * ring size = 0x100 => AP can write value > 0x1100
-	 * into the doorbell address. Eg: 0x 1110
-	 */
-	iowrite32(in->u_rx.rx.event_ring_size / 4 + 10, db_addr);
-	gsi_query_evt_ring_db_addr(ep_tx->gsi_evt_ring_hdl,
-		&evt_ring_db_addr_low, &evt_ring_db_addr_high);
-
-	/* only 32 bit lsb is used */
-	db_addr = ioremap((phys_addr_t)(evt_ring_db_addr_low), 4);
-	/*
-	 * IPA/GSI driver should ring the event DB once after
-	 * initialization of the event, with a value that is
-	 * outside of the ring range. Eg: ring base = 0x1000,
-	 * ring size = 0x100 => AP can write value > 0x1100
-	 * into the doorbell address. Eg: 0x 1110
-	 */
-	iowrite32(in->u_tx.tx.event_ring_size / 4 + 10, db_addr);
-
-fail:
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-	return result;
-}
-
-int ipa3_disconn_wdi3_pipes(int ipa_ep_idx_tx, int ipa_ep_idx_rx)
-{
-	struct ipa3_ep_context *ep_tx, *ep_rx;
-	int result = 0;
-
-	/* wdi3 only support over gsi */
-	if (!ipa3_ctx->ipa_wdi3_over_gsi) {
-		IPAERR("wdi3 over uc offload not supported");
-		WARN_ON(1);
-		return -EFAULT;
-	}
-
-	IPADBG("ep_tx = %d\n", ipa_ep_idx_tx);
-	IPADBG("ep_rx = %d\n", ipa_ep_idx_rx);
-
-	if (ipa_ep_idx_tx < 0 || ipa_ep_idx_tx >= IPA3_MAX_NUM_PIPES ||
-		ipa_ep_idx_rx < 0 || ipa_ep_idx_rx >= IPA3_MAX_NUM_PIPES) {
-		IPAERR("invalid ipa ep index\n");
-		return -EINVAL;
-	}
-
-	ep_tx = &ipa3_ctx->ep[ipa_ep_idx_tx];
-	ep_rx = &ipa3_ctx->ep[ipa_ep_idx_rx];
-
-	IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(ipa_ep_idx_tx));
-	/* tear down tx pipe */
-	result = ipa3_reset_gsi_channel(ipa_ep_idx_tx);
-	if (result != GSI_STATUS_SUCCESS) {
-		IPAERR("failed to reset gsi channel: %d.\n", result);
-		goto exit;
-	}
-	result = gsi_reset_evt_ring(ep_tx->gsi_evt_ring_hdl);
-	if (result != GSI_STATUS_SUCCESS) {
-		IPAERR("failed to reset evt ring: %d.\n", result);
-		goto exit;
-	}
-	result = ipa3_release_gsi_channel(ipa_ep_idx_tx);
-	if (result) {
-		IPAERR("failed to release gsi channel: %d\n", result);
-		goto exit;
-	}
-
-	memset(ep_tx, 0, sizeof(struct ipa3_ep_context));
-	IPADBG("tx client (ep: %d) disconnected\n", ipa_ep_idx_tx);
-
-	/* tear down rx pipe */
-	result = ipa3_reset_gsi_channel(ipa_ep_idx_rx);
-	if (result != GSI_STATUS_SUCCESS) {
-		IPAERR("failed to reset gsi channel: %d.\n", result);
-		goto exit;
-	}
-	result = gsi_reset_evt_ring(ep_rx->gsi_evt_ring_hdl);
-	if (result != GSI_STATUS_SUCCESS) {
-		IPAERR("failed to reset evt ring: %d.\n", result);
-		goto exit;
-	}
-	result = ipa3_release_gsi_channel(ipa_ep_idx_rx);
-	if (result) {
-		IPAERR("failed to release gsi channel: %d\n", result);
-		goto exit;
-	}
-
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_5)
-		ipa3_uc_debug_stats_dealloc(IPA_HW_PROTOCOL_WDI3);
-	ipa3_delete_dflt_flt_rules(ipa_ep_idx_rx);
-	memset(ep_rx, 0, sizeof(struct ipa3_ep_context));
-	IPADBG("rx client (ep: %d) disconnected\n", ipa_ep_idx_rx);
-
-exit:
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_by_pipe(ipa_ep_idx_tx));
-	return result;
-}
-
-int ipa3_enable_wdi3_pipes(int ipa_ep_idx_tx, int ipa_ep_idx_rx)
-{
-	struct ipa3_ep_context *ep_tx, *ep_rx;
-	int result = 0;
-
-	/* wdi3 only support over gsi */
-	if (!ipa3_ctx->ipa_wdi3_over_gsi) {
-		IPAERR("wdi3 over uc offload not supported");
-		WARN_ON(1);
-		return -EFAULT;
-	}
-
-	IPADBG("ep_tx = %d\n", ipa_ep_idx_tx);
-	IPADBG("ep_rx = %d\n", ipa_ep_idx_rx);
-
-	ep_tx = &ipa3_ctx->ep[ipa_ep_idx_tx];
-	ep_rx = &ipa3_ctx->ep[ipa_ep_idx_rx];
-
-	IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(ipa_ep_idx_tx));
-
-	/* start gsi tx channel */
-	result = gsi_start_channel(ep_tx->gsi_chan_hdl);
-	if (result) {
-		IPAERR("failed to start gsi tx channel\n");
-		result = -EFAULT;
-		goto exit;
-	}
-
-	/* start gsi rx channel */
-	result = gsi_start_channel(ep_rx->gsi_chan_hdl);
-	if (result) {
-		IPAERR("failed to start gsi rx channel\n");
-		result = -EFAULT;
-		goto exit;
-	}
-	/* start uC gsi dbg stats monitor */
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_5) {
-		ipa3_ctx->gsi_info[IPA_HW_PROTOCOL_WDI3].ch_id_info[0].ch_id
-			= ep_rx->gsi_chan_hdl;
-		ipa3_ctx->gsi_info[IPA_HW_PROTOCOL_WDI3].ch_id_info[0].dir
-			= DIR_PRODUCER;
-		ipa3_ctx->gsi_info[IPA_HW_PROTOCOL_WDI3].ch_id_info[1].ch_id
-			= ep_tx->gsi_chan_hdl;
-		ipa3_ctx->gsi_info[IPA_HW_PROTOCOL_WDI3].ch_id_info[1].dir
-			= DIR_CONSUMER;
-		ipa3_uc_debug_stats_alloc(
-			ipa3_ctx->gsi_info[IPA_HW_PROTOCOL_WDI3]);
-	}
-	/* enable data path */
-	result = ipa3_enable_data_path(ipa_ep_idx_rx);
-	if (result) {
-		IPAERR("enable data path failed res=%d clnt=%d.\n", result,
-			ipa_ep_idx_rx);
-		result = -EFAULT;
-		goto exit;
-	}
-
-	result = ipa3_enable_data_path(ipa_ep_idx_tx);
-	if (result) {
-		IPAERR("enable data path failed res=%d clnt=%d.\n", result,
-			ipa_ep_idx_tx);
-		result = -EFAULT;
-		goto exit;
-	}
-
-exit:
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(ipa_ep_idx_tx));
-	return result;
-}
-
-int ipa3_disable_wdi3_pipes(int ipa_ep_idx_tx, int ipa_ep_idx_rx)
-{
-	int result = 0;
-	struct ipa3_ep_context *ep;
-	u32 source_pipe_bitmask = 0;
-	bool disable_force_clear = false;
-	struct ipahal_ep_cfg_ctrl_scnd ep_ctrl_scnd = { 0 };
-
-	/* wdi3 only support over gsi */
-	if (!ipa3_ctx->ipa_wdi3_over_gsi) {
-		IPAERR("wdi3 over uc offload not supported");
-		WARN_ON(1);
-		return -EFAULT;
-	}
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	/* disable tx data path */
-	result = ipa3_disable_data_path(ipa_ep_idx_tx);
-	if (result) {
-		IPAERR("enable data path failed res=%d clnt=%d.\n", result,
-			ipa_ep_idx_tx);
-		result = -EFAULT;
-		goto fail;
-	}
-
-	/* disable rx data path */
-	result = ipa3_disable_data_path(ipa_ep_idx_rx);
-	if (result) {
-		IPAERR("disable data path failed res=%d clnt=%d.\n", result,
-			ipa_ep_idx_rx);
-		result = -EFAULT;
-		goto fail;
-	}
-	/*
-	 * For WDI 3.0 need to ensure pipe will be empty before suspend
-	 * as IPA uC will fail to suspend the pipe otherwise.
-	 */
-	ep = &ipa3_ctx->ep[ipa_ep_idx_rx];
-	source_pipe_bitmask = 1 <<
-			ipa3_get_ep_mapping(ep->client);
-	result = ipa3_enable_force_clear(ipa_ep_idx_rx,
-			false, source_pipe_bitmask);
-	if (result) {
-		/*
-		 * assuming here modem SSR, AP can remove
-		 * the delay in this case
-		 */
-		IPAERR("failed to force clear %d\n", result);
-		IPAERR("remove delay from SCND reg\n");
-		ep_ctrl_scnd.endp_delay = false;
-		ipahal_write_reg_n_fields(
-			IPA_ENDP_INIT_CTRL_SCND_n, ipa_ep_idx_rx,
-			&ep_ctrl_scnd);
-	} else {
-		disable_force_clear = true;
-	}
-
-	/* stop gsi rx channel */
-	result = ipa3_stop_gsi_channel(ipa_ep_idx_rx);
-	if (result) {
-		IPAERR("failed to stop gsi rx channel\n");
-		result = -EFAULT;
-		goto fail;
-	}
-	/* stop gsi tx channel */
-	result = ipa3_stop_gsi_channel(ipa_ep_idx_tx);
-	if (result) {
-		IPAERR("failed to stop gsi tx channel\n");
-		result = -EFAULT;
-		goto fail;
-	}
-	/* stop uC gsi dbg stats monitor */
-	if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_5) {
-		ipa3_ctx->gsi_info[IPA_HW_PROTOCOL_WDI3].ch_id_info[0].ch_id
-			= 0xff;
-		ipa3_ctx->gsi_info[IPA_HW_PROTOCOL_WDI3].ch_id_info[0].dir
-			= DIR_PRODUCER;
-		ipa3_ctx->gsi_info[IPA_HW_PROTOCOL_WDI3].ch_id_info[1].ch_id
-			= 0xff;
-		ipa3_ctx->gsi_info[IPA_HW_PROTOCOL_WDI3].ch_id_info[1].dir
-			= DIR_CONSUMER;
-		ipa3_uc_debug_stats_alloc(
-			ipa3_ctx->gsi_info[IPA_HW_PROTOCOL_WDI3]);
-	}
-	if (disable_force_clear)
-		ipa3_disable_force_clear(ipa_ep_idx_rx);
-
-fail:
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-	return result;
-
-}
-
-int ipa3_write_qmapid_wdi3_gsi_pipe(u32 clnt_hdl, u8 qmap_id)
-{
-	int result = 0;
-	struct ipa3_ep_context *ep;
-	union __packed gsi_channel_scratch ch_scratch;
-
-	memset(&ch_scratch, 0, sizeof(ch_scratch));
-	if (clnt_hdl >= ipa3_ctx->ipa_num_pipes ||
-		ipa3_ctx->ep[clnt_hdl].valid == 0) {
-		IPAERR_RL("bad parm, %d\n", clnt_hdl);
-		return -EINVAL;
-	}
-	ep = &ipa3_ctx->ep[clnt_hdl];
-	IPA_ACTIVE_CLIENTS_INC_EP(ipa3_get_client_mapping(clnt_hdl));
-	result = gsi_read_channel_scratch(ep->gsi_chan_hdl, &ch_scratch);
-
-	if (result != GSI_STATUS_SUCCESS) {
-		IPAERR("failed to read channel scratch %d\n", result);
-		goto exit;
-	}
-	result = gsi_stop_channel(ep->gsi_chan_hdl);
-	if (result != GSI_STATUS_SUCCESS && result != -GSI_STATUS_AGAIN &&
-		result != -GSI_STATUS_TIMED_OUT) {
-		IPAERR("failed to stop gsi channel %d\n", result);
-		goto exit;
-	}
-
-	ch_scratch.wdi3.qmap_id = qmap_id;
-	result = gsi_write_channel_scratch(ep->gsi_chan_hdl,
-			ch_scratch);
-	if (result != GSI_STATUS_SUCCESS) {
-		IPAERR("failed to write channel scratch %d\n", result);
-		goto exit;
-	}
-
-	result =  gsi_start_channel(ep->gsi_chan_hdl);
-	if (result != GSI_STATUS_SUCCESS) {
-		IPAERR("failed to start gsi channel %d\n", result);
-		goto exit;
-	}
-
-exit:
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(clnt_hdl));
-	return result;
-}
-
-/**
- * ipa3_get_wdi3_gsi_stats() - Query WDI3 gsi stats from uc
- * @stats:	[inout] stats blob from client populated by driver
- *
- * Returns:	0 on success, negative on failure
- *
- * @note Cannot be called from atomic context
- *
- */
-int ipa3_get_wdi3_gsi_stats(struct ipa3_uc_dbg_ring_stats *stats)
-{
-	int i;
-
-	if (!ipa3_ctx->wdi3_ctx.dbg_stats.uc_dbg_stats_mmio) {
-		IPAERR("bad NULL parms for wdi3_gsi_stats\n");
-		return -EINVAL;
-	}
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-	for (i = 0; i < MAX_WDI3_CHANNELS; i++) {
-		stats->ring[i].ringFull = ioread32(
-			ipa3_ctx->wdi3_ctx.dbg_stats.uc_dbg_stats_mmio
-			+ i * IPA3_UC_DEBUG_STATS_OFF +
-			IPA3_UC_DEBUG_STATS_RINGFULL_OFF);
-		stats->ring[i].ringEmpty = ioread32(
-			ipa3_ctx->wdi3_ctx.dbg_stats.uc_dbg_stats_mmio
-			+ i * IPA3_UC_DEBUG_STATS_OFF +
-			IPA3_UC_DEBUG_STATS_RINGEMPTY_OFF);
-		stats->ring[i].ringUsageHigh = ioread32(
-			ipa3_ctx->wdi3_ctx.dbg_stats.uc_dbg_stats_mmio
-			+ i * IPA3_UC_DEBUG_STATS_OFF +
-			IPA3_UC_DEBUG_STATS_RINGUSAGEHIGH_OFF);
-		stats->ring[i].ringUsageLow = ioread32(
-			ipa3_ctx->wdi3_ctx.dbg_stats.uc_dbg_stats_mmio
-			+ i * IPA3_UC_DEBUG_STATS_OFF +
-			IPA3_UC_DEBUG_STATS_RINGUSAGELOW_OFF);
-		stats->ring[i].RingUtilCount = ioread32(
-			ipa3_ctx->wdi3_ctx.dbg_stats.uc_dbg_stats_mmio
-			+ i * IPA3_UC_DEBUG_STATS_OFF +
-			IPA3_UC_DEBUG_STATS_RINGUTILCOUNT_OFF);
-	}
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-
-	return 0;
-}

ipa/ipa_v3/ipa_wigig_i.c

@@ -1,1914 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
- */
-
-#include "ipa_i.h"
-#include <linux/if_ether.h>
-#include <linux/log2.h>
-#include <linux/debugfs.h>
-#include <linux/ipa_wigig.h>
-
-#define IPA_WIGIG_DESC_RING_EL_SIZE	32
-#define IPA_WIGIG_STATUS_RING_EL_SIZE	16
-
-#define GSI_STOP_MAX_RETRY_CNT 10
-
-#define IPA_WIGIG_CONNECTED BIT(0)
-#define IPA_WIGIG_ENABLED BIT(1)
-#define IPA_WIGIG_MSB_MASK 0xFFFFFFFF00000000
-#define IPA_WIGIG_LSB_MASK 0x00000000FFFFFFFF
-#define IPA_WIGIG_MSB(num) ((u32)((num & IPA_WIGIG_MSB_MASK) >> 32))
-#define IPA_WIGIG_LSB(num) ((u32)(num & IPA_WIGIG_LSB_MASK))
-/* extract PCIE addresses [0:39] relevant msb */
-#define IPA_WIGIG_8_MSB_MASK 0xFF00000000
-#define IPA_WIGIG_8_MSB(num) ((u32)((num & IPA_WIGIG_8_MSB_MASK) >> 32))
-#define W11AD_RX 0
-#define W11AD_TX 1
-#define W11AD_TO_GSI_DB_m 1
-#define W11AD_TO_GSI_DB_n 1
-
-static LIST_HEAD(smmu_reg_addr_list);
-static LIST_HEAD(smmu_ring_addr_list);
-static DEFINE_MUTEX(smmu_lock);
-struct dentry *wigig_dent;
-
-struct ipa_wigig_smmu_reg_addr {
-	struct list_head link;
-	phys_addr_t phys_addr;
-	enum ipa_smmu_cb_type cb_type;
-	u8 count;
-};
-
-struct ipa_wigig_smmu_ring_addr {
-	struct list_head link;
-	u64 iova;
-	enum ipa_smmu_cb_type cb_type;
-	u8 count;
-};
-
-
-static int ipa3_wigig_uc_loaded_handler(struct notifier_block *self,
-	unsigned long val, void *data)
-{
-	IPADBG("val %ld\n", val);
-
-	if (!ipa3_ctx) {
-		IPAERR("IPA ctx is null\n");
-		return -EINVAL;
-	}
-
-	WARN_ON(data != ipa3_ctx);
-
-	if (ipa3_ctx->uc_wigig_ctx.uc_ready_cb) {
-		ipa3_ctx->uc_wigig_ctx.uc_ready_cb(
-			ipa3_ctx->uc_wigig_ctx.priv);
-
-		ipa3_ctx->uc_wigig_ctx.uc_ready_cb =
-			NULL;
-		ipa3_ctx->uc_wigig_ctx.priv = NULL;
-	}
-
-	IPADBG("exit\n");
-	return 0;
-}
-
-static struct notifier_block uc_loaded_notifier = {
-	.notifier_call = ipa3_wigig_uc_loaded_handler,
-};
-
-int ipa3_wigig_init_i(void)
-{
-	IPADBG("\n");
-
-	ipa3_uc_register_ready_cb(&uc_loaded_notifier);
-
-	IPADBG("exit\n");
-
-	return 0;
-}
-
-int ipa3_wigig_internal_init(
-	struct ipa_wdi_uc_ready_params *inout,
-	ipa_wigig_misc_int_cb int_notify,
-	phys_addr_t *uc_db_pa)
-{
-	int result = 0;
-
-	IPADBG("\n");
-
-	if (inout == NULL) {
-		IPAERR("inout is NULL");
-		return -EINVAL;
-	}
-
-	if (int_notify == NULL) {
-		IPAERR("int_notify is NULL");
-		return -EINVAL;
-	}
-
-	result = ipa3_uc_state_check();
-	if (result) {
-		inout->is_uC_ready = false;
-		ipa3_ctx->uc_wigig_ctx.uc_ready_cb = inout->notify;
-	} else {
-		inout->is_uC_ready = true;
-	}
-	ipa3_ctx->uc_wigig_ctx.priv = inout->priv;
-	ipa3_ctx->uc_wigig_ctx.misc_notify_cb = int_notify;
-
-	*uc_db_pa = ipa3_ctx->ipa_wrapper_base +
-		ipahal_get_reg_base() +
-		ipahal_get_reg_mn_ofst(
-			IPA_UC_MAILBOX_m_n,
-			W11AD_TO_GSI_DB_m,
-			W11AD_TO_GSI_DB_n);
-
-	IPADBG("exit\n");
-
-	return 0;
-}
-
-static int ipa3_wigig_tx_bit_to_ep(
-	const u8 tx_bit_num,
-	enum ipa_client_type *type)
-{
-	IPADBG("tx_bit_num %d\n", tx_bit_num);
-
-	switch (tx_bit_num) {
-	case 2:
-		*type = IPA_CLIENT_WIGIG1_CONS;
-		break;
-	case 3:
-		*type = IPA_CLIENT_WIGIG2_CONS;
-		break;
-	case 4:
-		*type = IPA_CLIENT_WIGIG3_CONS;
-		break;
-	case 5:
-		*type = IPA_CLIENT_WIGIG4_CONS;
-		break;
-	default:
-		IPAERR("invalid tx_bit_num %d\n", tx_bit_num);
-		return -EINVAL;
-	}
-
-	IPADBG("exit\n");
-	return 0;
-}
-
-static int ipa3_wigig_smmu_map_buffers(bool Rx,
-	struct ipa_wigig_pipe_setup_info_smmu *pipe_smmu,
-	void *buff,
-	bool map)
-{
-	int result;
-
-	/* data buffers */
-	if (Rx) {
-		struct ipa_wigig_rx_pipe_data_buffer_info_smmu *dbuff_smmu =
-			(struct ipa_wigig_rx_pipe_data_buffer_info_smmu *)buff;
-
-		int num_elem =
-			pipe_smmu->desc_ring_size /
-			IPA_WIGIG_DESC_RING_EL_SIZE;
-
-		result = ipa3_smmu_map_peer_buff(
-			dbuff_smmu->data_buffer_base_iova,
-			dbuff_smmu->data_buffer_size * num_elem,
-			map,
-			&dbuff_smmu->data_buffer_base,
-			IPA_SMMU_CB_11AD);
-		if (result) {
-			IPAERR(
-				"failed to %s rx data_buffer %d, num elem %d\n"
-				, map ? "map" : "unmap",
-				result, num_elem);
-			goto fail_map_buff;
-		}
-
-	} else {
-		int i;
-		struct ipa_wigig_tx_pipe_data_buffer_info_smmu *dbuff_smmu =
-			(struct ipa_wigig_tx_pipe_data_buffer_info_smmu *)buff;
-
-		for (i = 0; i < dbuff_smmu->num_buffers; i++) {
-			result = ipa3_smmu_map_peer_buff(
-				*(dbuff_smmu->data_buffer_base_iova + i),
-				dbuff_smmu->data_buffer_size,
-				map,
-				(dbuff_smmu->data_buffer_base + i),
-				IPA_SMMU_CB_11AD);
-			if (result) {
-				IPAERR(
-					"%d: failed to %s tx data buffer %d\n"
-					, i, map ? "map" : "unmap",
-					result);
-				for (i--; i >= 0; i--) {
-					result = ipa3_smmu_map_peer_buff(
-					*(dbuff_smmu->data_buffer_base_iova +
-						i),
-					dbuff_smmu->data_buffer_size,
-					!map,
-					(dbuff_smmu->data_buffer_base +
-						i),
-					IPA_SMMU_CB_11AD);
-				}
-				goto fail_map_buff;
-			}
-		}
-	}
-
-	IPADBG("exit\n");
-	return 0;
-
-fail_map_buff:
-	return result;
-}
-
-static int ipa3_wigig_smmu_map_reg(phys_addr_t phys_addr, bool map,
-	enum ipa_smmu_cb_type cb_type)
-{
-	struct ipa_wigig_smmu_reg_addr *entry;
-	struct ipa_wigig_smmu_reg_addr *next;
-	int result = 0;
-
-	IPADBG("addr %pa, %s\n", &phys_addr, map ? "map" : "unmap");
-	mutex_lock(&smmu_lock);
-	list_for_each_entry_safe(entry, next, &smmu_reg_addr_list, link) {
-		if ((entry->phys_addr == phys_addr) &&
-			(entry->cb_type == cb_type)) {
-			IPADBG("cb %d, page %pa already mapped, ", cb_type,
-				&phys_addr);
-			if (map) {
-				entry->count++;
-				IPADBG("inc to %d\n", (entry->count));
-			} else {
-				--entry->count;
-				IPADBG("dec to %d\n", entry->count);
-				if (!(entry->count)) {
-					IPADBG("unmap and delete\n");
-					result = ipa3_smmu_map_peer_reg(
-						phys_addr, map, cb_type);
-					if (result) {
-						IPAERR("failed to unmap %pa\n",
-							&phys_addr);
-						goto finish;
-					}
-					list_del(&entry->link);
-					kfree(entry);
-				}
-			}
-			goto finish;
-		}
-	}
-	IPADBG("new page found %pa, map and add to list CB %d\n", &phys_addr,
-		cb_type);
-	result = ipa3_smmu_map_peer_reg(phys_addr, map, cb_type);
-	if (result) {
-		IPAERR("failed to map %pa\n", &phys_addr);
-		goto finish;
-	}
-
-	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
-	if (entry == NULL) {
-		IPAERR("couldn't allocate for %pa\n", &phys_addr);
-		ipa3_smmu_map_peer_reg(phys_addr, !map, cb_type);
-		result = -ENOMEM;
-		goto finish;
-	}
-	INIT_LIST_HEAD(&entry->link);
-	entry->phys_addr = phys_addr;
-	entry->cb_type = cb_type;
-	entry->count = 1;
-	list_add(&entry->link, &smmu_reg_addr_list);
-
-finish:
-	mutex_unlock(&smmu_lock);
-	IPADBG("exit\n");
-	return result;
-}
-
-static int ipa3_wigig_smmu_map_ring(u64 iova, u32 size, bool map,
-	struct sg_table *sgt, enum ipa_smmu_cb_type cb_type)
-{
-	struct ipa_wigig_smmu_ring_addr *entry;
-	struct ipa_wigig_smmu_ring_addr *next;
-	int result = 0;
-
-	IPADBG("iova %llX, %s\n", iova, map ? "map" : "unmap");
-	mutex_lock(&smmu_lock);
-	list_for_each_entry_safe(entry, next, &smmu_ring_addr_list, link) {
-		if ((entry->iova == iova) &&
-			(entry->cb_type == cb_type)) {
-			IPADBG("cb %d, page 0x%llX already mapped, ", cb_type,
-				iova);
-			if (map) {
-				entry->count++;
-				IPADBG("inc to %d\n", (entry->count));
-			} else {
-				--entry->count;
-				IPADBG("dec to %d\n", entry->count);
-				if (!(entry->count)) {
-					IPADBG("unmap and delete\n");
-					result = ipa3_smmu_map_peer_buff(
-						iova, size, map, sgt, cb_type);
-					if (result) {
-						IPAERR(
-							"failed to unmap 0x%llX\n",
-							iova);
-						goto finish;
-					}
-					list_del(&entry->link);
-					kfree(entry);
-				}
-			}
-			goto finish;
-		}
-	}
-	IPADBG("new page found 0x%llX, map and add to list\n", iova);
-	result = ipa3_smmu_map_peer_buff(iova, size, map, sgt, cb_type);
-	if (result) {
-		IPAERR("failed to map 0x%llX\n", iova);
-		goto finish;
-	}
-
-	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
-	if (entry == NULL) {
-		IPAERR("couldn't allocate for 0x%llX\n", iova);
-		ipa3_smmu_map_peer_buff(iova, size, !map, sgt, cb_type);
-		result = -ENOMEM;
-		goto finish;
-	}
-	INIT_LIST_HEAD(&entry->link);
-	entry->iova = iova;
-	entry->cb_type = cb_type;
-	entry->count = 1;
-	list_add(&entry->link, &smmu_ring_addr_list);
-
-finish:
-	mutex_unlock(&smmu_lock);
-	IPADBG("exit\n");
-	return result;
-}
-
-static int ipa3_wigig_smmu_map_channel(bool Rx,
-	struct ipa_wigig_pipe_setup_info_smmu *pipe_smmu,
-	void *buff,
-	bool map)
-{
-	int result = 0;
-	struct ipa_smmu_cb_ctx *smmu_ctx = ipa3_get_smmu_ctx(IPA_SMMU_CB_11AD);
-
-	IPADBG("\n");
-
-	/*
-	 * --------------------------------------------------------------------
-	 *  entity         |HWHEAD|HWTAIL|HWHEAD|HWTAIL| misc | buffers| rings|
-	 *                 |Sring |Sring |Dring |Dring | regs |        |      |
-	 * --------------------------------------------------------------------
-	 *  GSI (apps CB)  |  TX  |RX, TX|      |RX, TX|      |        |Rx, TX|
-	 * --------------------------------------------------------------------
-	 *  IPA (11AD CB)  |      |      |      |      |      | RX, TX |      |
-	 * --------------------------------------------------------------------
-	 *  uc (uC CB)     |  RX  |      |  TX  |      |always|        |      |
-	 * --------------------------------------------------------------------
-	 *
-	 * buffers are mapped to 11AD CB. in case this context bank is shared,
-	 * mapping is done by 11ad driver only and applies to both 11ad and
-	 * IPA HWs (page tables are shared). Otherwise, mapping is done here.
-	 */
-
-	if (!smmu_ctx) {
-		IPAERR("11AD SMMU ctx is null\n");
-		return -EINVAL;
-	}
-
-	if (Rx) {
-		IPADBG("RX %s status_ring_HWHEAD_pa %pa uC CB\n",
-			map ? "map" : "unmap",
-			&pipe_smmu->status_ring_HWHEAD_pa);
-		result = ipa3_wigig_smmu_map_reg(
-			rounddown(pipe_smmu->status_ring_HWHEAD_pa, PAGE_SIZE),
-			map,
-			IPA_SMMU_CB_UC);
-		if (result) {
-			IPAERR(
-				"failed to %s status_ring_HWAHEAD %d\n",
-				map ? "map" : "unmap",
-				result);
-			goto fail;
-		}
-	} else {
-		IPADBG("TX %s status_ring_HWHEAD_pa %pa AP CB\n",
-			map ? "map" : "unmap",
-			&pipe_smmu->status_ring_HWHEAD_pa);
-		result = ipa3_wigig_smmu_map_reg(
-			rounddown(pipe_smmu->status_ring_HWHEAD_pa,
-				PAGE_SIZE),
-			map,
-			IPA_SMMU_CB_AP);
-		if (result) {
-			IPAERR(
-				"failed to %s status_ring_HWAHEAD %d\n",
-				map ? "map" : "unmap",
-				result);
-			goto fail;
-		}
-
-		IPADBG("TX %s desc_ring_HWHEAD_pa %pa uC CB\n",
-			map ? "map" : "unmap",
-			&pipe_smmu->desc_ring_HWHEAD_pa);
-		result = ipa3_wigig_smmu_map_reg(
-			rounddown(pipe_smmu->desc_ring_HWHEAD_pa,
-				PAGE_SIZE),
-			map,
-			IPA_SMMU_CB_UC);
-		if (result) {
-			IPAERR("failed to %s desc_ring_HWHEAD %d\n",
-				map ? "map" : "unmap",
-				result);
-			goto fail_desc_HWHEAD;
-		}
-	}
-
-	IPADBG("%s status_ring_HWTAIL_pa %pa AP CB\n",
-		map ? "map" : "unmap",
-		&pipe_smmu->status_ring_HWTAIL_pa);
-	result = ipa3_wigig_smmu_map_reg(
-		rounddown(pipe_smmu->status_ring_HWTAIL_pa, PAGE_SIZE),
-		map,
-		IPA_SMMU_CB_AP);
-	if (result) {
-		IPAERR(
-			"failed to %s status_ring_HWTAIL %d\n",
-			map ? "map" : "unmap",
-			result);
-		goto fail_status_HWTAIL;
-	}
-
-	IPADBG("%s desc_ring_HWTAIL_pa %pa AP CB\n",
-		map ? "map" : "unmap",
-		&pipe_smmu->desc_ring_HWTAIL_pa);
-	result = ipa3_wigig_smmu_map_reg(
-		rounddown(pipe_smmu->desc_ring_HWTAIL_pa, PAGE_SIZE),
-		map,
-		IPA_SMMU_CB_AP);
-	if (result) {
-		IPAERR("failed to %s desc_ring_HWTAIL %d\n",
-			map ? "map" : "unmap",
-			result);
-		goto fail_desc_HWTAIL;
-	}
-
-	/* rings */
-	IPADBG("%s desc_ring_base_iova %llX AP CB\n",
-		map ? "map" : "unmap",
-		pipe_smmu->desc_ring_base_iova);
-	result = ipa3_wigig_smmu_map_ring(
-		pipe_smmu->desc_ring_base_iova,
-		pipe_smmu->desc_ring_size,
-		map,
-		&pipe_smmu->desc_ring_base,
-		IPA_SMMU_CB_AP);
-	if (result) {
-		IPAERR("failed to %s desc_ring_base %d\n",
-			map ? "map" : "unmap",
-			result);
-		goto fail_desc_ring;
-	}
-
-	IPADBG("%s status_ring_base_iova %llX AP CB\n",
-		map ? "map" : "unmap",
-		pipe_smmu->status_ring_base_iova);
-	result = ipa3_wigig_smmu_map_ring(
-		pipe_smmu->status_ring_base_iova,
-		pipe_smmu->status_ring_size,
-		map,
-		&pipe_smmu->status_ring_base,
-		IPA_SMMU_CB_AP);
-	if (result) {
-		IPAERR("failed to %s status_ring_base %d\n",
-			map ? "map" : "unmap",
-			result);
-		goto fail_status_ring;
-	}
-
-	if (!smmu_ctx->shared) {
-		IPADBG("CB not shared - map buffers\n");
-		result = ipa3_wigig_smmu_map_buffers(Rx, pipe_smmu, buff, map);
-		if (result) {
-			IPAERR("failed to %s buffers %d\n",
-				map ? "map" : "unmap",
-				result);
-			goto fail_buffers;
-		}
-	}
-
-	IPADBG("exit\n");
-	return 0;
-fail_buffers:
-	ipa3_wigig_smmu_map_ring(
-		pipe_smmu->status_ring_base_iova, pipe_smmu->status_ring_size,
-		!map, &pipe_smmu->status_ring_base, IPA_SMMU_CB_AP);
-fail_status_ring:
-	ipa3_wigig_smmu_map_ring(
-		pipe_smmu->desc_ring_base_iova,	pipe_smmu->desc_ring_size,
-		!map, &pipe_smmu->desc_ring_base, IPA_SMMU_CB_AP);
-fail_desc_ring:
-	ipa3_wigig_smmu_map_reg(
-		rounddown(pipe_smmu->desc_ring_HWTAIL_pa, PAGE_SIZE),
-		!map, IPA_SMMU_CB_AP);
-fail_desc_HWTAIL:
-	ipa3_wigig_smmu_map_reg(
-		rounddown(pipe_smmu->status_ring_HWTAIL_pa, PAGE_SIZE),
-		!map, IPA_SMMU_CB_AP);
-fail_status_HWTAIL:
-	if (Rx)
-		ipa3_wigig_smmu_map_reg(
-			rounddown(pipe_smmu->status_ring_HWHEAD_pa, PAGE_SIZE),
-			!map, IPA_SMMU_CB_UC);
-	else
-		ipa3_wigig_smmu_map_reg(
-			rounddown(pipe_smmu->desc_ring_HWHEAD_pa, PAGE_SIZE),
-			!map, IPA_SMMU_CB_UC);
-fail_desc_HWHEAD:
-	if (!Rx)
-		ipa3_wigig_smmu_map_reg(
-			rounddown(pipe_smmu->status_ring_HWHEAD_pa, PAGE_SIZE),
-			!map, IPA_SMMU_CB_AP);
-fail:
-	return result;
-}
-
-static void ipa_gsi_chan_err_cb(struct gsi_chan_err_notify *notify)
-{
-	switch (notify->evt_id) {
-	case GSI_CHAN_INVALID_TRE_ERR:
-		IPAERR("Got GSI_CHAN_INVALID_TRE_ERR\n");
-		break;
-	case GSI_CHAN_NON_ALLOCATED_EVT_ACCESS_ERR:
-		IPAERR("Got GSI_CHAN_NON_ALLOCATED_EVT_ACCESS_ERR\n");
-		break;
-	case GSI_CHAN_OUT_OF_BUFFERS_ERR:
-		IPAERR("Got GSI_CHAN_OUT_OF_BUFFERS_ERR\n");
-		break;
-	case GSI_CHAN_OUT_OF_RESOURCES_ERR:
-		IPAERR("Got GSI_CHAN_OUT_OF_RESOURCES_ERR\n");
-		break;
-	case GSI_CHAN_UNSUPPORTED_INTER_EE_OP_ERR:
-		IPAERR("Got GSI_CHAN_UNSUPPORTED_INTER_EE_OP_ERR\n");
-		break;
-	case GSI_CHAN_HWO_1_ERR:
-		IPAERR("Got GSI_CHAN_HWO_1_ERR\n");
-		break;
-	default:
-		IPAERR("Unexpected err evt: %d\n", notify->evt_id);
-	}
-	ipa_assert();
-}
-
-static void ipa_gsi_evt_ring_err_cb(struct gsi_evt_err_notify *notify)
-{
-	switch (notify->evt_id) {
-	case GSI_EVT_OUT_OF_BUFFERS_ERR:
-		IPAERR("Got GSI_EVT_OUT_OF_BUFFERS_ERR\n");
-		break;
-	case GSI_EVT_OUT_OF_RESOURCES_ERR:
-		IPAERR("Got GSI_EVT_OUT_OF_RESOURCES_ERR\n");
-		break;
-	case GSI_EVT_UNSUPPORTED_INTER_EE_OP_ERR:
-		IPAERR("Got GSI_EVT_UNSUPPORTED_INTER_EE_OP_ERR\n");
-		break;
-	case GSI_EVT_EVT_RING_EMPTY_ERR:
-		IPAERR("Got GSI_EVT_EVT_RING_EMPTY_ERR\n");
-		break;
-	default:
-		IPAERR("Unexpected err evt: %d\n", notify->evt_id);
-	}
-	ipa_assert();
-}
-
-static uint16_t int_modt = 15;
-static uint8_t int_modc = 200;
-static uint8_t tx_hwtail_mod_threshold = 200;
-static uint8_t rx_hwtail_mod_threshold = 200;
-
-static int ipa3_wigig_config_gsi(bool Rx,
-	bool smmu_en,
-	void *pipe_info,
-	void *buff,
-	const struct ipa_gsi_ep_config *ep_gsi,
-	struct ipa3_ep_context *ep)
-{
-	struct gsi_evt_ring_props evt_props;
-	struct gsi_chan_props channel_props;
-	union __packed gsi_channel_scratch gsi_scratch;
-	int gsi_res;
-	struct ipa_wigig_pipe_setup_info_smmu *pipe_smmu;
-	struct ipa_wigig_pipe_setup_info *pipe;
-	struct ipa_wigig_rx_pipe_data_buffer_info *rx_dbuff;
-	struct ipa_wigig_rx_pipe_data_buffer_info_smmu *rx_dbuff_smmu;
-	struct ipa_wigig_tx_pipe_data_buffer_info *tx_dbuff;
-	struct ipa_wigig_tx_pipe_data_buffer_info_smmu *tx_dbuff_smmu;
-
-	IPADBG("%s, %s\n", Rx ? "Rx" : "Tx", smmu_en ? "smmu en" : "smmu dis");
-
-	/* alloc event ring */
-	memset(&evt_props, 0, sizeof(evt_props));
-	evt_props.intf = GSI_EVT_CHTYPE_11AD_EV;
-	evt_props.re_size = GSI_EVT_RING_RE_SIZE_16B;
-	evt_props.intr = GSI_INTR_MSI;
-	evt_props.intvec = 0;
-	evt_props.exclusive = true;
-	evt_props.err_cb = ipa_gsi_evt_ring_err_cb;
-	evt_props.user_data = NULL;
-	evt_props.int_modc = int_modc;
-	evt_props.int_modt = int_modt;
-	evt_props.ring_base_vaddr = NULL;
-
-	if (smmu_en) {
-		pipe_smmu = (struct ipa_wigig_pipe_setup_info_smmu *)pipe_info;
-		evt_props.ring_base_addr =
-			pipe_smmu->desc_ring_base_iova;
-		evt_props.ring_len = pipe_smmu->desc_ring_size;
-		evt_props.msi_addr = pipe_smmu->desc_ring_HWTAIL_pa;
-	} else {
-		pipe = (struct ipa_wigig_pipe_setup_info *)pipe_info;
-		evt_props.ring_base_addr = pipe->desc_ring_base_pa;
-		evt_props.ring_len = pipe->desc_ring_size;
-		evt_props.msi_addr = pipe->desc_ring_HWTAIL_pa;
-	}
-
-	gsi_res = gsi_alloc_evt_ring(&evt_props,
-		ipa3_ctx->gsi_dev_hdl,
-		&ep->gsi_evt_ring_hdl);
-	if (gsi_res != GSI_STATUS_SUCCESS) {
-		IPAERR("Error allocating event ring: %d\n", gsi_res);
-		return -EFAULT;
-	}
-
-	/* event scratch not configured by SW for TX channels */
-	if (Rx) {
-		union __packed gsi_evt_scratch evt_scratch;
-
-		memset(&evt_scratch, 0, sizeof(evt_scratch));
-		evt_scratch.w11ad.update_status_hwtail_mod_threshold =
-			rx_hwtail_mod_threshold;
-		gsi_res = gsi_write_evt_ring_scratch(ep->gsi_evt_ring_hdl,
-			evt_scratch);
-		if (gsi_res != GSI_STATUS_SUCCESS) {
-			IPAERR("Error writing WIGIG event ring scratch: %d\n",
-				gsi_res);
-			goto fail_write_evt_scratch;
-		}
-	}
-
-	ep->gsi_mem_info.evt_ring_len = evt_props.ring_len;
-	ep->gsi_mem_info.evt_ring_base_addr = evt_props.ring_base_addr;
-	ep->gsi_mem_info.evt_ring_base_vaddr = evt_props.ring_base_vaddr;
-
-	/* alloc channel ring */
-	memset(&channel_props, 0, sizeof(channel_props));
-	memset(&gsi_scratch, 0, sizeof(gsi_scratch));
-
-	if (Rx)
-		channel_props.dir = GSI_CHAN_DIR_TO_GSI;
-	else
-		channel_props.dir = GSI_CHAN_DIR_FROM_GSI;
-
-	channel_props.re_size = GSI_CHAN_RE_SIZE_16B;
-	channel_props.prot = GSI_CHAN_PROT_11AD;
-	channel_props.ch_id = ep_gsi->ipa_gsi_chan_num;
-	channel_props.evt_ring_hdl = ep->gsi_evt_ring_hdl;
-	channel_props.xfer_cb = NULL;
-
-	channel_props.use_db_eng = GSI_CHAN_DB_MODE;
-	channel_props.max_prefetch = GSI_ONE_PREFETCH_SEG;
-	channel_props.prefetch_mode = ep_gsi->prefetch_mode;
-	channel_props.empty_lvl_threshold = ep_gsi->prefetch_threshold;
-	channel_props.low_weight = 1;
-	channel_props.err_cb = ipa_gsi_chan_err_cb;
-
-	channel_props.ring_base_vaddr = NULL;
-
-	if (Rx) {
-		if (smmu_en) {
-			rx_dbuff_smmu =
-			(struct ipa_wigig_rx_pipe_data_buffer_info_smmu *)buff;
-
-			channel_props.ring_base_addr =
-				pipe_smmu->status_ring_base_iova;
-			channel_props.ring_len =
-				pipe_smmu->status_ring_size;
-
-			gsi_scratch.rx_11ad.status_ring_hwtail_address_lsb =
-				IPA_WIGIG_LSB(
-					pipe_smmu->status_ring_HWTAIL_pa);
-			gsi_scratch.rx_11ad.status_ring_hwtail_address_msb =
-				IPA_WIGIG_MSB(
-					pipe_smmu->status_ring_HWTAIL_pa);
-
-			gsi_scratch.rx_11ad.data_buffers_base_address_lsb =
-				IPA_WIGIG_LSB(
-					rx_dbuff_smmu->data_buffer_base_iova);
-			gsi_scratch.rx_11ad.data_buffers_base_address_msb =
-				IPA_WIGIG_MSB(
-					rx_dbuff_smmu->data_buffer_base_iova);
-			gsi_scratch.rx_11ad.fixed_data_buffer_size_pow_2 =
-				ilog2(rx_dbuff_smmu->data_buffer_size);
-		} else {
-			rx_dbuff =
-			(struct ipa_wigig_rx_pipe_data_buffer_info *)buff;
-
-			channel_props.ring_base_addr =
-				pipe->status_ring_base_pa;
-			channel_props.ring_len = pipe->status_ring_size;
-
-			gsi_scratch.rx_11ad.status_ring_hwtail_address_lsb =
-				IPA_WIGIG_LSB(pipe->status_ring_HWTAIL_pa);
-			gsi_scratch.rx_11ad.status_ring_hwtail_address_msb =
-				IPA_WIGIG_MSB(pipe->status_ring_HWTAIL_pa);
-
-			gsi_scratch.rx_11ad.data_buffers_base_address_lsb =
-				IPA_WIGIG_LSB(rx_dbuff->data_buffer_base_pa);
-			gsi_scratch.rx_11ad.data_buffers_base_address_msb =
-				IPA_WIGIG_MSB(rx_dbuff->data_buffer_base_pa);
-			gsi_scratch.rx_11ad.fixed_data_buffer_size_pow_2 =
-				ilog2(rx_dbuff->data_buffer_size);
-		}
-		IPADBG("rx scratch: status_ring_hwtail_address_lsb 0x%X\n",
-			gsi_scratch.rx_11ad.status_ring_hwtail_address_lsb);
-		IPADBG("rx scratch: status_ring_hwtail_address_msb 0x%X\n",
-			gsi_scratch.rx_11ad.status_ring_hwtail_address_msb);
-		IPADBG("rx scratch: data_buffers_base_address_lsb 0x%X\n",
-			gsi_scratch.rx_11ad.data_buffers_base_address_lsb);
-		IPADBG("rx scratch: data_buffers_base_address_msb 0x%X\n",
-			gsi_scratch.rx_11ad.data_buffers_base_address_msb);
-		IPADBG("rx scratch: fixed_data_buffer_size_pow_2 %d\n",
-			gsi_scratch.rx_11ad.fixed_data_buffer_size_pow_2);
-		IPADBG("rx scratch 0x[%X][%X][%X][%X]\n",
-			gsi_scratch.data.word1,
-			gsi_scratch.data.word2,
-			gsi_scratch.data.word3,
-			gsi_scratch.data.word4);
-	} else {
-		if (smmu_en) {
-			tx_dbuff_smmu =
-			(struct ipa_wigig_tx_pipe_data_buffer_info_smmu *)buff;
-			channel_props.ring_base_addr =
-				pipe_smmu->desc_ring_base_iova;
-			channel_props.ring_len =
-				pipe_smmu->desc_ring_size;
-
-			gsi_scratch.tx_11ad.status_ring_hwtail_address_lsb =
-				IPA_WIGIG_LSB(
-					pipe_smmu->status_ring_HWTAIL_pa);
-			gsi_scratch.tx_11ad.status_ring_hwhead_address_lsb =
-				IPA_WIGIG_LSB(
-					pipe_smmu->status_ring_HWHEAD_pa);
-			gsi_scratch.tx_11ad.status_ring_hwhead_hwtail_8_msb =
-				IPA_WIGIG_8_MSB(
-					pipe_smmu->status_ring_HWHEAD_pa);
-
-			gsi_scratch.tx_11ad.fixed_data_buffer_size_pow_2 =
-				ilog2(tx_dbuff_smmu->data_buffer_size);
-
-			gsi_scratch.tx_11ad.status_ring_num_elem =
-				pipe_smmu->status_ring_size /
-				IPA_WIGIG_STATUS_RING_EL_SIZE;
-		} else {
-			tx_dbuff =
-			(struct ipa_wigig_tx_pipe_data_buffer_info *)buff;
-
-			channel_props.ring_base_addr = pipe->desc_ring_base_pa;
-			channel_props.ring_len = pipe->desc_ring_size;
-
-			gsi_scratch.tx_11ad.status_ring_hwtail_address_lsb =
-				IPA_WIGIG_LSB(
-					pipe->status_ring_HWTAIL_pa);
-			gsi_scratch.tx_11ad.status_ring_hwhead_address_lsb =
-				IPA_WIGIG_LSB(
-					pipe->status_ring_HWHEAD_pa);
-			gsi_scratch.tx_11ad.status_ring_hwhead_hwtail_8_msb =
-				IPA_WIGIG_8_MSB(pipe->status_ring_HWHEAD_pa);
-
-			gsi_scratch.tx_11ad.status_ring_num_elem =
-				pipe->status_ring_size /
-				IPA_WIGIG_STATUS_RING_EL_SIZE;
-
-			gsi_scratch.tx_11ad.fixed_data_buffer_size_pow_2 =
-				ilog2(tx_dbuff->data_buffer_size);
-		}
-		gsi_scratch.tx_11ad.update_status_hwtail_mod_threshold =
-			tx_hwtail_mod_threshold;
-		IPADBG("tx scratch: status_ring_hwtail_address_lsb 0x%X\n",
-			gsi_scratch.tx_11ad.status_ring_hwtail_address_lsb);
-		IPADBG("tx scratch: status_ring_hwhead_address_lsb 0x%X\n",
-			gsi_scratch.tx_11ad.status_ring_hwhead_address_lsb);
-		IPADBG("tx scratch: status_ring_hwhead_hwtail_8_msb 0x%X\n",
-			gsi_scratch.tx_11ad.status_ring_hwhead_hwtail_8_msb);
-		IPADBG("tx scratch:status_ring_num_elem %d\n",
-			gsi_scratch.tx_11ad.status_ring_num_elem);
-		IPADBG("tx scratch:fixed_data_buffer_size_pow_2 %d\n",
-			gsi_scratch.tx_11ad.fixed_data_buffer_size_pow_2);
-		IPADBG("tx scratch 0x[%X][%X][%X][%X]\n",
-			gsi_scratch.data.word1,
-			gsi_scratch.data.word2,
-			gsi_scratch.data.word3,
-			gsi_scratch.data.word4);
-	}
-
-	IPADBG("ch_id: %d\n", channel_props.ch_id);
-	IPADBG("evt_ring_hdl: %ld\n", channel_props.evt_ring_hdl);
-	IPADBG("re_size: %d\n", channel_props.re_size);
-	IPADBG("GSI channel ring len: %d\n", channel_props.ring_len);
-	IPADBG("channel ring  base addr = 0x%llX\n",
-		(unsigned long long)channel_props.ring_base_addr);
-
-	IPADBG("Allocating GSI channel\n");
-	gsi_res = gsi_alloc_channel(&channel_props,
-		ipa3_ctx->gsi_dev_hdl,
-		&ep->gsi_chan_hdl);
-	if (gsi_res != GSI_STATUS_SUCCESS) {
-		IPAERR("gsi_alloc_channel failed %d\n", gsi_res);
-		goto fail_alloc_channel;
-	}
-
-	IPADBG("Writing Channel scratch\n");
-	ep->gsi_mem_info.chan_ring_len = channel_props.ring_len;
-	ep->gsi_mem_info.chan_ring_base_addr = channel_props.ring_base_addr;
-	ep->gsi_mem_info.chan_ring_base_vaddr =
-		channel_props.ring_base_vaddr;
-
-	gsi_res = gsi_write_channel_scratch(ep->gsi_chan_hdl,
-		gsi_scratch);
-	if (gsi_res != GSI_STATUS_SUCCESS) {
-		IPAERR("gsi_write_channel_scratch failed %d\n",
-			gsi_res);
-		goto fail_write_channel_scratch;
-	}
-
-	IPADBG("exit\n");
-
-	return 0;
-fail_write_channel_scratch:
-	gsi_dealloc_channel(ep->gsi_chan_hdl);
-fail_alloc_channel:
-fail_write_evt_scratch:
-	gsi_dealloc_evt_ring(ep->gsi_evt_ring_hdl);
-	return -EFAULT;
-}
-
-static int ipa3_wigig_config_uc(bool init,
-	bool Rx,
-	u8 wifi_ch,
-	u8 gsi_ch,
-	phys_addr_t HWHEAD)
-{
-	struct ipa_mem_buffer cmd;
-	enum ipa_cpu_2_hw_offload_commands command;
-	int result;
-
-	IPADBG("%s\n", init ? "init" : "Deinit");
-	if (init) {
-		struct IpaHwOffloadSetUpCmdData_t_v4_0 *cmd_data;
-
-		cmd.size = sizeof(*cmd_data);
-		cmd.base = dma_alloc_coherent(ipa3_ctx->uc_pdev, cmd.size,
-			&cmd.phys_base, GFP_KERNEL);
-		if (cmd.base == NULL) {
-			IPAERR("fail to get DMA memory.\n");
-			return -ENOMEM;
-		}
-
-		cmd_data =
-			(struct IpaHwOffloadSetUpCmdData_t_v4_0 *)cmd.base;
-
-		cmd_data->protocol = IPA_HW_PROTOCOL_11ad;
-		cmd_data->SetupCh_params.W11AdSetupCh_params.dir =
-			Rx ? W11AD_RX : W11AD_TX;
-		cmd_data->SetupCh_params.W11AdSetupCh_params.gsi_ch = gsi_ch;
-		cmd_data->SetupCh_params.W11AdSetupCh_params.wifi_ch = wifi_ch;
-		cmd_data->SetupCh_params.W11AdSetupCh_params.wifi_hp_addr_msb =
-			IPA_WIGIG_MSB(HWHEAD);
-		cmd_data->SetupCh_params.W11AdSetupCh_params.wifi_hp_addr_lsb =
-			IPA_WIGIG_LSB(HWHEAD);
-		command = IPA_CPU_2_HW_CMD_OFFLOAD_CHANNEL_SET_UP;
-
-	} else {
-		struct IpaHwOffloadCommonChCmdData_t_v4_0 *cmd_data;
-
-		cmd.size = sizeof(*cmd_data);
-		cmd.base = dma_alloc_coherent(ipa3_ctx->uc_pdev, cmd.size,
-			&cmd.phys_base, GFP_KERNEL);
-		if (cmd.base == NULL) {
-			IPAERR("fail to get DMA memory.\n");
-			return -ENOMEM;
-		}
-
-		cmd_data =
-			(struct IpaHwOffloadCommonChCmdData_t_v4_0 *)cmd.base;
-
-		cmd_data->protocol = IPA_HW_PROTOCOL_11ad;
-		cmd_data->CommonCh_params.W11AdCommonCh_params.gsi_ch = gsi_ch;
-		command = IPA_CPU_2_HW_CMD_OFFLOAD_TEAR_DOWN;
-	}
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	result = ipa3_uc_send_cmd((u32)(cmd.phys_base),
-		command,
-		IPA_HW_2_CPU_OFFLOAD_CMD_STATUS_SUCCESS,
-		false, 10 * HZ);
-	if (result) {
-		IPAERR("fail to %s uc for %s gsi channel %d\n",
-			init ? "init" : "deinit",
-			Rx ? "Rx" : "Tx", gsi_ch);
-	}
-
-	dma_free_coherent(ipa3_ctx->uc_pdev,
-		cmd.size, cmd.base, cmd.phys_base);
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-
-	IPADBG("exit\n");
-	return result;
-}
-
-int ipa3_conn_wigig_rx_pipe_i(void *in, struct ipa_wigig_conn_out_params *out,
-	struct dentry **parent)
-{
-	int ipa_ep_idx;
-	struct ipa3_ep_context *ep;
-	struct ipa_ep_cfg ep_cfg;
-	enum ipa_client_type rx_client = IPA_CLIENT_WIGIG_PROD;
-	bool is_smmu_enabled;
-	struct ipa_wigig_conn_rx_in_params_smmu *input_smmu = NULL;
-	struct ipa_wigig_conn_rx_in_params *input = NULL;
-	const struct ipa_gsi_ep_config *ep_gsi;
-	void *pipe_info;
-	void *buff;
-	phys_addr_t status_ring_HWHEAD_pa;
-	int result;
-
-	IPADBG("\n");
-
-	*parent = wigig_dent;
-
-	ipa_ep_idx = ipa_get_ep_mapping(rx_client);
-	if (ipa_ep_idx == IPA_EP_NOT_ALLOCATED ||
-		ipa_ep_idx >= IPA3_MAX_NUM_PIPES) {
-		IPAERR("fail to get ep (IPA_CLIENT_WIGIG_PROD) %d.\n",
-			ipa_ep_idx);
-		return -EFAULT;
-	}
-
-	ep = &ipa3_ctx->ep[ipa_ep_idx];
-	if (ep->valid) {
-		IPAERR("EP %d already allocated.\n", ipa_ep_idx);
-		return -EFAULT;
-	}
-
-	if (ep->gsi_offload_state) {
-		IPAERR("WIGIG channel bad state 0x%X\n",
-			ep->gsi_offload_state);
-		return -EFAULT;
-	}
-
-	ep_gsi = ipa3_get_gsi_ep_info(rx_client);
-	if (!ep_gsi) {
-		IPAERR("Failed getting GSI EP info for client=%d\n",
-			rx_client);
-		return -EPERM;
-	}
-
-	memset(ep, 0, offsetof(struct ipa3_ep_context, sys));
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	/* setup rx ep cfg */
-	ep->valid = 1;
-	ep->client = rx_client;
-	result = ipa3_disable_data_path(ipa_ep_idx);
-	if (result) {
-		IPAERR("disable data path failed res=%d clnt=%d.\n", result,
-			ipa_ep_idx);
-		IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-		return -EFAULT;
-	}
-
-	is_smmu_enabled = !ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_11AD];
-	if (is_smmu_enabled) {
-		struct ipa_wigig_rx_pipe_data_buffer_info_smmu *dbuff_smmu;
-
-		input_smmu = (struct ipa_wigig_conn_rx_in_params_smmu *)in;
-		dbuff_smmu = &input_smmu->dbuff_smmu;
-		ep->client_notify = input_smmu->notify;
-		ep->priv = input_smmu->priv;
-
-		IPADBG(
-		"desc_ring_base_iova 0x%llX desc_ring_size %d status_ring_base_iova 0x%llX status_ring_size %d",
-		(unsigned long long)input_smmu->pipe_smmu.desc_ring_base_iova,
-		input_smmu->pipe_smmu.desc_ring_size,
-		(unsigned long long)input_smmu->pipe_smmu.status_ring_base_iova,
-		input_smmu->pipe_smmu.status_ring_size);
-		IPADBG("data_buffer_base_iova 0x%llX data_buffer_size %d",
-			(unsigned long long)dbuff_smmu->data_buffer_base_iova,
-			input_smmu->dbuff_smmu.data_buffer_size);
-
-		if (IPA_WIGIG_MSB(
-			dbuff_smmu->data_buffer_base_iova) &
-			0xFFFFFF00) {
-			IPAERR(
-			"data_buffers_base_address_msb is over the 8 bit limit (0x%llX)\n",
-			(unsigned long long)dbuff_smmu->data_buffer_base_iova);
-			IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-			return -EFAULT;
-		}
-		if (dbuff_smmu->data_buffer_size >> 16) {
-			IPAERR(
-				"data_buffer_size is over the 16 bit limit (%d)\n"
-				, dbuff_smmu->data_buffer_size);
-			IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-			return -EFAULT;
-		}
-	} else {
-		input = (struct ipa_wigig_conn_rx_in_params *)in;
-		ep->client_notify = input->notify;
-		ep->priv = input->priv;
-
-		IPADBG(
-			"desc_ring_base_pa %pa desc_ring_size %d status_ring_base_pa %pa status_ring_size %d",
-			&input->pipe.desc_ring_base_pa,
-			input->pipe.desc_ring_size,
-			&input->pipe.status_ring_base_pa,
-			input->pipe.status_ring_size);
-		IPADBG("data_buffer_base_pa %pa data_buffer_size %d",
-			&input->dbuff.data_buffer_base_pa,
-			input->dbuff.data_buffer_size);
-
-		if (
-		IPA_WIGIG_MSB(input->dbuff.data_buffer_base_pa) & 0xFFFFFF00) {
-			IPAERR(
-				"data_buffers_base_address_msb is over the 8 bit limit (0x%pa)\n"
-				, &input->dbuff.data_buffer_base_pa);
-			IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-			return -EFAULT;
-		}
-		if (input->dbuff.data_buffer_size >> 16) {
-			IPAERR(
-				"data_buffer_size is over the 16 bit limit (0x%X)\n"
-				, input->dbuff.data_buffer_size);
-			IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-			return -EFAULT;
-		}
-	}
-
-	memset(&ep_cfg, 0, sizeof(ep_cfg));
-	ep_cfg.nat.nat_en = IPA_SRC_NAT;
-	ep_cfg.hdr.hdr_len = ETH_HLEN;
-	ep_cfg.hdr.hdr_ofst_pkt_size_valid = 0;
-	ep_cfg.hdr.hdr_ofst_pkt_size = 0;
-	ep_cfg.hdr.hdr_additional_const_len = 0;
-	ep_cfg.hdr_ext.hdr_little_endian = true;
-	ep_cfg.hdr.hdr_ofst_metadata_valid = 0;
-	ep_cfg.hdr.hdr_metadata_reg_valid = 1;
-	ep_cfg.mode.mode = IPA_BASIC;
-
-
-	if (ipa3_cfg_ep(ipa_ep_idx, &ep_cfg)) {
-		IPAERR("fail to setup rx pipe cfg\n");
-		result = -EFAULT;
-		goto fail;
-	}
-
-	if (is_smmu_enabled) {
-		result = ipa3_wigig_smmu_map_channel(true,
-			&input_smmu->pipe_smmu,
-			&input_smmu->dbuff_smmu,
-			true);
-		if (result) {
-			IPAERR("failed to setup rx pipe smmu map\n");
-			result = -EFAULT;
-			goto fail;
-		}
-
-		pipe_info = &input_smmu->pipe_smmu;
-		buff = &input_smmu->dbuff_smmu;
-		status_ring_HWHEAD_pa =
-			input_smmu->pipe_smmu.status_ring_HWHEAD_pa;
-	} else {
-		pipe_info = &input->pipe;
-		buff = &input->dbuff;
-		status_ring_HWHEAD_pa =
-			input->pipe.status_ring_HWHEAD_pa;
-	}
-
-	result = ipa3_wigig_config_gsi(true,
-		is_smmu_enabled,
-		pipe_info,
-		buff,
-		ep_gsi, ep);
-	if (result)
-		goto fail_gsi;
-
-	result = ipa3_wigig_config_uc(
-		true, true, 0,
-		ep_gsi->ipa_gsi_chan_num,
-		status_ring_HWHEAD_pa);
-	if (result)
-		goto fail_uc_config;
-
-	ipa3_install_dflt_flt_rules(ipa_ep_idx);
-
-	out->client = IPA_CLIENT_WIGIG_PROD;
-	ep->gsi_offload_state |= IPA_WIGIG_CONNECTED;
-
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-
-	IPADBG("wigig rx pipe connected successfully\n");
-	IPADBG("exit\n");
-
-	return 0;
-
-fail_uc_config:
-	/* Release channel and evt*/
-	ipa3_release_gsi_channel(ipa_ep_idx);
-fail_gsi:
-	if (input_smmu)
-		ipa3_wigig_smmu_map_channel(true, &input_smmu->pipe_smmu,
-			&input_smmu->dbuff_smmu, false);
-fail:
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-	return result;
-}
-
-int ipa3_conn_wigig_client_i(void *in,
-	struct ipa_wigig_conn_out_params *out,
-	ipa_notify_cb tx_notify,
-	void *priv)
-{
-	int ipa_ep_idx;
-	struct ipa3_ep_context *ep;
-	struct ipa_ep_cfg ep_cfg;
-	enum ipa_client_type tx_client;
-	bool is_smmu_enabled;
-	struct ipa_wigig_conn_tx_in_params_smmu *input_smmu = NULL;
-	struct ipa_wigig_conn_tx_in_params *input = NULL;
-	const struct ipa_gsi_ep_config *ep_gsi;
-	u32 aggr_byte_limit;
-	int result;
-	void *pipe_info;
-	void *buff;
-	phys_addr_t desc_ring_HWHEAD_pa;
-	u8 wifi_ch;
-
-	IPADBG("\n");
-
-	is_smmu_enabled = !ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_11AD];
-	if (is_smmu_enabled) {
-		input_smmu = (struct ipa_wigig_conn_tx_in_params_smmu *)in;
-
-		IPADBG(
-		"desc_ring_base_iova 0x%llX desc_ring_size %d status_ring_base_iova 0x%llX status_ring_size %d",
-		(unsigned long long)input_smmu->pipe_smmu.desc_ring_base_iova,
-		input_smmu->pipe_smmu.desc_ring_size,
-		(unsigned long long)input_smmu->pipe_smmu.status_ring_base_iova,
-		input_smmu->pipe_smmu.status_ring_size);
-		IPADBG("num buffers %d, data buffer size %d\n",
-			input_smmu->dbuff_smmu.num_buffers,
-			input_smmu->dbuff_smmu.data_buffer_size);
-
-		if (ipa3_wigig_tx_bit_to_ep(input_smmu->int_gen_tx_bit_num,
-			&tx_client)) {
-			return -EINVAL;
-		}
-		if (input_smmu->dbuff_smmu.data_buffer_size >> 16) {
-			IPAERR(
-				"data_buffer_size is over the 16 bit limit (0x%X)\n"
-				, input_smmu->dbuff_smmu.data_buffer_size);
-			return -EFAULT;
-		}
-
-		if (IPA_WIGIG_8_MSB(
-			input_smmu->pipe_smmu.status_ring_HWHEAD_pa)
-			!= IPA_WIGIG_8_MSB(
-				input_smmu->pipe_smmu.status_ring_HWTAIL_pa)) {
-			IPAERR(
-				"status ring HWHEAD and HWTAIL differ in 8 MSbs head 0x%llX tail 0x%llX\n"
-			, input_smmu->pipe_smmu.status_ring_HWHEAD_pa,
-			input_smmu->pipe_smmu.status_ring_HWTAIL_pa);
-			return -EFAULT;
-		}
-
-		wifi_ch = input_smmu->int_gen_tx_bit_num;
-
-		/* convert to kBytes */
-		aggr_byte_limit = IPA_ADJUST_AGGR_BYTE_HARD_LIMIT(
-			input_smmu->dbuff_smmu.data_buffer_size);
-	} else {
-		input = (struct ipa_wigig_conn_tx_in_params *)in;
-
-		IPADBG(
-			"desc_ring_base_pa %pa desc_ring_size %d status_ring_base_pa %pa status_ring_size %d",
-			&input->pipe.desc_ring_base_pa,
-			input->pipe.desc_ring_size,
-			&input->pipe.status_ring_base_pa,
-			input->pipe.status_ring_size);
-		IPADBG("data_buffer_size %d", input->dbuff.data_buffer_size);
-
-		if (ipa3_wigig_tx_bit_to_ep(input->int_gen_tx_bit_num,
-			&tx_client)) {
-			return -EINVAL;
-		}
-
-		if (input->dbuff.data_buffer_size >> 16) {
-			IPAERR(
-				"data_buffer_size is over the 16 bit limit (0x%X)\n"
-				, input->dbuff.data_buffer_size);
-			return -EFAULT;
-		}
-
-		if (IPA_WIGIG_8_MSB(
-			input->pipe.status_ring_HWHEAD_pa)
-			!= IPA_WIGIG_8_MSB(
-				input->pipe.status_ring_HWTAIL_pa)) {
-			IPAERR(
-				"status ring HWHEAD and HWTAIL differ in 8 MSbs head 0x%llX tail 0x%llX\n"
-				, input->pipe.status_ring_HWHEAD_pa,
-				input->pipe.status_ring_HWTAIL_pa);
-			return -EFAULT;
-		}
-
-		wifi_ch = input->int_gen_tx_bit_num;
-
-		/* convert to kBytes */
-		aggr_byte_limit = IPA_ADJUST_AGGR_BYTE_HARD_LIMIT(
-			input->dbuff.data_buffer_size);
-	}
-	IPADBG("client type is %d\n", tx_client);
-
-	ipa_ep_idx = ipa_get_ep_mapping(tx_client);
-	if (ipa_ep_idx == IPA_EP_NOT_ALLOCATED ||
-		ipa_ep_idx >= IPA3_MAX_NUM_PIPES) {
-		IPAERR("fail to get ep (%d) %d.\n",
-			tx_client, ipa_ep_idx);
-		return -EFAULT;
-	}
-
-	ep = &ipa3_ctx->ep[ipa_ep_idx];
-	if (ep->valid) {
-		IPAERR("EP %d already allocated.\n", ipa_ep_idx);
-		return -EFAULT;
-	}
-
-	if (ep->gsi_offload_state) {
-		IPAERR("WIGIG channel bad state 0x%X\n",
-			ep->gsi_offload_state);
-		return -EFAULT;
-	}
-
-	ep_gsi = ipa3_get_gsi_ep_info(tx_client);
-	if (!ep_gsi) {
-		IPAERR("Failed getting GSI EP info for client=%d\n",
-			tx_client);
-		return -EFAULT;
-	}
-
-	memset(ep, 0, offsetof(struct ipa3_ep_context, sys));
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	/* setup tx ep cfg */
-	ep->valid = 1;
-	ep->client = tx_client;
-	result = ipa3_disable_data_path(ipa_ep_idx);
-	if (result) {
-		IPAERR("disable data path failed res=%d clnt=%d.\n", result,
-			ipa_ep_idx);
-		goto fail;
-	}
-
-	ep->client_notify = tx_notify;
-	ep->priv = priv;
-
-	memset(&ep_cfg, 0, sizeof(ep_cfg));
-	ep_cfg.nat.nat_en = IPA_DST_NAT;
-	ep_cfg.hdr.hdr_len = ETH_HLEN;
-	ep_cfg.hdr.hdr_ofst_pkt_size_valid = 0;
-	ep_cfg.hdr.hdr_ofst_pkt_size = 0;
-	ep_cfg.hdr.hdr_additional_const_len = 0;
-	ep_cfg.hdr_ext.hdr_little_endian = true;
-	ep_cfg.mode.mode = IPA_BASIC;
-
-	/* config hard byte limit, max is the buffer size (in kB)*/
-	ep_cfg.aggr.aggr_en = IPA_ENABLE_AGGR;
-	ep_cfg.aggr.aggr = IPA_GENERIC;
-	ep_cfg.aggr.aggr_pkt_limit = 1;
-	ep_cfg.aggr.aggr_byte_limit = aggr_byte_limit;
-	ep_cfg.aggr.aggr_hard_byte_limit_en = IPA_ENABLE_AGGR;
-
-	if (ipa3_cfg_ep(ipa_ep_idx, &ep_cfg)) {
-		IPAERR("fail to setup rx pipe cfg\n");
-		result = -EFAULT;
-		goto fail;
-	}
-
-	if (is_smmu_enabled) {
-		result = ipa3_wigig_smmu_map_channel(false,
-			&input_smmu->pipe_smmu,
-			&input_smmu->dbuff_smmu,
-			true);
-		if (result) {
-			IPAERR(
-				"failed to setup tx pipe smmu map client %d (ep %d)\n"
-			, tx_client, ipa_ep_idx);
-			result = -EFAULT;
-			goto fail;
-		}
-
-		pipe_info = &input_smmu->pipe_smmu;
-		buff = &input_smmu->dbuff_smmu;
-		desc_ring_HWHEAD_pa =
-			input_smmu->pipe_smmu.desc_ring_HWHEAD_pa;
-	} else {
-		pipe_info = &input->pipe;
-		buff = &input->dbuff;
-		desc_ring_HWHEAD_pa =
-			input->pipe.desc_ring_HWHEAD_pa;
-	}
-
-	result = ipa3_wigig_config_gsi(false,
-		is_smmu_enabled,
-		pipe_info,
-		buff,
-		ep_gsi, ep);
-	if (result)
-		goto fail_gsi;
-
-	result = ipa3_wigig_config_uc(
-		true, false, wifi_ch,
-		ep_gsi->ipa_gsi_chan_num,
-		desc_ring_HWHEAD_pa);
-	if (result)
-		goto fail_uc_config;
-
-	out->client = tx_client;
-	ep->gsi_offload_state |= IPA_WIGIG_CONNECTED;
-
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-
-	IPADBG("wigig client %d (ep %d) connected successfully\n", tx_client,
-		ipa_ep_idx);
-	return 0;
-
-fail_uc_config:
-	/* Release channel and evt*/
-	ipa3_release_gsi_channel(ipa_ep_idx);
-fail_gsi:
-	if (input_smmu)
-		ipa3_wigig_smmu_map_channel(false, &input_smmu->pipe_smmu,
-			&input_smmu->dbuff_smmu, false);
-fail:
-	ep->valid = 0;
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-	return result;
-}
-
-int ipa3_disconn_wigig_pipe_i(enum ipa_client_type client,
-	struct ipa_wigig_pipe_setup_info_smmu *pipe_smmu,
-	void *dbuff)
-{
-	bool is_smmu_enabled;
-	int ipa_ep_idx;
-	struct ipa3_ep_context *ep;
-	const struct ipa_gsi_ep_config *ep_gsi;
-	int result;
-	bool rx = false;
-
-	IPADBG("\n");
-
-	ipa_ep_idx = ipa_get_ep_mapping(client);
-	if (ipa_ep_idx == IPA_EP_NOT_ALLOCATED ||
-		ipa_ep_idx >= IPA3_MAX_NUM_PIPES) {
-		IPAERR("fail to get ep (%d) %d.\n",
-			client, ipa_ep_idx);
-		return -EFAULT;
-	}
-
-	ep = &ipa3_ctx->ep[ipa_ep_idx];
-	if (!ep->valid) {
-		IPAERR("Invalid EP\n");
-		return -EFAULT;
-	}
-
-	ep_gsi = ipa3_get_gsi_ep_info(client);
-	if (!ep_gsi) {
-		IPAERR("Failed getting GSI EP info for client=%d\n",
-			client);
-		return -EFAULT;
-	}
-
-	if (ep->gsi_offload_state != IPA_WIGIG_CONNECTED) {
-		IPAERR("client in bad state(client %d) 0x%X\n",
-			client, ep->gsi_offload_state);
-		return -EFAULT;
-	}
-
-	if (client == IPA_CLIENT_WIGIG_PROD)
-		rx = true;
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	/* Release channel and evt*/
-	result = ipa3_release_gsi_channel(ipa_ep_idx);
-	if (result) {
-		IPAERR("failed to deallocate channel\n");
-		goto fail;
-	}
-
-	/* only gsi ch number and dir are necessary */
-	result = ipa3_wigig_config_uc(
-		false, rx, 0,
-		ep_gsi->ipa_gsi_chan_num, 0);
-	if (result) {
-		IPAERR("failed uC channel teardown %d\n", result);
-		WARN_ON(1);
-	}
-
-	is_smmu_enabled = !ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_11AD];
-	if (is_smmu_enabled) {
-		if (!pipe_smmu || !dbuff) {
-			IPAERR("smmu input is null %pK %pK\n",
-				pipe_smmu, dbuff);
-			WARN_ON(1);
-		} else {
-			result = ipa3_wigig_smmu_map_channel(rx,
-				pipe_smmu,
-				dbuff,
-				false);
-			if (result) {
-				IPAERR(
-					"failed to unmap pipe smmu %d (ep %d)\n"
-					, client, ipa_ep_idx);
-				result = -EFAULT;
-				goto fail;
-			}
-		}
-
-		if (rx) {
-			if (!list_empty(&smmu_reg_addr_list)) {
-				IPAERR("smmu_reg_addr_list not empty\n");
-				WARN_ON(1);
-			}
-
-			if (!list_empty(&smmu_ring_addr_list)) {
-				IPAERR("smmu_ring_addr_list not empty\n");
-				WARN_ON(1);
-			}
-		}
-	} else if (pipe_smmu || dbuff) {
-		IPAERR("smmu input is not null %pK %pK\n",
-			pipe_smmu, dbuff);
-		WARN_ON(1);
-	}
-
-	memset(ep, 0, sizeof(struct ipa3_ep_context));
-
-	ep->gsi_offload_state = 0;
-
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-
-	IPADBG("client (ep: %d) disconnected\n", ipa_ep_idx);
-
-	IPADBG("exit\n");
-	return 0;
-
-fail:
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-	return result;
-}
-
-int ipa3_wigig_uc_msi_init(bool init,
-	phys_addr_t periph_baddr_pa,
-	phys_addr_t pseudo_cause_pa,
-	phys_addr_t int_gen_tx_pa,
-	phys_addr_t int_gen_rx_pa,
-	phys_addr_t dma_ep_misc_pa)
-{
-	int result;
-	struct ipa_mem_buffer cmd;
-	enum ipa_cpu_2_hw_offload_commands command;
-	bool map = false;
-
-	IPADBG("params: %s, %pa, %pa, %pa, %pa, %pa\n",
-		init ? "init" : "deInit",
-		&periph_baddr_pa,
-		&pseudo_cause_pa,
-		&int_gen_tx_pa,
-		&int_gen_rx_pa,
-		&dma_ep_misc_pa);
-
-	/* first make sure registers are SMMU mapped if necessary*/
-	if ((!ipa3_ctx->s1_bypass_arr[IPA_SMMU_CB_UC])) {
-		if (init)
-			map = true;
-
-		IPADBG("SMMU enabled, map %d\n", map);
-
-		result = ipa3_smmu_map_peer_reg(
-			rounddown(pseudo_cause_pa, PAGE_SIZE),
-			map,
-			IPA_SMMU_CB_UC);
-		if (result) {
-			IPAERR(
-				"failed to %s pseudo_cause reg %d\n",
-				map ? "map" : "unmap",
-				result);
-			goto fail;
-		}
-
-		result = ipa3_smmu_map_peer_reg(
-			rounddown(int_gen_tx_pa, PAGE_SIZE),
-			map,
-			IPA_SMMU_CB_UC);
-		if (result) {
-			IPAERR(
-				"failed to %s int_gen_tx reg %d\n",
-				map ? "map" : "unmap",
-				result);
-			goto fail_gen_tx;
-		}
-
-		result = ipa3_smmu_map_peer_reg(
-			rounddown(int_gen_rx_pa, PAGE_SIZE),
-			map,
-			IPA_SMMU_CB_UC);
-		if (result) {
-			IPAERR(
-				"failed to %s int_gen_rx reg %d\n",
-				map ? "map" : "unmap",
-				result);
-			goto fail_gen_rx;
-		}
-
-		result = ipa3_smmu_map_peer_reg(
-			rounddown(dma_ep_misc_pa, PAGE_SIZE),
-			map,
-			IPA_SMMU_CB_UC);
-		if (result) {
-			IPAERR(
-				"failed to %s dma_ep_misc reg %d\n",
-				map ? "map" : "unmap",
-				result);
-			goto fail_dma_ep_misc;
-		}
-	}
-
-	/*  now send the wigig hw base address to uC*/
-	if (init) {
-		struct IpaHwPeripheralInitCmdData_t *cmd_data;
-
-		cmd.size = sizeof(*cmd_data);
-		cmd.base = dma_alloc_coherent(ipa3_ctx->uc_pdev, cmd.size,
-			&cmd.phys_base, GFP_KERNEL);
-		if (cmd.base == NULL) {
-			IPAERR("fail to get DMA memory.\n");
-			result = -ENOMEM;
-			if (map)
-				goto fail_alloc;
-			return result;
-		}
-		cmd_data = (struct IpaHwPeripheralInitCmdData_t *)cmd.base;
-		cmd_data->protocol = IPA_HW_PROTOCOL_11ad;
-		cmd_data->Init_params.W11AdInit_params.periph_baddr_msb =
-			IPA_WIGIG_MSB(periph_baddr_pa);
-		cmd_data->Init_params.W11AdInit_params.periph_baddr_lsb =
-			IPA_WIGIG_LSB(periph_baddr_pa);
-		command = IPA_CPU_2_HW_CMD_PERIPHERAL_INIT;
-	} else {
-		struct IpaHwPeripheralDeinitCmdData_t *cmd_data;
-
-		cmd.size = sizeof(*cmd_data);
-		cmd.base = dma_alloc_coherent(ipa3_ctx->uc_pdev, cmd.size,
-			&cmd.phys_base, GFP_KERNEL);
-		if (cmd.base == NULL) {
-			IPAERR("fail to get DMA memory.\n");
-			result = -ENOMEM;
-			if (map)
-				goto fail_alloc;
-			return result;
-		}
-		cmd_data = (struct IpaHwPeripheralDeinitCmdData_t *)cmd.base;
-		cmd_data->protocol = IPA_HW_PROTOCOL_11ad;
-		command = IPA_CPU_2_HW_CMD_PERIPHERAL_DEINIT;
-	}
-
-	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
-
-	result = ipa3_uc_send_cmd((u32)(cmd.phys_base),
-		command,
-		IPA_HW_2_CPU_OFFLOAD_CMD_STATUS_SUCCESS,
-		false, 10 * HZ);
-	if (result) {
-		IPAERR("fail to %s uc MSI config\n", init ? "init" : "deinit");
-		goto fail_command;
-	}
-
-	dma_free_coherent(ipa3_ctx->uc_pdev, cmd.size,
-		cmd.base, cmd.phys_base);
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-
-	IPADBG("exit\n");
-
-	return 0;
-fail_command:
-	dma_free_coherent(ipa3_ctx->uc_pdev,
-		cmd.size,
-		cmd.base, cmd.phys_base);
-	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();
-fail_alloc:
-	ipa3_smmu_map_peer_reg(
-		rounddown(dma_ep_misc_pa, PAGE_SIZE), !map, IPA_SMMU_CB_UC);
-fail_dma_ep_misc:
-	ipa3_smmu_map_peer_reg(
-		rounddown(int_gen_rx_pa, PAGE_SIZE), !map, IPA_SMMU_CB_UC);
-fail_gen_rx:
-	ipa3_smmu_map_peer_reg(
-		rounddown(int_gen_tx_pa, PAGE_SIZE), !map, IPA_SMMU_CB_UC);
-fail_gen_tx:
-	ipa3_smmu_map_peer_reg(
-		rounddown(pseudo_cause_pa, PAGE_SIZE), !map, IPA_SMMU_CB_UC);
-fail:
-	return result;
-}
-
-int ipa3_enable_wigig_pipe_i(enum ipa_client_type client)
-{
-	int ipa_ep_idx, res;
-	struct ipa3_ep_context *ep;
-	struct ipa_ep_cfg_ctrl ep_cfg_ctrl;
-	int retry_cnt = 0;
-	uint64_t val;
-
-	IPADBG("\n");
-
-	ipa_ep_idx = ipa_get_ep_mapping(client);
-	if (ipa_ep_idx == IPA_EP_NOT_ALLOCATED ||
-		ipa_ep_idx >= IPA3_MAX_NUM_PIPES) {
-		IPAERR("fail to get ep (%d) %d.\n",
-			client, ipa_ep_idx);
-		return -EFAULT;
-	}
-
-	ep = &ipa3_ctx->ep[ipa_ep_idx];
-
-	if (!ep->valid) {
-		IPAERR("Invalid EP\n");
-		return -EFAULT;
-	}
-
-	if (ep->gsi_offload_state != IPA_WIGIG_CONNECTED) {
-		IPAERR("WIGIG channel bad state 0x%X\n",
-			ep->gsi_offload_state);
-		return -EFAULT;
-	}
-
-	IPA_ACTIVE_CLIENTS_INC_EP(client);
-
-	res = ipa3_enable_data_path(ipa_ep_idx);
-	if (res)
-		goto fail_enable_datapath;
-
-	memset(&ep_cfg_ctrl, 0, sizeof(struct ipa_ep_cfg_ctrl));
-	ipa3_cfg_ep_ctrl(ipa_ep_idx, &ep_cfg_ctrl);
-
-	/* ring the event db (outside the ring boundary)*/
-	val = ep->gsi_mem_info.evt_ring_base_addr +
-		ep->gsi_mem_info.evt_ring_len;
-	res = gsi_ring_evt_ring_db(ep->gsi_evt_ring_hdl, val);
-	if (res) {
-		IPAERR(
-			"fail to ring evt ring db %d. hdl=%lu wp=0x%llx\n"
-			, res, ep->gsi_evt_ring_hdl,
-			(unsigned long long)val);
-		res = -EFAULT;
-		goto fail_ring_evt;
-	}
-
-	IPADBG("start channel\n");
-	res = gsi_start_channel(ep->gsi_chan_hdl);
-	if (res != GSI_STATUS_SUCCESS) {
-		IPAERR("gsi_start_channel failed %d\n", res);
-		WARN_ON(1);
-		res = -EFAULT;
-		goto fail_gsi_start;
-	}
-
-	/* for TX we have to ring the channel db (last desc in the ring) */
-	if (client != IPA_CLIENT_WIGIG_PROD) {
-		uint64_t val;
-
-		val  = ep->gsi_mem_info.chan_ring_base_addr +
-			ep->gsi_mem_info.chan_ring_len -
-			IPA_WIGIG_DESC_RING_EL_SIZE;
-
-		IPADBG("ring ch doorbell (0x%llX) TX %ld\n", val,
-			ep->gsi_chan_hdl);
-		res = gsi_ring_ch_ring_db(ep->gsi_chan_hdl, val);
-		if (res) {
-			IPAERR(
-				"fail to ring channel db %d. hdl=%lu wp=0x%llx\n"
-				, res, ep->gsi_chan_hdl,
-				(unsigned long long)val);
-			res = -EFAULT;
-			goto fail_ring_ch;
-		}
-	}
-
-	ep->gsi_offload_state |= IPA_WIGIG_ENABLED;
-
-	IPADBG("exit\n");
-
-	return 0;
-
-fail_ring_ch:
-	res = ipa3_stop_gsi_channel(ipa_ep_idx);
-	if (res != 0 && res != -GSI_STATUS_AGAIN &&
-		res != -GSI_STATUS_TIMED_OUT) {
-		IPAERR("failed to stop channel res = %d\n", res);
-	} else if (res == -GSI_STATUS_AGAIN) {
-		IPADBG("GSI stop channel failed retry cnt = %d\n",
-			retry_cnt);
-		retry_cnt++;
-		if (retry_cnt < GSI_STOP_MAX_RETRY_CNT)
-			goto fail_ring_ch;
-	} else {
-		IPADBG("GSI channel %ld STOP\n", ep->gsi_chan_hdl);
-	}
-	res = -EFAULT;
-fail_gsi_start:
-fail_ring_evt:
-	ipa3_disable_data_path(ipa_ep_idx);
-fail_enable_datapath:
-	IPA_ACTIVE_CLIENTS_DEC_EP(client);
-	return res;
-}
-
-int ipa3_disable_wigig_pipe_i(enum ipa_client_type client)
-{
-	int ipa_ep_idx, res;
-	struct ipa3_ep_context *ep;
-	struct ipahal_ep_cfg_ctrl_scnd ep_ctrl_scnd = { 0 };
-	struct ipa_ep_cfg_ctrl ep_cfg_ctrl;
-	bool disable_force_clear = false;
-	u32 source_pipe_bitmask = 0;
-	int retry_cnt = 0;
-
-	IPADBG("\n");
-
-	ipa_ep_idx = ipa_get_ep_mapping(client);
-	if (ipa_ep_idx == IPA_EP_NOT_ALLOCATED ||
-		ipa_ep_idx >= IPA3_MAX_NUM_PIPES) {
-		IPAERR("fail to get ep (%d) %d.\n",
-			client, ipa_ep_idx);
-		return -EFAULT;
-	}
-	if (ipa_ep_idx >= IPA3_MAX_NUM_PIPES) {
-		IPAERR("ep %d out of range.\n", ipa_ep_idx);
-		return -EFAULT;
-	}
-
-	ep = &ipa3_ctx->ep[ipa_ep_idx];
-
-	if (!ep->valid) {
-		IPAERR("Invalid EP\n");
-		return -EFAULT;
-	}
-
-	if (ep->gsi_offload_state !=
-		(IPA_WIGIG_CONNECTED | IPA_WIGIG_ENABLED)) {
-		IPAERR("WIGIG channel bad state 0x%X\n",
-			ep->gsi_offload_state);
-		return -EFAULT;
-	}
-
-	IPADBG("pipe %d\n", ipa_ep_idx);
-	source_pipe_bitmask = 1 << ipa_ep_idx;
-	res = ipa3_enable_force_clear(ipa_ep_idx,
-		false, source_pipe_bitmask);
-	if (res) {
-		/*
-		 * assuming here modem SSR, AP can remove
-		 * the delay in this case
-		 */
-		IPAERR("failed to force clear %d\n", res);
-		IPAERR("remove delay from SCND reg\n");
-		ep_ctrl_scnd.endp_delay = false;
-		ipahal_write_reg_n_fields(
-			IPA_ENDP_INIT_CTRL_SCND_n, ipa_ep_idx,
-			&ep_ctrl_scnd);
-	} else {
-		disable_force_clear = true;
-	}
-retry_gsi_stop:
-	res = ipa3_stop_gsi_channel(ipa_ep_idx);
-	if (res != 0 && res != -GSI_STATUS_AGAIN &&
-		res != -GSI_STATUS_TIMED_OUT) {
-		IPAERR("failed to stop channel res = %d\n", res);
-		goto fail_stop_channel;
-	} else if (res == -GSI_STATUS_AGAIN) {
-		IPADBG("GSI stop channel failed retry cnt = %d\n",
-			retry_cnt);
-		retry_cnt++;
-		if (retry_cnt >= GSI_STOP_MAX_RETRY_CNT)
-			goto fail_stop_channel;
-		goto retry_gsi_stop;
-	} else {
-		IPADBG("GSI channel %ld STOP\n", ep->gsi_chan_hdl);
-	}
-
-	res = ipa3_reset_gsi_channel(ipa_ep_idx);
-	if (res != GSI_STATUS_SUCCESS) {
-		IPAERR("Failed to reset chan: %d.\n", res);
-		goto fail_stop_channel;
-	}
-
-	if (disable_force_clear)
-		ipa3_disable_force_clear(ipa_ep_idx);
-
-	res = ipa3_disable_data_path(ipa_ep_idx);
-	if (res) {
-		WARN_ON(1);
-		return res;
-	}
-
-	/* Set the delay after disabling IPA Producer pipe */
-	if (IPA_CLIENT_IS_PROD(ep->client)) {
-		memset(&ep_cfg_ctrl, 0, sizeof(struct ipa_ep_cfg_ctrl));
-		ep_cfg_ctrl.ipa_ep_delay = true;
-		ipa3_cfg_ep_ctrl(ipa_ep_idx, &ep_cfg_ctrl);
-	}
-
-	ep->gsi_offload_state &= ~IPA_WIGIG_ENABLED;
-
-	IPA_ACTIVE_CLIENTS_DEC_EP(ipa3_get_client_mapping(ipa_ep_idx));
-	IPADBG("exit\n");
-	return 0;
-
-fail_stop_channel:
-	ipa_assert();
-	return res;
-}
-
-#ifndef CONFIG_DEBUG_FS
-int ipa3_wigig_init_debugfs_i(struct dentry *parent) { return 0; }
-#else
-int ipa3_wigig_init_debugfs_i(struct dentry *parent)
-{
-	const mode_t read_write_mode = 0664;
-	struct dentry *file = NULL;
-	struct dentry *dent;
-
-	dent = debugfs_create_dir("ipa_wigig", parent);
-	if (IS_ERR_OR_NULL(dent)) {
-		IPAERR("fail to create folder in debug_fs\n");
-		return -EFAULT;
-	}
-
-	wigig_dent = dent;
-
-	file = debugfs_create_u8("modc", read_write_mode, dent,
-		&int_modc);
-	if (IS_ERR_OR_NULL(file)) {
-		IPAERR("fail to create file modc\n");
-		goto fail;
-	}
-
-	file = debugfs_create_u16("modt", read_write_mode, dent,
-		&int_modt);
-	if (IS_ERR_OR_NULL(file)) {
-		IPAERR("fail to create file modt\n");
-		goto fail;
-	}
-
-	file = debugfs_create_u8("rx_mod_th", read_write_mode, dent,
-		&rx_hwtail_mod_threshold);
-	if (IS_ERR_OR_NULL(file)) {
-		IPAERR("fail to create file rx_mod_th\n");
-		goto fail;
-	}
-
-	file = debugfs_create_u8("tx_mod_th", read_write_mode, dent,
-		&tx_hwtail_mod_threshold);
-	if (IS_ERR_OR_NULL(file)) {
-		IPAERR("fail to create file tx_mod_th\n");
-		goto fail;
-	}
-
-	return 0;
-fail:
-	debugfs_remove_recursive(dent);
-	wigig_dent = NULL;
-	return -EFAULT;
-}
-#endif

ipa/ipa_v3/ipahal/Makefile

@@ -1,5 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-
-obj-$(CONFIG_IPA3) += ipa_hal.o
-
-ipa_hal-y := ipahal.o ipahal_reg.o ipahal_fltrt.o ipahal_hw_stats.o ipahal_nat.o

ipa/ipa_v3/ipahal/ipahal.c

@@ -1,1616 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2016-2019, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/debugfs.h>
-#include "ipahal.h"
-#include "ipahal_i.h"
-#include "ipahal_reg_i.h"
-#include "ipahal_fltrt_i.h"
-#include "ipahal_hw_stats_i.h"
-#include "ipahal_nat_i.h"
-
-struct ipahal_context *ipahal_ctx;
-
-static const char *ipahal_imm_cmd_name_to_str[IPA_IMM_CMD_MAX] = {
-	__stringify(IPA_IMM_CMD_IP_V4_FILTER_INIT),
-	__stringify(IPA_IMM_CMD_IP_V6_FILTER_INIT),
-	__stringify(IPA_IMM_CMD_IP_V4_NAT_INIT),
-	__stringify(IPA_IMM_CMD_IP_V4_ROUTING_INIT),
-	__stringify(IPA_IMM_CMD_IP_V6_ROUTING_INIT),
-	__stringify(IPA_IMM_CMD_HDR_INIT_LOCAL),
-	__stringify(IPA_IMM_CMD_HDR_INIT_SYSTEM),
-	__stringify(IPA_IMM_CMD_REGISTER_WRITE),
-	__stringify(IPA_IMM_CMD_NAT_DMA),
-	__stringify(IPA_IMM_CMD_IP_PACKET_INIT),
-	__stringify(IPA_IMM_CMD_DMA_SHARED_MEM),
-	__stringify(IPA_IMM_CMD_IP_PACKET_TAG_STATUS),
-	__stringify(IPA_IMM_CMD_DMA_TASK_32B_ADDR),
-	__stringify(IPA_IMM_CMD_TABLE_DMA),
-	__stringify(IPA_IMM_CMD_IP_V6_CT_INIT)
-};
-
-static const char *ipahal_pkt_status_exception_to_str
-	[IPAHAL_PKT_STATUS_EXCEPTION_MAX] = {
-	__stringify(IPAHAL_PKT_STATUS_EXCEPTION_NONE),
-	__stringify(IPAHAL_PKT_STATUS_EXCEPTION_DEAGGR),
-	__stringify(IPAHAL_PKT_STATUS_EXCEPTION_IPTYPE),
-	__stringify(IPAHAL_PKT_STATUS_EXCEPTION_PACKET_LENGTH),
-	__stringify(IPAHAL_PKT_STATUS_EXCEPTION_PACKET_THRESHOLD),
-	__stringify(IPAHAL_PKT_STATUS_EXCEPTION_FRAG_RULE_MISS),
-	__stringify(IPAHAL_PKT_STATUS_EXCEPTION_SW_FILT),
-	__stringify(IPAHAL_PKT_STATUS_EXCEPTION_NAT),
-	__stringify(IPAHAL_PKT_STATUS_EXCEPTION_IPV6CT),
-};
-
-static u16 ipahal_imm_cmd_get_opcode(enum ipahal_imm_cmd_name cmd);
-
-
-static struct ipahal_imm_cmd_pyld *ipa_imm_cmd_construct_dma_task_32b_addr(
-	enum ipahal_imm_cmd_name cmd, const void *params, bool is_atomic_ctx)
-{
-	struct ipahal_imm_cmd_pyld *pyld;
-	struct ipa_imm_cmd_hw_dma_task_32b_addr *data;
-	struct ipahal_imm_cmd_dma_task_32b_addr *dma_params =
-		(struct ipahal_imm_cmd_dma_task_32b_addr *)params;
-
-	pyld = IPAHAL_MEM_ALLOC(sizeof(*pyld) + sizeof(*data), is_atomic_ctx);
-	if (unlikely(!pyld))
-		return pyld;
-
-	/* Currently supports only one packet */
-	pyld->opcode = ipahal_imm_cmd_get_opcode(cmd) + (1 << 8);
-	pyld->len = sizeof(*data);
-	data = (struct ipa_imm_cmd_hw_dma_task_32b_addr *)pyld->data;
-
-	if (unlikely(dma_params->size1 & ~0xFFFF)) {
-		WARN(1, "Size1 is bigger than 16bit width 0x%x\n",
-			dma_params->size1);
-	}
-	if (unlikely(dma_params->packet_size & ~0xFFFF)) {
-		WARN(1, "Pkt size is bigger than 16bit width 0x%x\n",
-			dma_params->packet_size);
-	}
-	data->cmplt = dma_params->cmplt ? 1 : 0;
-	data->eof = dma_params->eof ? 1 : 0;
-	data->flsh = dma_params->flsh ? 1 : 0;
-	data->lock = dma_params->lock ? 1 : 0;
-	data->unlock = dma_params->unlock ? 1 : 0;
-	data->size1 = dma_params->size1;
-	data->addr1 = dma_params->addr1;
-	data->packet_size = dma_params->packet_size;
-
-	return pyld;
-}
-
-static struct ipahal_imm_cmd_pyld *ipa_imm_cmd_construct_ip_packet_tag_status(
-	enum ipahal_imm_cmd_name cmd, const void *params, bool is_atomic_ctx)
-{
-	struct ipahal_imm_cmd_pyld *pyld;
-	struct ipa_imm_cmd_hw_ip_packet_tag_status *data;
-	struct ipahal_imm_cmd_ip_packet_tag_status *tag_params =
-		(struct ipahal_imm_cmd_ip_packet_tag_status *)params;
-
-	pyld = IPAHAL_MEM_ALLOC(sizeof(*pyld) + sizeof(*data), is_atomic_ctx);
-	if (unlikely(!pyld)) {
-		IPAHAL_ERR("kzalloc err\n");
-		return pyld;
-	}
-	pyld->opcode = ipahal_imm_cmd_get_opcode(cmd);
-	pyld->len = sizeof(*data);
-	data = (struct ipa_imm_cmd_hw_ip_packet_tag_status *)pyld->data;
-
-	if (unlikely(tag_params->tag & ~0xFFFFFFFFFFFF)) {
-		IPAHAL_ERR("tag is bigger than 48bit width 0x%llx\n",
-			tag_params->tag);
-		WARN_ON(1);
-	}
-	data->tag = tag_params->tag;
-
-	return pyld;
-}
-
-static struct ipahal_imm_cmd_pyld *ipa_imm_cmd_construct_dma_shared_mem(
-	enum ipahal_imm_cmd_name cmd, const void *params, bool is_atomic_ctx)
-{
-	struct ipahal_imm_cmd_pyld *pyld;
-	struct ipa_imm_cmd_hw_dma_shared_mem *data;
-	struct ipahal_imm_cmd_dma_shared_mem *mem_params =
-		(struct ipahal_imm_cmd_dma_shared_mem *)params;
-
-	pyld = IPAHAL_MEM_ALLOC(sizeof(*pyld) + sizeof(*data), is_atomic_ctx);
-	if (unlikely(!pyld))
-		return pyld;
-
-	pyld->opcode = ipahal_imm_cmd_get_opcode(cmd);
-	pyld->len = sizeof(*data);
-	data = (struct ipa_imm_cmd_hw_dma_shared_mem *)pyld->data;
-
-	if (unlikely(mem_params->size & ~0xFFFF)) {
-		WARN(1, "Size is bigger than 16bit width 0x%x\n",
-			mem_params->size);
-	}
-	if (unlikely(mem_params->local_addr & ~0xFFFF)) {
-		WARN(1, "Local addr is bigger than 16bit width 0x%x\n",
-			mem_params->local_addr);
-	}
-	data->direction = mem_params->is_read ? 1 : 0;
-	data->size = mem_params->size;
-	data->local_addr = mem_params->local_addr;
-	data->system_addr = mem_params->system_addr;
-	data->skip_pipeline_clear = mem_params->skip_pipeline_clear ? 1 : 0;
-	switch (mem_params->pipeline_clear_options) {
-	case IPAHAL_HPS_CLEAR:
-		data->pipeline_clear_options = 0;
-		break;
-	case IPAHAL_SRC_GRP_CLEAR:
-		data->pipeline_clear_options = 1;
-		break;
-	case IPAHAL_FULL_PIPELINE_CLEAR:
-		data->pipeline_clear_options = 2;
-		break;
-	default:
-		IPAHAL_ERR("unsupported pipline clear option %d\n",
-			mem_params->pipeline_clear_options);
-		WARN_ON(1);
-	}
-
-	return pyld;
-}
-
-static struct ipahal_imm_cmd_pyld *ipa_imm_cmd_construct_dma_shared_mem_v_4_0(
-	enum ipahal_imm_cmd_name cmd, const void *params, bool is_atomic_ctx)
-{
-	struct ipahal_imm_cmd_pyld *pyld;
-	struct ipa_imm_cmd_hw_dma_shared_mem_v_4_0 *data;
-	struct ipahal_imm_cmd_dma_shared_mem *mem_params =
-		(struct ipahal_imm_cmd_dma_shared_mem *)params;
-
-	if (unlikely(mem_params->size & ~0xFFFF)) {
-		IPAHAL_ERR("Size is bigger than 16bit width 0x%x\n",
-			mem_params->size);
-		WARN_ON(1);
-		return NULL;
-	}
-	if (unlikely(mem_params->local_addr & ~0xFFFF)) {
-		IPAHAL_ERR("Local addr is bigger than 16bit width 0x%x\n",
-			mem_params->local_addr);
-		WARN_ON(1);
-		return NULL;
-	}
-
-	pyld = IPAHAL_MEM_ALLOC(sizeof(*pyld) + sizeof(*data), is_atomic_ctx);
-	if (unlikely(!pyld)) {
-		WARN_ON(1);
-		return pyld;
-	}
-
-	pyld->opcode = ipahal_imm_cmd_get_opcode(cmd);
-	pyld->len = sizeof(*data);
-	data = (struct ipa_imm_cmd_hw_dma_shared_mem_v_4_0 *)pyld->data;
-
-	data->direction = mem_params->is_read ? 1 : 0;
-	data->clear_after_read = mem_params->clear_after_read;
-	data->size = mem_params->size;
-	data->local_addr = mem_params->local_addr;
-	data->system_addr = mem_params->system_addr;
-	pyld->opcode |= (mem_params->skip_pipeline_clear ? 1 : 0) << 8;
-	switch (mem_params->pipeline_clear_options) {
-	case IPAHAL_HPS_CLEAR:
-		break;
-	case IPAHAL_SRC_GRP_CLEAR:
-		pyld->opcode |= (1 << 9);
-		break;
-	case IPAHAL_FULL_PIPELINE_CLEAR:
-		pyld->opcode |= (2 << 9);
-		break;
-	default:
-		IPAHAL_ERR("unsupported pipline clear option %d\n",
-			mem_params->pipeline_clear_options);
-		WARN_ON(1);
-	}
-
-	return pyld;
-}
-
-static struct ipahal_imm_cmd_pyld *ipa_imm_cmd_construct_register_write(
-	enum ipahal_imm_cmd_name cmd, const void *params, bool is_atomic_ctx)
-{
-	struct ipahal_imm_cmd_pyld *pyld;
-	struct ipa_imm_cmd_hw_register_write *data;
-	struct ipahal_imm_cmd_register_write *regwrt_params =
-		(struct ipahal_imm_cmd_register_write *)params;
-
-	pyld = IPAHAL_MEM_ALLOC(sizeof(*pyld) + sizeof(*data), is_atomic_ctx);
-	if (unlikely(!pyld)) {
-		IPAHAL_ERR("kzalloc err\n");
-		return pyld;
-	}
-	pyld->opcode = ipahal_imm_cmd_get_opcode(cmd);
-	pyld->len = sizeof(*data);
-	data = (struct ipa_imm_cmd_hw_register_write *)pyld->data;
-
-	if (unlikely(regwrt_params->offset & ~0xFFFF)) {
-		IPAHAL_ERR("Offset is bigger than 16bit width 0x%x\n",
-			regwrt_params->offset);
-		WARN_ON(1);
-	}
-	data->offset = regwrt_params->offset;
-	data->value = regwrt_params->value;
-	data->value_mask = regwrt_params->value_mask;
-
-	data->skip_pipeline_clear = regwrt_params->skip_pipeline_clear ? 1 : 0;
-	switch (regwrt_params->pipeline_clear_options) {
-	case IPAHAL_HPS_CLEAR:
-		data->pipeline_clear_options = 0;
-		break;
-	case IPAHAL_SRC_GRP_CLEAR:
-		data->pipeline_clear_options = 1;
-		break;
-	case IPAHAL_FULL_PIPELINE_CLEAR:
-		data->pipeline_clear_options = 2;
-		break;
-	default:
-		IPAHAL_ERR("unsupported pipline clear option %d\n",
-			regwrt_params->pipeline_clear_options);
-		WARN_ON(1);
-	}
-
-	return pyld;
-}
-
-static struct ipahal_imm_cmd_pyld *ipa_imm_cmd_construct_register_write_v_4_0(
-	enum ipahal_imm_cmd_name cmd, const void *params, bool is_atomic_ctx)
-{
-	struct ipahal_imm_cmd_pyld *pyld;
-	struct ipa_imm_cmd_hw_register_write_v_4_0 *data;
-	struct ipahal_imm_cmd_register_write *regwrt_params =
-		(struct ipahal_imm_cmd_register_write *)params;
-
-	if (unlikely(regwrt_params->offset & ~0xFFFF)) {
-		IPAHAL_ERR("Offset is bigger than 16bit width 0x%x\n",
-			regwrt_params->offset);
-		WARN_ON(1);
-		return NULL;
-	}
-
-	pyld = IPAHAL_MEM_ALLOC(sizeof(*pyld) + sizeof(*data), is_atomic_ctx);
-	if (unlikely(!pyld)) {
-		WARN_ON(1);
-		return pyld;
-	}
-	pyld->opcode = ipahal_imm_cmd_get_opcode(cmd);
-	pyld->len = sizeof(*data);
-	data = (struct ipa_imm_cmd_hw_register_write_v_4_0 *)pyld->data;
-
-	data->offset = regwrt_params->offset;
-	data->offset_high = regwrt_params->offset >> 16;
-	data->value = regwrt_params->value;
-	data->value_mask = regwrt_params->value_mask;
-
-	pyld->opcode |= (regwrt_params->skip_pipeline_clear ? 1 : 0) << 8;
-	switch (regwrt_params->pipeline_clear_options) {
-	case IPAHAL_HPS_CLEAR:
-		break;
-	case IPAHAL_SRC_GRP_CLEAR:
-		pyld->opcode |= (1 << 9);
-		break;
-	case IPAHAL_FULL_PIPELINE_CLEAR:
-		pyld->opcode |= (2 << 9);
-		break;
-	default:
-		IPAHAL_ERR("unsupported pipline clear option %d\n",
-			regwrt_params->pipeline_clear_options);
-		WARN_ON(1);
-	}
-
-	return pyld;
-}
-
-static struct ipahal_imm_cmd_pyld *ipa_imm_cmd_construct_ip_packet_init(
-	enum ipahal_imm_cmd_name cmd, const void *params, bool is_atomic_ctx)
-{
-	struct ipahal_imm_cmd_pyld *pyld;
-	struct ipa_imm_cmd_hw_ip_packet_init *data;
-	struct ipahal_imm_cmd_ip_packet_init *pktinit_params =
-		(struct ipahal_imm_cmd_ip_packet_init *)params;
-
-	pyld = IPAHAL_MEM_ALLOC(sizeof(*pyld) + sizeof(*data), is_atomic_ctx);
-	if (unlikely(!pyld)) {
-		IPAHAL_ERR("kzalloc err\n");
-		return pyld;
-	}
-	pyld->opcode = ipahal_imm_cmd_get_opcode(cmd);
-	pyld->len = sizeof(*data);
-	data = (struct ipa_imm_cmd_hw_ip_packet_init *)pyld->data;
-
-	if (unlikely(pktinit_params->destination_pipe_index & ~0x1F)) {
-		IPAHAL_ERR("Dst pipe idx is bigger than 5bit width 0x%x\n",
-			pktinit_params->destination_pipe_index);
-		WARN_ON(1);
-	}
-	data->destination_pipe_index = pktinit_params->destination_pipe_index;
-
-	return pyld;
-}
-
-static struct ipahal_imm_cmd_pyld *ipa_imm_cmd_construct_nat_dma(
-	enum ipahal_imm_cmd_name cmd, const void *params, bool is_atomic_ctx)
-{
-	struct ipahal_imm_cmd_pyld *pyld;
-	struct ipa_imm_cmd_hw_nat_dma *data;
-	struct ipahal_imm_cmd_table_dma *nat_params =
-		(struct ipahal_imm_cmd_table_dma *)params;
-
-	pyld = IPAHAL_MEM_ALLOC(sizeof(*pyld) + sizeof(*data), is_atomic_ctx);
-	if (unlikely(!pyld)) {
-		IPAHAL_ERR("kzalloc err\n");
-		return pyld;
-	}
-	pyld->opcode = ipahal_imm_cmd_get_opcode(cmd);
-	pyld->len = sizeof(*data);
-	data = (struct ipa_imm_cmd_hw_nat_dma *)pyld->data;
-
-	data->table_index = nat_params->table_index;
-	data->base_addr = nat_params->base_addr;
-	data->offset = nat_params->offset;
-	data->data = nat_params->data;
-
-	return pyld;
-}
-
-static struct ipahal_imm_cmd_pyld *ipa_imm_cmd_construct_table_dma_ipav4(
-	enum ipahal_imm_cmd_name cmd, const void *params, bool is_atomic_ctx)
-{
-	struct ipahal_imm_cmd_pyld *pyld;
-	struct ipa_imm_cmd_hw_table_dma_ipav4 *data;
-	struct ipahal_imm_cmd_table_dma *nat_params =
-		(struct ipahal_imm_cmd_table_dma *)params;
-
-	pyld = IPAHAL_MEM_ALLOC(sizeof(*pyld) + sizeof(*data), is_atomic_ctx);
-	if (unlikely(!pyld)) {
-		IPAHAL_ERR("kzalloc err\n");
-		return pyld;
-	}
-	pyld->opcode = ipahal_imm_cmd_get_opcode(cmd);
-	pyld->len = sizeof(*data);
-	data = (struct ipa_imm_cmd_hw_table_dma_ipav4 *)pyld->data;
-
-	data->table_index = nat_params->table_index;
-	data->base_addr = nat_params->base_addr;
-	data->offset = nat_params->offset;
-	data->data = nat_params->data;
-
-	return pyld;
-}
-
-static struct ipahal_imm_cmd_pyld *ipa_imm_cmd_construct_hdr_init_system(
-	enum ipahal_imm_cmd_name cmd, const void *params, bool is_atomic_ctx)
-{
-	struct ipahal_imm_cmd_pyld *pyld;
-	struct ipa_imm_cmd_hw_hdr_init_system *data;
-	struct ipahal_imm_cmd_hdr_init_system *syshdr_params =
-		(struct ipahal_imm_cmd_hdr_init_system *)params;
-
-	pyld = IPAHAL_MEM_ALLOC(sizeof(*pyld) + sizeof(*data), is_atomic_ctx);
-	if (unlikely(!pyld)) {
-		IPAHAL_ERR("kzalloc err\n");
-		return pyld;
-	}
-	pyld->opcode = ipahal_imm_cmd_get_opcode(cmd);
-	pyld->len = sizeof(*data);
-	data = (struct ipa_imm_cmd_hw_hdr_init_system *)pyld->data;
-
-	data->hdr_table_addr = syshdr_params->hdr_table_addr;
-
-	return pyld;
-}
-
-static struct ipahal_imm_cmd_pyld *ipa_imm_cmd_construct_hdr_init_local(
-	enum ipahal_imm_cmd_name cmd, const void *params, bool is_atomic_ctx)
-{
-	struct ipahal_imm_cmd_pyld *pyld;
-	struct ipa_imm_cmd_hw_hdr_init_local *data;
-	struct ipahal_imm_cmd_hdr_init_local *lclhdr_params =
-		(struct ipahal_imm_cmd_hdr_init_local *)params;
-
-	pyld = IPAHAL_MEM_ALLOC(sizeof(*pyld) + sizeof(*data), is_atomic_ctx);
-	if (unlikely(!pyld)) {
-		IPAHAL_ERR("kzalloc err\n");
-		return pyld;
-	}
-	pyld->opcode = ipahal_imm_cmd_get_opcode(cmd);
-	pyld->len = sizeof(*data);
-	data = (struct ipa_imm_cmd_hw_hdr_init_local *)pyld->data;
-
-	if (unlikely(lclhdr_params->size_hdr_table & ~0xFFF)) {
-		IPAHAL_ERR("Hdr tble size is bigger than 12bit width 0x%x\n",
-			lclhdr_params->size_hdr_table);
-		WARN_ON(1);
-	}
-	data->hdr_table_addr = lclhdr_params->hdr_table_addr;
-	data->size_hdr_table = lclhdr_params->size_hdr_table;
-	data->hdr_addr = lclhdr_params->hdr_addr;
-
-	return pyld;
-}
-
-static struct ipahal_imm_cmd_pyld *ipa_imm_cmd_construct_ip_v6_routing_init(
-	enum ipahal_imm_cmd_name cmd, const void *params, bool is_atomic_ctx)
-{
-	struct ipahal_imm_cmd_pyld *pyld;
-	struct ipa_imm_cmd_hw_ip_v6_routing_init *data;
-	struct ipahal_imm_cmd_ip_v6_routing_init *rt6_params =
-		(struct ipahal_imm_cmd_ip_v6_routing_init *)params;
-
-	pyld = IPAHAL_MEM_ALLOC(sizeof(*pyld) + sizeof(*data), is_atomic_ctx);
-	if (unlikely(!pyld)) {
-		IPAHAL_ERR("kzalloc err\n");
-		return pyld;
-	}
-	pyld->opcode = ipahal_imm_cmd_get_opcode(cmd);
-	pyld->len = sizeof(*data);
-	data = (struct ipa_imm_cmd_hw_ip_v6_routing_init *)pyld->data;
-
-	data->hash_rules_addr = rt6_params->hash_rules_addr;
-	data->hash_rules_size = rt6_params->hash_rules_size;
-	data->hash_local_addr = rt6_params->hash_local_addr;
-	data->nhash_rules_addr = rt6_params->nhash_rules_addr;
-	data->nhash_rules_size = rt6_params->nhash_rules_size;
-	data->nhash_local_addr = rt6_params->nhash_local_addr;
-
-	return pyld;
-}
-
-static struct ipahal_imm_cmd_pyld *ipa_imm_cmd_construct_ip_v4_routing_init(
-	enum ipahal_imm_cmd_name cmd, const void *params, bool is_atomic_ctx)
-{
-	struct ipahal_imm_cmd_pyld *pyld;
-	struct ipa_imm_cmd_hw_ip_v4_routing_init *data;
-	struct ipahal_imm_cmd_ip_v4_routing_init *rt4_params =
-		(struct ipahal_imm_cmd_ip_v4_routing_init *)params;
-
-	pyld = IPAHAL_MEM_ALLOC(sizeof(*pyld) + sizeof(*data), is_atomic_ctx);
-	if (unlikely(!pyld)) {
-		IPAHAL_ERR("kzalloc err\n");
-		return pyld;
-	}
-	pyld->opcode = ipahal_imm_cmd_get_opcode(cmd);
-	pyld->len = sizeof(*data);
-	data = (struct ipa_imm_cmd_hw_ip_v4_routing_init *)pyld->data;
-
-	data->hash_rules_addr = rt4_params->hash_rules_addr;
-	data->hash_rules_size = rt4_params->hash_rules_size;
-	data->hash_local_addr = rt4_params->hash_local_addr;
-	data->nhash_rules_addr = rt4_params->nhash_rules_addr;
-	data->nhash_rules_size = rt4_params->nhash_rules_size;
-	data->nhash_local_addr = rt4_params->nhash_local_addr;
-
-	return pyld;
-}
-
-static struct ipahal_imm_cmd_pyld *ipa_imm_cmd_construct_ip_v4_nat_init(
-	enum ipahal_imm_cmd_name cmd, const void *params, bool is_atomic_ctx)
-{
-	struct ipahal_imm_cmd_pyld *pyld;
-	struct ipa_imm_cmd_hw_ip_v4_nat_init *data;
-	struct ipahal_imm_cmd_ip_v4_nat_init *nat4_params =
-		(struct ipahal_imm_cmd_ip_v4_nat_init *)params;
-
-	pyld = IPAHAL_MEM_ALLOC(sizeof(*pyld) + sizeof(*data), is_atomic_ctx);
-	if (unlikely(!pyld)) {
-		IPAHAL_ERR("kzalloc err\n");
-		return pyld;
-	}
-	pyld->opcode = ipahal_imm_cmd_get_opcode(cmd);
-	pyld->len = sizeof(*data);
-	data = (struct ipa_imm_cmd_hw_ip_v4_nat_init *)pyld->data;
-
-	data->ipv4_rules_addr = nat4_params->table_init.base_table_addr;
-	data->ipv4_expansion_rules_addr =
-		nat4_params->table_init.expansion_table_addr;
-	data->index_table_addr = nat4_params->index_table_addr;
-	data->index_table_expansion_addr =
-		nat4_params->index_table_expansion_addr;
-	data->table_index = nat4_params->table_init.table_index;
-	data->ipv4_rules_addr_type =
-		nat4_params->table_init.base_table_addr_shared ? 1 : 0;
-	data->ipv4_expansion_rules_addr_type =
-		nat4_params->table_init.expansion_table_addr_shared ? 1 : 0;
-	data->index_table_addr_type =
-		nat4_params->index_table_addr_shared ? 1 : 0;
-	data->index_table_expansion_addr_type =
-		nat4_params->index_table_expansion_addr_shared ? 1 : 0;
-	data->size_base_tables = nat4_params->table_init.size_base_table;
-	data->size_expansion_tables =
-		nat4_params->table_init.size_expansion_table;
-	data->public_addr_info = nat4_params->public_addr_info;
-
-	return pyld;
-}
-
-static struct ipahal_imm_cmd_pyld *ipa_imm_cmd_construct_ip_v6_ct_init(
-	enum ipahal_imm_cmd_name cmd, const void *params, bool is_atomic_ctx)
-{
-	struct ipahal_imm_cmd_pyld *pyld;
-	struct ipa_imm_cmd_hw_ip_v6_ct_init *data;
-	struct ipahal_imm_cmd_ip_v6_ct_init *ipv6ct_params =
-		(struct ipahal_imm_cmd_ip_v6_ct_init *)params;
-
-	pyld = IPAHAL_MEM_ALLOC(sizeof(*pyld) + sizeof(*data), is_atomic_ctx);
-	if (unlikely(!pyld))
-		return pyld;
-	pyld->opcode = ipahal_imm_cmd_get_opcode(cmd);
-	pyld->len = sizeof(*data);
-	data = (struct ipa_imm_cmd_hw_ip_v6_ct_init *)pyld->data;
-
-	data->table_addr = ipv6ct_params->table_init.base_table_addr;
-	data->expansion_table_addr =
-		ipv6ct_params->table_init.expansion_table_addr;
-	data->table_index = ipv6ct_params->table_init.table_index;
-	data->table_addr_type =
-		ipv6ct_params->table_init.base_table_addr_shared ? 1 : 0;
-	data->expansion_table_addr_type =
-		ipv6ct_params->table_init.expansion_table_addr_shared ? 1 : 0;
-	data->size_base_table = ipv6ct_params->table_init.size_base_table;
-	data->size_expansion_table =
-		ipv6ct_params->table_init.size_expansion_table;
-
-	return pyld;
-}
-
-static struct ipahal_imm_cmd_pyld *ipa_imm_cmd_construct_ip_v6_filter_init(
-	enum ipahal_imm_cmd_name cmd, const void *params, bool is_atomic_ctx)
-{
-	struct ipahal_imm_cmd_pyld *pyld;
-	struct ipa_imm_cmd_hw_ip_v6_filter_init *data;
-	struct ipahal_imm_cmd_ip_v6_filter_init *flt6_params =
-		(struct ipahal_imm_cmd_ip_v6_filter_init *)params;
-
-	pyld = IPAHAL_MEM_ALLOC(sizeof(*pyld) + sizeof(*data), is_atomic_ctx);
-	if (unlikely(!pyld)) {
-		IPAHAL_ERR("kzalloc err\n");
-		return pyld;
-	}
-	pyld->opcode = ipahal_imm_cmd_get_opcode(cmd);
-	pyld->len = sizeof(*data);
-	data = (struct ipa_imm_cmd_hw_ip_v6_filter_init *)pyld->data;
-
-	data->hash_rules_addr = flt6_params->hash_rules_addr;
-	data->hash_rules_size = flt6_params->hash_rules_size;
-	data->hash_local_addr = flt6_params->hash_local_addr;
-	data->nhash_rules_addr = flt6_params->nhash_rules_addr;
-	data->nhash_rules_size = flt6_params->nhash_rules_size;
-	data->nhash_local_addr = flt6_params->nhash_local_addr;
-
-	return pyld;
-}
-
-static struct ipahal_imm_cmd_pyld *ipa_imm_cmd_construct_ip_v4_filter_init(
-	enum ipahal_imm_cmd_name cmd, const void *params, bool is_atomic_ctx)
-{
-	struct ipahal_imm_cmd_pyld *pyld;
-	struct ipa_imm_cmd_hw_ip_v4_filter_init *data;
-	struct ipahal_imm_cmd_ip_v4_filter_init *flt4_params =
-		(struct ipahal_imm_cmd_ip_v4_filter_init *)params;
-
-	pyld = IPAHAL_MEM_ALLOC(sizeof(*pyld) + sizeof(*data), is_atomic_ctx);
-	if (unlikely(!pyld)) {
-		IPAHAL_ERR("kzalloc err\n");
-		return pyld;
-	}
-	pyld->opcode = ipahal_imm_cmd_get_opcode(cmd);
-	pyld->len = sizeof(*data);
-	data = (struct ipa_imm_cmd_hw_ip_v4_filter_init *)pyld->data;
-
-	data->hash_rules_addr = flt4_params->hash_rules_addr;
-	data->hash_rules_size = flt4_params->hash_rules_size;
-	data->hash_local_addr = flt4_params->hash_local_addr;
-	data->nhash_rules_addr = flt4_params->nhash_rules_addr;
-	data->nhash_rules_size = flt4_params->nhash_rules_size;
-	data->nhash_local_addr = flt4_params->nhash_local_addr;
-
-	return pyld;
-}
-
-static struct ipahal_imm_cmd_pyld *ipa_imm_cmd_construct_dummy(
-	enum ipahal_imm_cmd_name cmd, const void *params, bool is_atomic_ctx)
-{
-	IPAHAL_ERR("no construct function for IMM_CMD=%s, IPA ver %d\n",
-		ipahal_imm_cmd_name_str(cmd), ipahal_ctx->hw_type);
-	WARN_ON(1);
-	return NULL;
-}
-
-/*
- * struct ipahal_imm_cmd_obj - immediate command H/W information for
- *  specific IPA version
- * @construct - CB to construct imm command payload from abstracted structure
- * @opcode - Immediate command OpCode
- */
-struct ipahal_imm_cmd_obj {
-	struct ipahal_imm_cmd_pyld *(*construct)(enum ipahal_imm_cmd_name cmd,
-		const void *params, bool is_atomic_ctx);
-	u16 opcode;
-};
-
-/*
- * This table contains the info regard each immediate command for IPAv3
- *  and later.
- * Information like: opcode and construct functions.
- * All the information on the IMM on IPAv3 are statically defined below.
- * If information is missing regard some IMM on some IPA version,
- *  the init function will fill it with the information from the previous
- *  IPA version.
- * Information is considered missing if all of the fields are 0
- * If opcode is -1, this means that the IMM is removed on the
- *  specific version
- */
-static struct ipahal_imm_cmd_obj
-		ipahal_imm_cmd_objs[IPA_HW_MAX][IPA_IMM_CMD_MAX] = {
-	/* IPAv3 */
-	[IPA_HW_v3_0][IPA_IMM_CMD_IP_V4_FILTER_INIT] = {
-		ipa_imm_cmd_construct_ip_v4_filter_init,
-		3},
-	[IPA_HW_v3_0][IPA_IMM_CMD_IP_V6_FILTER_INIT] = {
-		ipa_imm_cmd_construct_ip_v6_filter_init,
-		4},
-	[IPA_HW_v3_0][IPA_IMM_CMD_IP_V4_NAT_INIT] = {
-		ipa_imm_cmd_construct_ip_v4_nat_init,
-		5},
-	[IPA_HW_v3_0][IPA_IMM_CMD_IP_V4_ROUTING_INIT] = {
-		ipa_imm_cmd_construct_ip_v4_routing_init,
-		7},
-	[IPA_HW_v3_0][IPA_IMM_CMD_IP_V6_ROUTING_INIT] = {
-		ipa_imm_cmd_construct_ip_v6_routing_init,
-		8},
-	[IPA_HW_v3_0][IPA_IMM_CMD_HDR_INIT_LOCAL] = {
-		ipa_imm_cmd_construct_hdr_init_local,
-		9},
-	[IPA_HW_v3_0][IPA_IMM_CMD_HDR_INIT_SYSTEM] = {
-		ipa_imm_cmd_construct_hdr_init_system,
-		10},
-	[IPA_HW_v3_0][IPA_IMM_CMD_REGISTER_WRITE] = {
-		ipa_imm_cmd_construct_register_write,
-		12},
-	[IPA_HW_v3_0][IPA_IMM_CMD_NAT_DMA] = {
-		ipa_imm_cmd_construct_nat_dma,
-		14},
-	[IPA_HW_v3_0][IPA_IMM_CMD_IP_PACKET_INIT] = {
-		ipa_imm_cmd_construct_ip_packet_init,
-		16},
-	[IPA_HW_v3_0][IPA_IMM_CMD_DMA_TASK_32B_ADDR] = {
-		ipa_imm_cmd_construct_dma_task_32b_addr,
-		17},
-	[IPA_HW_v3_0][IPA_IMM_CMD_DMA_SHARED_MEM] = {
-		ipa_imm_cmd_construct_dma_shared_mem,
-		19},
-	[IPA_HW_v3_0][IPA_IMM_CMD_IP_PACKET_TAG_STATUS] = {
-		ipa_imm_cmd_construct_ip_packet_tag_status,
-		20},
-
-	/* IPAv4 */
-	[IPA_HW_v4_0][IPA_IMM_CMD_REGISTER_WRITE] = {
-		ipa_imm_cmd_construct_register_write_v_4_0,
-		12},
-	/* NAT_DMA was renamed to TABLE_DMA for IPAv4 */
-	[IPA_HW_v4_0][IPA_IMM_CMD_NAT_DMA] = {
-		ipa_imm_cmd_construct_dummy,
-		-1},
-	[IPA_HW_v4_0][IPA_IMM_CMD_TABLE_DMA] = {
-		ipa_imm_cmd_construct_table_dma_ipav4,
-		14},
-	[IPA_HW_v4_0][IPA_IMM_CMD_DMA_SHARED_MEM] = {
-		ipa_imm_cmd_construct_dma_shared_mem_v_4_0,
-		19},
-	[IPA_HW_v4_0][IPA_IMM_CMD_IP_V6_CT_INIT] = {
-		ipa_imm_cmd_construct_ip_v6_ct_init,
-		23}
-};
-
-/*
- * ipahal_imm_cmd_init() - Build the Immediate command information table
- *  See ipahal_imm_cmd_objs[][] comments
- */
-static int ipahal_imm_cmd_init(enum ipa_hw_type ipa_hw_type)
-{
-	int i;
-	int j;
-	struct ipahal_imm_cmd_obj zero_obj;
-
-	IPAHAL_DBG_LOW("Entry - HW_TYPE=%d\n", ipa_hw_type);
-
-	if ((ipa_hw_type < 0) || (ipa_hw_type >= IPA_HW_MAX)) {
-		IPAHAL_ERR("invalid IPA HW type (%d)\n", ipa_hw_type);
-		return -EINVAL;
-	}
-
-	memset(&zero_obj, 0, sizeof(zero_obj));
-	for (i = IPA_HW_v3_0 ; i < ipa_hw_type ; i++) {
-		for (j = 0; j < IPA_IMM_CMD_MAX ; j++) {
-			if (!memcmp(&ipahal_imm_cmd_objs[i+1][j], &zero_obj,
-				sizeof(struct ipahal_imm_cmd_obj))) {
-				memcpy(&ipahal_imm_cmd_objs[i+1][j],
-					&ipahal_imm_cmd_objs[i][j],
-					sizeof(struct ipahal_imm_cmd_obj));
-			} else {
-				/*
-				 * explicitly overridden immediate command.
-				 * Check validity
-				 */
-				if (!ipahal_imm_cmd_objs[i+1][j].opcode) {
-					IPAHAL_ERR(
-					  "imm_cmd=%s with zero opcode ipa_ver=%d\n",
-					  ipahal_imm_cmd_name_str(j), i+1);
-					WARN_ON(1);
-				}
-				if (!ipahal_imm_cmd_objs[i+1][j].construct) {
-					IPAHAL_ERR(
-					  "imm_cmd=%s with NULL construct func ipa_ver=%d\n",
-					  ipahal_imm_cmd_name_str(j), i+1);
-					WARN_ON(1);
-				}
-			}
-		}
-	}
-
-	return 0;
-}
-
-/*
- * ipahal_imm_cmd_name_str() - returns string that represent the imm cmd
- * @cmd_name: [in] Immediate command name
- */
-const char *ipahal_imm_cmd_name_str(enum ipahal_imm_cmd_name cmd_name)
-{
-	if (cmd_name < 0 || cmd_name >= IPA_IMM_CMD_MAX) {
-		IPAHAL_ERR("requested name of invalid imm_cmd=%d\n", cmd_name);
-		return "Invalid IMM_CMD";
-	}
-
-	return ipahal_imm_cmd_name_to_str[cmd_name];
-}
-
-/*
- * ipahal_imm_cmd_get_opcode() - Get the fixed opcode of the immediate command
- */
-static u16 ipahal_imm_cmd_get_opcode(enum ipahal_imm_cmd_name cmd)
-{
-	u32 opcode;
-
-	if (cmd >= IPA_IMM_CMD_MAX) {
-		IPAHAL_ERR("Invalid immediate command imm_cmd=%u\n", cmd);
-		ipa_assert();
-		return -EFAULT;
-	}
-
-	IPAHAL_DBG_LOW("Get opcode of IMM_CMD=%s\n",
-		ipahal_imm_cmd_name_str(cmd));
-	opcode = ipahal_imm_cmd_objs[ipahal_ctx->hw_type][cmd].opcode;
-	if (opcode == -1) {
-		IPAHAL_ERR("Try to get opcode of obsolete IMM_CMD=%s\n",
-			ipahal_imm_cmd_name_str(cmd));
-		ipa_assert();
-		return -EFAULT;
-	}
-
-	return opcode;
-}
-
-/*
- * ipahal_construct_imm_cmd() - Construct immdiate command
- * This function builds imm cmd bulk that can be be sent to IPA
- * The command will be allocated dynamically.
- * After done using it, call ipahal_destroy_imm_cmd() to release it
- */
-struct ipahal_imm_cmd_pyld *ipahal_construct_imm_cmd(
-	enum ipahal_imm_cmd_name cmd, const void *params, bool is_atomic_ctx)
-{
-	if (!params) {
-		IPAHAL_ERR("Input error: params=%pK\n", params);
-		ipa_assert();
-		return NULL;
-	}
-
-	if (cmd >= IPA_IMM_CMD_MAX) {
-		IPAHAL_ERR("Invalid immediate command %u\n", cmd);
-		return NULL;
-	}
-
-	IPAHAL_DBG_LOW("construct IMM_CMD:%s\n", ipahal_imm_cmd_name_str(cmd));
-	return ipahal_imm_cmd_objs[ipahal_ctx->hw_type][cmd].construct(
-		cmd, params, is_atomic_ctx);
-}
-
-/*
- * ipahal_construct_nop_imm_cmd() - Construct immediate comamnd for NO-Op
- * Core driver may want functionality to inject NOP commands to IPA
- *  to ensure e.g., PIPLINE clear before someother operation.
- * The functionality given by this function can be reached by
- *  ipahal_construct_imm_cmd(). This function is helper to the core driver
- *  to reach this NOP functionlity easily.
- * @skip_pipline_clear: if to skip pipeline clear waiting (don't wait)
- * @pipline_clr_opt: options for pipeline clear waiting
- * @is_atomic_ctx: is called in atomic context or can sleep?
- */
-struct ipahal_imm_cmd_pyld *ipahal_construct_nop_imm_cmd(
-	bool skip_pipline_clear,
-	enum ipahal_pipeline_clear_option pipline_clr_opt,
-	bool is_atomic_ctx)
-{
-	struct ipahal_imm_cmd_register_write cmd;
-	struct ipahal_imm_cmd_pyld *cmd_pyld;
-
-	memset(&cmd, 0, sizeof(cmd));
-	cmd.skip_pipeline_clear = skip_pipline_clear;
-	cmd.pipeline_clear_options = pipline_clr_opt;
-	cmd.value_mask = 0x0;
-
-	cmd_pyld = ipahal_construct_imm_cmd(IPA_IMM_CMD_REGISTER_WRITE,
-		&cmd, is_atomic_ctx);
-
-	if (!cmd_pyld)
-		IPAHAL_ERR("failed to construct register_write imm cmd\n");
-
-	return cmd_pyld;
-}
-
-
-/* IPA Packet Status Logic */
-
-#define IPA_PKT_STATUS_SET_MSK(__hw_bit_msk, __shft) \
-	(status->status_mask |= \
-		((hw_status->status_mask & (__hw_bit_msk) ? 1 : 0) << (__shft)))
-
-static void ipa_pkt_status_parse(
-	const void *unparsed_status, struct ipahal_pkt_status *status)
-{
-	enum ipahal_pkt_status_opcode opcode = 0;
-	enum ipahal_pkt_status_exception exception_type = 0;
-	bool is_ipv6;
-
-	struct ipa_pkt_status_hw *hw_status =
-		(struct ipa_pkt_status_hw *)unparsed_status;
-
-	is_ipv6 = (hw_status->status_mask & 0x80) ? false : true;
-
-	status->pkt_len = hw_status->pkt_len;
-	status->endp_src_idx = hw_status->endp_src_idx;
-	status->endp_dest_idx = hw_status->endp_dest_idx;
-	status->metadata = hw_status->metadata;
-	status->flt_local = hw_status->flt_local;
-	status->flt_hash = hw_status->flt_hash;
-	status->flt_global = hw_status->flt_hash;
-	status->flt_ret_hdr = hw_status->flt_ret_hdr;
-	status->flt_miss = (hw_status->rt_rule_id ==
-		IPAHAL_PKT_STATUS_FLTRT_RULE_MISS_ID);
-	status->flt_rule_id = hw_status->flt_rule_id;
-	status->rt_local = hw_status->rt_local;
-	status->rt_hash = hw_status->rt_hash;
-	status->ucp = hw_status->ucp;
-	status->rt_tbl_idx = hw_status->rt_tbl_idx;
-	status->rt_miss = (hw_status->rt_rule_id ==
-		IPAHAL_PKT_STATUS_FLTRT_RULE_MISS_ID);
-	status->rt_rule_id = hw_status->rt_rule_id;
-	status->nat_hit = hw_status->nat_hit;
-	status->nat_entry_idx = hw_status->nat_entry_idx;
-	status->tag_info = hw_status->tag_info;
-	status->seq_num = hw_status->seq_num;
-	status->time_of_day_ctr = hw_status->time_of_day_ctr;
-	status->hdr_local = hw_status->hdr_local;
-	status->hdr_offset = hw_status->hdr_offset;
-	status->frag_hit = hw_status->frag_hit;
-	status->frag_rule = hw_status->frag_rule;
-
-	switch (hw_status->status_opcode) {
-	case 0x1:
-		opcode = IPAHAL_PKT_STATUS_OPCODE_PACKET;
-		break;
-	case 0x2:
-		opcode = IPAHAL_PKT_STATUS_OPCODE_NEW_FRAG_RULE;
-		break;
-	case 0x4:
-		opcode = IPAHAL_PKT_STATUS_OPCODE_DROPPED_PACKET;
-		break;
-	case 0x8:
-		opcode = IPAHAL_PKT_STATUS_OPCODE_SUSPENDED_PACKET;
-		break;
-	case 0x10:
-		opcode = IPAHAL_PKT_STATUS_OPCODE_LOG;
-		break;
-	case 0x20:
-		opcode = IPAHAL_PKT_STATUS_OPCODE_DCMP;
-		break;
-	case 0x40:
-		opcode = IPAHAL_PKT_STATUS_OPCODE_PACKET_2ND_PASS;
-		break;
-	default:
-		IPAHAL_ERR("unsupported Status Opcode 0x%x\n",
-			hw_status->status_opcode);
-		WARN_ON(1);
-	}
-	status->status_opcode = opcode;
-
-	switch (hw_status->nat_type) {
-	case 0:
-		status->nat_type = IPAHAL_PKT_STATUS_NAT_NONE;
-		break;
-	case 1:
-		status->nat_type = IPAHAL_PKT_STATUS_NAT_SRC;
-		break;
-	case 2:
-		status->nat_type = IPAHAL_PKT_STATUS_NAT_DST;
-		break;
-	default:
-		IPAHAL_ERR("unsupported Status NAT type 0x%x\n",
-			hw_status->nat_type);
-		WARN_ON(1);
-	}
-
-	switch (hw_status->exception) {
-	case 0:
-		exception_type = IPAHAL_PKT_STATUS_EXCEPTION_NONE;
-		break;
-	case 1:
-		exception_type = IPAHAL_PKT_STATUS_EXCEPTION_DEAGGR;
-		break;
-	case 4:
-		exception_type = IPAHAL_PKT_STATUS_EXCEPTION_IPTYPE;
-		break;
-	case 8:
-		exception_type = IPAHAL_PKT_STATUS_EXCEPTION_PACKET_LENGTH;
-		break;
-	case 16:
-		exception_type = IPAHAL_PKT_STATUS_EXCEPTION_FRAG_RULE_MISS;
-		break;
-	case 32:
-		exception_type = IPAHAL_PKT_STATUS_EXCEPTION_SW_FILT;
-		break;
-	case 64:
-		if (is_ipv6)
-			exception_type = IPAHAL_PKT_STATUS_EXCEPTION_IPV6CT;
-		else
-			exception_type = IPAHAL_PKT_STATUS_EXCEPTION_NAT;
-		break;
-	case 229:
-		exception_type = IPAHAL_PKT_STATUS_EXCEPTION_CSUM;
-		break;
-	default:
-		IPAHAL_ERR("unsupported Status Exception type 0x%x\n",
-			hw_status->exception);
-		WARN_ON(1);
-	}
-	status->exception = exception_type;
-
-	IPA_PKT_STATUS_SET_MSK(0x1, IPAHAL_PKT_STATUS_MASK_FRAG_PROCESS_SHFT);
-	IPA_PKT_STATUS_SET_MSK(0x2, IPAHAL_PKT_STATUS_MASK_FILT_PROCESS_SHFT);
-	IPA_PKT_STATUS_SET_MSK(0x4, IPAHAL_PKT_STATUS_MASK_NAT_PROCESS_SHFT);
-	IPA_PKT_STATUS_SET_MSK(0x8, IPAHAL_PKT_STATUS_MASK_ROUTE_PROCESS_SHFT);
-	IPA_PKT_STATUS_SET_MSK(0x10, IPAHAL_PKT_STATUS_MASK_TAG_VALID_SHFT);
-	IPA_PKT_STATUS_SET_MSK(0x20, IPAHAL_PKT_STATUS_MASK_FRAGMENT_SHFT);
-	IPA_PKT_STATUS_SET_MSK(0x40,
-		IPAHAL_PKT_STATUS_MASK_FIRST_FRAGMENT_SHFT);
-	IPA_PKT_STATUS_SET_MSK(0x80, IPAHAL_PKT_STATUS_MASK_V4_SHFT);
-	IPA_PKT_STATUS_SET_MSK(0x100,
-		IPAHAL_PKT_STATUS_MASK_CKSUM_PROCESS_SHFT);
-	IPA_PKT_STATUS_SET_MSK(0x200, IPAHAL_PKT_STATUS_MASK_AGGR_PROCESS_SHFT);
-	IPA_PKT_STATUS_SET_MSK(0x400, IPAHAL_PKT_STATUS_MASK_DEST_EOT_SHFT);
-	IPA_PKT_STATUS_SET_MSK(0x800,
-		IPAHAL_PKT_STATUS_MASK_DEAGGR_PROCESS_SHFT);
-	IPA_PKT_STATUS_SET_MSK(0x1000, IPAHAL_PKT_STATUS_MASK_DEAGG_FIRST_SHFT);
-	IPA_PKT_STATUS_SET_MSK(0x2000, IPAHAL_PKT_STATUS_MASK_SRC_EOT_SHFT);
-	IPA_PKT_STATUS_SET_MSK(0x4000, IPAHAL_PKT_STATUS_MASK_PREV_EOT_SHFT);
-	IPA_PKT_STATUS_SET_MSK(0x8000, IPAHAL_PKT_STATUS_MASK_BYTE_LIMIT_SHFT);
-	status->status_mask &= 0xFFFF;
-}
-
-/*
- * struct ipahal_pkt_status_obj - Pakcet Status H/W information for
- *  specific IPA version
- * @size: H/W size of the status packet
- * @parse: CB that parses the H/W packet status into the abstracted structure
- */
-struct ipahal_pkt_status_obj {
-	u32 size;
-	void (*parse)(const void *unparsed_status,
-		struct ipahal_pkt_status *status);
-};
-
-/*
- * This table contains the info regard packet status for IPAv3 and later
- * Information like: size of packet status and parsing function
- * All the information on the pkt Status on IPAv3 are statically defined below.
- * If information is missing regard some IPA version, the init function
- *  will fill it with the information from the previous IPA version.
- * Information is considered missing if all of the fields are 0
- */
-static struct ipahal_pkt_status_obj ipahal_pkt_status_objs[IPA_HW_MAX] = {
-	/* IPAv3 */
-	[IPA_HW_v3_0] = {
-		IPA3_0_PKT_STATUS_SIZE,
-		ipa_pkt_status_parse,
-		},
-};
-
-/*
- * ipahal_pkt_status_init() - Build the packet status information array
- *  for the different IPA versions
- *  See ipahal_pkt_status_objs[] comments
- */
-static int ipahal_pkt_status_init(enum ipa_hw_type ipa_hw_type)
-{
-	int i;
-	struct ipahal_pkt_status_obj zero_obj;
-
-	IPAHAL_DBG_LOW("Entry - HW_TYPE=%d\n", ipa_hw_type);
-
-	if ((ipa_hw_type < 0) || (ipa_hw_type >= IPA_HW_MAX)) {
-		IPAHAL_ERR("invalid IPA HW type (%d)\n", ipa_hw_type);
-		return -EINVAL;
-	}
-
-	/*
-	 * Since structure alignment is implementation dependent,
-	 * add test to avoid different and incompatible data layouts.
-	 *
-	 * In case new H/W has different size or structure of status packet,
-	 * add a compile time validty check for it like below (as well as
-	 * the new defines and/or the new strucutre in the internal header).
-	 */
-	BUILD_BUG_ON(sizeof(struct ipa_pkt_status_hw) !=
-		IPA3_0_PKT_STATUS_SIZE);
-
-	memset(&zero_obj, 0, sizeof(zero_obj));
-	for (i = IPA_HW_v3_0 ; i < ipa_hw_type ; i++) {
-		if (!memcmp(&ipahal_pkt_status_objs[i+1], &zero_obj,
-			sizeof(struct ipahal_pkt_status_obj))) {
-			memcpy(&ipahal_pkt_status_objs[i+1],
-				&ipahal_pkt_status_objs[i],
-				sizeof(struct ipahal_pkt_status_obj));
-		} else {
-			/*
-			 * explicitly overridden Packet Status info
-			 * Check validity
-			 */
-			if (!ipahal_pkt_status_objs[i+1].size) {
-				IPAHAL_ERR(
-				  "Packet Status with zero size ipa_ver=%d\n",
-				  i+1);
-				WARN_ON(1);
-			}
-			if (!ipahal_pkt_status_objs[i+1].parse) {
-				IPAHAL_ERR(
-				  "Packet Status without Parse func ipa_ver=%d\n",
-				  i+1);
-				WARN_ON(1);
-			}
-		}
-	}
-
-	return 0;
-}
-
-/*
- * ipahal_pkt_status_get_size() - Get H/W size of packet status
- */
-u32 ipahal_pkt_status_get_size(void)
-{
-	return ipahal_pkt_status_objs[ipahal_ctx->hw_type].size;
-}
-
-/*
- * ipahal_pkt_status_parse() - Parse Packet Status payload to abstracted form
- * @unparsed_status: Pointer to H/W format of the packet status as read from H/W
- * @status: Pointer to pre-allocated buffer where the parsed info will be stored
- */
-void ipahal_pkt_status_parse(const void *unparsed_status,
-	struct ipahal_pkt_status *status)
-{
-	if (!unparsed_status || !status) {
-		IPAHAL_ERR("Input Error: unparsed_status=%pK status=%pK\n",
-			unparsed_status, status);
-		return;
-	}
-
-	IPAHAL_DBG_LOW("Parse Status Packet\n");
-	memset(status, 0, sizeof(*status));
-	ipahal_pkt_status_objs[ipahal_ctx->hw_type].parse(unparsed_status,
-		status);
-}
-
-/*
- * ipahal_pkt_status_exception_str() - returns string represents exception type
- * @exception: [in] The exception type
- */
-const char *ipahal_pkt_status_exception_str(
-	enum ipahal_pkt_status_exception exception)
-{
-	if (exception < 0 || exception >= IPAHAL_PKT_STATUS_EXCEPTION_MAX) {
-		IPAHAL_ERR(
-			"requested string of invalid pkt_status exception=%d\n",
-			exception);
-		return "Invalid PKT_STATUS_EXCEPTION";
-	}
-
-	return ipahal_pkt_status_exception_to_str[exception];
-}
-
-#ifdef CONFIG_DEBUG_FS
-static void ipahal_debugfs_init(void)
-{
-	ipahal_ctx->dent = debugfs_create_dir("ipahal", 0);
-	if (!ipahal_ctx->dent || IS_ERR(ipahal_ctx->dent)) {
-		IPAHAL_ERR("fail to create ipahal debugfs folder\n");
-		goto fail;
-	}
-
-	return;
-fail:
-	debugfs_remove_recursive(ipahal_ctx->dent);
-	ipahal_ctx->dent = NULL;
-}
-
-static void ipahal_debugfs_remove(void)
-{
-	if (!ipahal_ctx)
-		return;
-
-	if (IS_ERR(ipahal_ctx->dent)) {
-		IPAHAL_ERR("ipahal debugfs folder was not created\n");
-		return;
-	}
-
-	debugfs_remove_recursive(ipahal_ctx->dent);
-}
-#else /* CONFIG_DEBUG_FS */
-static void ipahal_debugfs_init(void) {}
-static void ipahal_debugfs_remove(void) {}
-#endif /* CONFIG_DEBUG_FS */
-
-/*
- * ipahal_cp_hdr_to_hw_buff_v3() - copy header to hardware buffer according to
- * base address and offset given.
- * @base: dma base address
- * @offset: offset from base address where the data will be copied
- * @hdr: the header to be copied
- * @hdr_len: the length of the header
- */
-static void ipahal_cp_hdr_to_hw_buff_v3(void *const base, u32 offset,
-		u8 *const hdr, u32 hdr_len)
-{
-	memcpy(base + offset, hdr, hdr_len);
-}
-
-/* Header address update logic. */
-#define IPAHAL_CP_PROC_CTX_HEADER_UPDATE(hdr_lsb, hdr_msb, addr) \
-	do { \
-		hdr_lsb = lower_32_bits(addr); \
-		hdr_msb = upper_32_bits(addr); \
-	} while (0)
-
-/*
- * ipahal_cp_proc_ctx_to_hw_buff_v3() - copy processing context to
- * base address and offset given.
- * @type: header processing context type (no processing context,
- *	IPA_HDR_PROC_ETHII_TO_ETHII etc.)
- * @base: dma base address
- * @offset: offset from base address where the data will be copied
- * @hdr_len: the length of the header
- * @is_hdr_proc_ctx: header is located in phys_base (true) or hdr_base_addr
- * @phys_base: memory location in DDR
- * @hdr_base_addr: base address in table
- * @offset_entry: offset from hdr_base_addr in table
- * @l2tp_params: l2tp parameters
- * @is_64: Indicates whether header base address/dma base address is 64 bit.
- */
-static int ipahal_cp_proc_ctx_to_hw_buff_v3(enum ipa_hdr_proc_type type,
-		void *const base, u32 offset,
-		u32 hdr_len, bool is_hdr_proc_ctx,
-		dma_addr_t phys_base, u64 hdr_base_addr,
-		struct ipa_hdr_offset_entry *offset_entry,
-		struct ipa_l2tp_hdr_proc_ctx_params l2tp_params, bool is_64)
-{
-	u64 hdr_addr;
-
-	if (type == IPA_HDR_PROC_NONE) {
-		struct ipa_hw_hdr_proc_ctx_add_hdr_seq *ctx;
-
-		ctx = (struct ipa_hw_hdr_proc_ctx_add_hdr_seq *)
-			(base + offset);
-		ctx->hdr_add.tlv.type = IPA_PROC_CTX_TLV_TYPE_HDR_ADD;
-		ctx->hdr_add.tlv.length = 2;
-		ctx->hdr_add.tlv.value = hdr_len;
-		hdr_addr = is_hdr_proc_ctx ? phys_base :
-			hdr_base_addr + offset_entry->offset;
-		IPAHAL_DBG("header address 0x%llx\n",
-			hdr_addr);
-		IPAHAL_CP_PROC_CTX_HEADER_UPDATE(ctx->hdr_add.hdr_addr,
-			ctx->hdr_add.hdr_addr_hi, hdr_addr);
-		if (!is_64)
-			ctx->hdr_add.hdr_addr_hi = 0;
-		ctx->end.type = IPA_PROC_CTX_TLV_TYPE_END;
-		ctx->end.length = 0;
-		ctx->end.value = 0;
-	} else if (type == IPA_HDR_PROC_L2TP_HEADER_ADD) {
-		struct ipa_hw_hdr_proc_ctx_add_l2tp_hdr_cmd_seq *ctx;
-
-		ctx = (struct ipa_hw_hdr_proc_ctx_add_l2tp_hdr_cmd_seq *)
-			(base + offset);
-		ctx->hdr_add.tlv.type = IPA_PROC_CTX_TLV_TYPE_HDR_ADD;
-		ctx->hdr_add.tlv.length = 2;
-		ctx->hdr_add.tlv.value = hdr_len;
-		hdr_addr = is_hdr_proc_ctx ? phys_base :
-			hdr_base_addr + offset_entry->offset;
-		IPAHAL_DBG("header address 0x%llx\n",
-			hdr_addr);
-		IPAHAL_CP_PROC_CTX_HEADER_UPDATE(ctx->hdr_add.hdr_addr,
-			ctx->hdr_add.hdr_addr_hi, hdr_addr);
-		if (!is_64)
-			ctx->hdr_add.hdr_addr_hi = 0;
-		ctx->l2tp_params.tlv.type = IPA_PROC_CTX_TLV_TYPE_PROC_CMD;
-		ctx->l2tp_params.tlv.length = 1;
-		ctx->l2tp_params.tlv.value =
-				IPA_HDR_UCP_L2TP_HEADER_ADD;
-		ctx->l2tp_params.l2tp_params.eth_hdr_retained =
-			l2tp_params.hdr_add_param.eth_hdr_retained;
-		ctx->l2tp_params.l2tp_params.input_ip_version =
-			l2tp_params.hdr_add_param.input_ip_version;
-		ctx->l2tp_params.l2tp_params.output_ip_version =
-			l2tp_params.hdr_add_param.output_ip_version;
-
-		IPAHAL_DBG("command id %d\n", ctx->l2tp_params.tlv.value);
-		ctx->end.type = IPA_PROC_CTX_TLV_TYPE_END;
-		ctx->end.length = 0;
-		ctx->end.value = 0;
-	} else if (type == IPA_HDR_PROC_L2TP_HEADER_REMOVE) {
-		struct ipa_hw_hdr_proc_ctx_remove_l2tp_hdr_cmd_seq *ctx;
-
-		ctx = (struct ipa_hw_hdr_proc_ctx_remove_l2tp_hdr_cmd_seq *)
-			(base + offset);
-		ctx->hdr_add.tlv.type = IPA_PROC_CTX_TLV_TYPE_HDR_ADD;
-		ctx->hdr_add.tlv.length = 2;
-		ctx->hdr_add.tlv.value = hdr_len;
-		hdr_addr = is_hdr_proc_ctx ? phys_base :
-			hdr_base_addr + offset_entry->offset;
-		IPAHAL_DBG("header address 0x%llx length %d\n",
-			hdr_addr, ctx->hdr_add.tlv.value);
-		IPAHAL_CP_PROC_CTX_HEADER_UPDATE(ctx->hdr_add.hdr_addr,
-			ctx->hdr_add.hdr_addr_hi, hdr_addr);
-		if (!is_64)
-			ctx->hdr_add.hdr_addr_hi = 0;
-		ctx->l2tp_params.tlv.type = IPA_PROC_CTX_TLV_TYPE_PROC_CMD;
-		ctx->l2tp_params.tlv.length = 1;
-		ctx->l2tp_params.tlv.value =
-				IPA_HDR_UCP_L2TP_HEADER_REMOVE;
-		ctx->l2tp_params.l2tp_params.hdr_len_remove =
-			l2tp_params.hdr_remove_param.hdr_len_remove;
-		ctx->l2tp_params.l2tp_params.eth_hdr_retained =
-			l2tp_params.hdr_remove_param.eth_hdr_retained;
-		ctx->l2tp_params.l2tp_params.hdr_ofst_pkt_size_valid =
-			l2tp_params.hdr_remove_param.hdr_ofst_pkt_size_valid;
-		ctx->l2tp_params.l2tp_params.hdr_ofst_pkt_size =
-			l2tp_params.hdr_remove_param.hdr_ofst_pkt_size;
-		ctx->l2tp_params.l2tp_params.hdr_endianness =
-			l2tp_params.hdr_remove_param.hdr_endianness;
-		IPAHAL_DBG("hdr ofst valid: %d, hdr ofst pkt size: %d\n",
-			ctx->l2tp_params.l2tp_params.hdr_ofst_pkt_size_valid,
-			ctx->l2tp_params.l2tp_params.hdr_ofst_pkt_size);
-		IPAHAL_DBG("endianness: %d\n",
-			ctx->l2tp_params.l2tp_params.hdr_endianness);
-
-		IPAHAL_DBG("command id %d\n", ctx->l2tp_params.tlv.value);
-		ctx->end.type = IPA_PROC_CTX_TLV_TYPE_END;
-		ctx->end.length = 0;
-		ctx->end.value = 0;
-	} else {
-		struct ipa_hw_hdr_proc_ctx_add_hdr_cmd_seq *ctx;
-
-		ctx = (struct ipa_hw_hdr_proc_ctx_add_hdr_cmd_seq *)
-			(base + offset);
-		ctx->hdr_add.tlv.type = IPA_PROC_CTX_TLV_TYPE_HDR_ADD;
-		ctx->hdr_add.tlv.length = 2;
-		ctx->hdr_add.tlv.value = hdr_len;
-		hdr_addr = is_hdr_proc_ctx ? phys_base :
-			hdr_base_addr + offset_entry->offset;
-		IPAHAL_DBG("header address 0x%llx\n",
-			hdr_addr);
-		IPAHAL_CP_PROC_CTX_HEADER_UPDATE(ctx->hdr_add.hdr_addr,
-			ctx->hdr_add.hdr_addr_hi, hdr_addr);
-		if (!is_64)
-			ctx->hdr_add.hdr_addr_hi = 0;
-		ctx->cmd.type = IPA_PROC_CTX_TLV_TYPE_PROC_CMD;
-		ctx->cmd.length = 0;
-		switch (type) {
-		case IPA_HDR_PROC_ETHII_TO_ETHII:
-			ctx->cmd.value = IPA_HDR_UCP_ETHII_TO_ETHII;
-			break;
-		case IPA_HDR_PROC_ETHII_TO_802_3:
-			ctx->cmd.value = IPA_HDR_UCP_ETHII_TO_802_3;
-			break;
-		case IPA_HDR_PROC_802_3_TO_ETHII:
-			ctx->cmd.value = IPA_HDR_UCP_802_3_TO_ETHII;
-			break;
-		case IPA_HDR_PROC_802_3_TO_802_3:
-			ctx->cmd.value = IPA_HDR_UCP_802_3_TO_802_3;
-			break;
-		default:
-			IPAHAL_ERR("unknown ipa_hdr_proc_type %d", type);
-			WARN_ON(1);
-			return -EINVAL;
-		}
-		IPAHAL_DBG("command id %d\n", ctx->cmd.value);
-		ctx->end.type = IPA_PROC_CTX_TLV_TYPE_END;
-		ctx->end.length = 0;
-		ctx->end.value = 0;
-	}
-
-	return 0;
-}
-
-/*
- * ipahal_get_proc_ctx_needed_len_v3() - calculates the needed length for
- * addition of header processing context according to the type of processing
- * context.
- * @type: header processing context type (no processing context,
- *	IPA_HDR_PROC_ETHII_TO_ETHII etc.)
- */
-static int ipahal_get_proc_ctx_needed_len_v3(enum ipa_hdr_proc_type type)
-{
-	return (type == IPA_HDR_PROC_NONE) ?
-			sizeof(struct ipa_hw_hdr_proc_ctx_add_hdr_seq) :
-			sizeof(struct ipa_hw_hdr_proc_ctx_add_hdr_cmd_seq);
-}
-
-/*
- * struct ipahal_hdr_funcs - headers handling functions for specific IPA
- * version
- * @ipahal_cp_hdr_to_hw_buff - copy function for regular headers
- */
-struct ipahal_hdr_funcs {
-	void (*ipahal_cp_hdr_to_hw_buff)(void *const base, u32 offset,
-			u8 *const hdr, u32 hdr_len);
-
-	int (*ipahal_cp_proc_ctx_to_hw_buff)(enum ipa_hdr_proc_type type,
-			void *const base, u32 offset, u32 hdr_len,
-			bool is_hdr_proc_ctx, dma_addr_t phys_base,
-			u64 hdr_base_addr,
-			struct ipa_hdr_offset_entry *offset_entry,
-			struct ipa_l2tp_hdr_proc_ctx_params l2tp_params,
-			bool is_64);
-
-	int (*ipahal_get_proc_ctx_needed_len)(enum ipa_hdr_proc_type type);
-};
-
-static struct ipahal_hdr_funcs hdr_funcs;
-
-static void ipahal_hdr_init(enum ipa_hw_type ipa_hw_type)
-{
-
-	IPAHAL_DBG("Entry - HW_TYPE=%d\n", ipa_hw_type);
-
-	/*
-	 * once there are changes in HW and need to use different case, insert
-	 * new case for the new h/w. put the default always for the latest HW
-	 * and make sure all previous supported versions have their cases.
-	 */
-	switch (ipa_hw_type) {
-	case IPA_HW_v3_0:
-	default:
-		hdr_funcs.ipahal_cp_hdr_to_hw_buff =
-				ipahal_cp_hdr_to_hw_buff_v3;
-		hdr_funcs.ipahal_cp_proc_ctx_to_hw_buff =
-				ipahal_cp_proc_ctx_to_hw_buff_v3;
-		hdr_funcs.ipahal_get_proc_ctx_needed_len =
-				ipahal_get_proc_ctx_needed_len_v3;
-	}
-	IPAHAL_DBG("Exit\n");
-}
-
-/*
- * ipahal_cp_hdr_to_hw_buff() - copy header to hardware buffer according to
- * base address and offset given.
- * @base: dma base address
- * @offset: offset from base address where the data will be copied
- * @hdr: the header to be copied
- * @hdr_len: the length of the header
- */
-void ipahal_cp_hdr_to_hw_buff(void *base, u32 offset, u8 *const hdr,
-		u32 hdr_len)
-{
-	IPAHAL_DBG_LOW("Entry\n");
-	IPAHAL_DBG("base %pK, offset %d, hdr %pK, hdr_len %d\n", base,
-			offset, hdr, hdr_len);
-	if (!base || !hdr_len || !hdr) {
-		IPAHAL_ERR("failed on validating params\n");
-		return;
-	}
-
-	hdr_funcs.ipahal_cp_hdr_to_hw_buff(base, offset, hdr, hdr_len);
-
-	IPAHAL_DBG_LOW("Exit\n");
-}
-
-/*
- * ipahal_cp_proc_ctx_to_hw_buff() - copy processing context to
- * base address and offset given.
- * @type: type of header processing context
- * @base: dma base address
- * @offset: offset from base address where the data will be copied
- * @hdr_len: the length of the header
- * @is_hdr_proc_ctx: header is located in phys_base (true) or hdr_base_addr
- * @phys_base: memory location in DDR
- * @hdr_base_addr: base address in table
- * @offset_entry: offset from hdr_base_addr in table
- * @l2tp_params: l2tp parameters
- * @is_64: Indicates whether header base address/dma base address is 64 bit.
- */
-int ipahal_cp_proc_ctx_to_hw_buff(enum ipa_hdr_proc_type type,
-		void *const base, u32 offset, u32 hdr_len,
-		bool is_hdr_proc_ctx, dma_addr_t phys_base,
-		u64 hdr_base_addr, struct ipa_hdr_offset_entry *offset_entry,
-		struct ipa_l2tp_hdr_proc_ctx_params l2tp_params, bool is_64)
-{
-	IPAHAL_DBG(
-		"type %d, base %pK, offset %d, hdr_len %d, is_hdr_proc_ctx %d, hdr_base_addr %llu, offset_entry %pK, bool %d\n"
-			, type, base, offset, hdr_len, is_hdr_proc_ctx,
-			hdr_base_addr, offset_entry, is_64);
-
-	if (!base ||
-		!hdr_len ||
-		(is_hdr_proc_ctx && !phys_base) ||
-		(!is_hdr_proc_ctx && !offset_entry) ||
-		(!is_hdr_proc_ctx && !hdr_base_addr)) {
-		IPAHAL_ERR(
-			"invalid input: hdr_len:%u phys_base:%pad hdr_base_addr:%llu is_hdr_proc_ctx:%d offset_entry:%pK\n"
-			, hdr_len, &phys_base, hdr_base_addr
-			, is_hdr_proc_ctx, offset_entry);
-		return -EINVAL;
-	}
-
-	return hdr_funcs.ipahal_cp_proc_ctx_to_hw_buff(type, base, offset,
-			hdr_len, is_hdr_proc_ctx, phys_base,
-			hdr_base_addr, offset_entry, l2tp_params, is_64);
-}
-
-/*
- * ipahal_get_proc_ctx_needed_len() - calculates the needed length for
- * addition of header processing context according to the type of processing
- * context
- * @type: header processing context type (no processing context,
- *	IPA_HDR_PROC_ETHII_TO_ETHII etc.)
- */
-int ipahal_get_proc_ctx_needed_len(enum ipa_hdr_proc_type type)
-{
-	int res;
-
-	IPAHAL_DBG("entry\n");
-
-	res = hdr_funcs.ipahal_get_proc_ctx_needed_len(type);
-
-	IPAHAL_DBG("Exit\n");
-
-	return res;
-}
-
-int ipahal_init(enum ipa_hw_type ipa_hw_type, void __iomem *base,
-	struct device *ipa_pdev)
-{
-	int result;
-
-	IPAHAL_DBG("Entry - IPA HW TYPE=%d base=%pK ipa_pdev=%pK\n",
-		ipa_hw_type, base, ipa_pdev);
-
-	ipahal_ctx = kzalloc(sizeof(*ipahal_ctx), GFP_KERNEL);
-	if (!ipahal_ctx) {
-		IPAHAL_ERR("kzalloc err for ipahal_ctx\n");
-		result = -ENOMEM;
-		goto bail_err_exit;
-	}
-
-	if (ipa_hw_type < IPA_HW_v3_0) {
-		IPAHAL_ERR("ipahal supported on IPAv3 and later only\n");
-		result = -EINVAL;
-		goto bail_free_ctx;
-	}
-
-	if (ipa_hw_type >= IPA_HW_MAX) {
-		IPAHAL_ERR("invalid IPA HW type (%d)\n", ipa_hw_type);
-		result = -EINVAL;
-		goto bail_free_ctx;
-	}
-
-	if (!base) {
-		IPAHAL_ERR("invalid memory io mapping addr\n");
-		result = -EINVAL;
-		goto bail_free_ctx;
-	}
-
-	if (!ipa_pdev) {
-		IPAHAL_ERR("invalid IPA platform device\n");
-		result = -EINVAL;
-		goto bail_free_ctx;
-	}
-
-	ipahal_ctx->hw_type = ipa_hw_type;
-	ipahal_ctx->base = base;
-	ipahal_ctx->ipa_pdev = ipa_pdev;
-
-	if (ipahal_reg_init(ipa_hw_type)) {
-		IPAHAL_ERR("failed to init ipahal reg\n");
-		result = -EFAULT;
-		goto bail_free_ctx;
-	}
-
-	if (ipahal_imm_cmd_init(ipa_hw_type)) {
-		IPAHAL_ERR("failed to init ipahal imm cmd\n");
-		result = -EFAULT;
-		goto bail_free_ctx;
-	}
-
-	if (ipahal_pkt_status_init(ipa_hw_type)) {
-		IPAHAL_ERR("failed to init ipahal pkt status\n");
-		result = -EFAULT;
-		goto bail_free_ctx;
-	}
-
-	ipahal_hdr_init(ipa_hw_type);
-
-	if (ipahal_fltrt_init(ipa_hw_type)) {
-		IPAHAL_ERR("failed to init ipahal flt rt\n");
-		result = -EFAULT;
-		goto bail_free_ctx;
-	}
-
-	if (ipahal_hw_stats_init(ipa_hw_type)) {
-		IPAHAL_ERR("failed to init ipahal hw stats\n");
-		result = -EFAULT;
-		goto bail_free_fltrt;
-	}
-
-	if (ipahal_nat_init(ipa_hw_type)) {
-		IPAHAL_ERR("failed to init ipahal NAT\n");
-		result = -EFAULT;
-		goto bail_free_fltrt;
-	}
-
-	/* create an IPC buffer for the registers dump */
-	ipahal_ctx->regdumpbuf = ipc_log_context_create(IPAHAL_IPC_LOG_PAGES,
-		"ipa_regs", 0);
-	if (ipahal_ctx->regdumpbuf == NULL)
-		IPAHAL_ERR("failed to create IPA regdump log, continue...\n");
-
-	ipahal_debugfs_init();
-
-	return 0;
-
-bail_free_fltrt:
-	ipahal_fltrt_destroy();
-bail_free_ctx:
-	if (ipahal_ctx->regdumpbuf)
-		ipc_log_context_destroy(ipahal_ctx->regdumpbuf);
-	kfree(ipahal_ctx);
-	ipahal_ctx = NULL;
-bail_err_exit:
-	return result;
-}
-
-void ipahal_destroy(void)
-{
-	IPAHAL_DBG("Entry\n");
-	ipahal_fltrt_destroy();
-	ipahal_debugfs_remove();
-	kfree(ipahal_ctx);
-	ipahal_ctx = NULL;
-}
-
-void ipahal_free_dma_mem(struct ipa_mem_buffer *mem)
-{
-	if (likely(mem)) {
-		dma_free_coherent(ipahal_ctx->ipa_pdev, mem->size, mem->base,
-			mem->phys_base);
-		mem->size = 0;
-		mem->base = NULL;
-		mem->phys_base = 0;
-	}
-}

ipa/ipa_v3/ipahal/ipahal.h

@@ -1,655 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2016-2019, The Linux Foundation. All rights reserved.
- */
-
-#ifndef _IPAHAL_H_
-#define _IPAHAL_H_
-
-#include "../ipa_defs.h"
-#include "../../ipa_common_i.h"
-
-/*
- * Immediate command names
- *
- * NOTE:: Any change to this enum, need to change to ipahal_imm_cmd_name_to_str
- *	array as well.
- */
-enum ipahal_imm_cmd_name {
-	IPA_IMM_CMD_IP_V4_FILTER_INIT,
-	IPA_IMM_CMD_IP_V6_FILTER_INIT,
-	IPA_IMM_CMD_IP_V4_NAT_INIT,
-	IPA_IMM_CMD_IP_V4_ROUTING_INIT,
-	IPA_IMM_CMD_IP_V6_ROUTING_INIT,
-	IPA_IMM_CMD_HDR_INIT_LOCAL,
-	IPA_IMM_CMD_HDR_INIT_SYSTEM,
-	IPA_IMM_CMD_REGISTER_WRITE,
-	IPA_IMM_CMD_NAT_DMA,
-	IPA_IMM_CMD_IP_PACKET_INIT,
-	IPA_IMM_CMD_DMA_SHARED_MEM,
-	IPA_IMM_CMD_IP_PACKET_TAG_STATUS,
-	IPA_IMM_CMD_DMA_TASK_32B_ADDR,
-	IPA_IMM_CMD_TABLE_DMA,
-	IPA_IMM_CMD_IP_V6_CT_INIT,
-	IPA_IMM_CMD_MAX,
-};
-
-/* Immediate commands abstracted structures */
-
-/*
- * struct ipahal_imm_cmd_ip_v4_filter_init - IP_V4_FILTER_INIT cmd payload
- * Inits IPv4 filter block.
- * @hash_rules_addr: Addr in sys mem where ipv4 hashable flt tbl starts
- * @nhash_rules_addr: Addr in sys mem where ipv4 non-hashable flt tbl starts
- * @hash_rules_size: Size in bytes of the hashable tbl to cpy to local mem
- * @hash_local_addr: Addr in shared mem where ipv4 hashable flt tbl should
- *  be copied to
- * @nhash_rules_size: Size in bytes of the non-hashable tbl to cpy to local mem
- * @nhash_local_addr: Addr in shared mem where ipv4 non-hashable flt tbl should
- *  be copied to
- */
-struct ipahal_imm_cmd_ip_v4_filter_init {
-	u64 hash_rules_addr;
-	u64 nhash_rules_addr;
-	u32 hash_rules_size;
-	u32 hash_local_addr;
-	u32 nhash_rules_size;
-	u32 nhash_local_addr;
-};
-
-/*
- * struct ipahal_imm_cmd_ip_v6_filter_init - IP_V6_FILTER_INIT cmd payload
- * Inits IPv6 filter block.
- * @hash_rules_addr: Addr in sys mem where ipv6 hashable flt tbl starts
- * @nhash_rules_addr: Addr in sys mem where ipv6 non-hashable flt tbl starts
- * @hash_rules_size: Size in bytes of the hashable tbl to cpy to local mem
- * @hash_local_addr: Addr in shared mem where ipv6 hashable flt tbl should
- *  be copied to
- * @nhash_rules_size: Size in bytes of the non-hashable tbl to cpy to local mem
- * @nhash_local_addr: Addr in shared mem where ipv6 non-hashable flt tbl should
- *  be copied to
- */
-struct ipahal_imm_cmd_ip_v6_filter_init {
-	u64 hash_rules_addr;
-	u64 nhash_rules_addr;
-	u32 hash_rules_size;
-	u32 hash_local_addr;
-	u32 nhash_rules_size;
-	u32 nhash_local_addr;
-};
-
-/*
- * struct ipahal_imm_cmd_nat_ipv6ct_init_common - NAT/IPv6CT table init command
- *                                                common part
- * @base_table_addr: Address in sys/shared mem where base table start
- * @expansion_table_addr: Address in sys/shared mem where expansion table
- *  starts. Entries that result in hash collision are located in this table.
- * @base_table_addr_shared: base_table_addr in shared mem (if not, then sys)
- * @expansion_table_addr_shared: expansion_rules_addr in
- *  shared mem (if not, then sys)
- * @size_base_table: Num of entries in the base table
- * @size_expansion_table: Num of entries in the expansion table
- * @table_index: For future support of multiple tables
- */
-struct ipahal_imm_cmd_nat_ipv6ct_init_common {
-	u64 base_table_addr;
-	u64 expansion_table_addr;
-	bool base_table_addr_shared;
-	bool expansion_table_addr_shared;
-	u16 size_base_table;
-	u16 size_expansion_table;
-	u8 table_index;
-};
-
-/*
- * struct ipahal_imm_cmd_ip_v4_nat_init - IP_V4_NAT_INIT cmd payload
- * Inits IPv4 NAT block. Initiate NAT table with it dimensions, location
- *  cache address and other related parameters.
- * @table_init: table initialization parameters
- * @index_table_addr: Addr in sys/shared mem where index table, which points
- *  to NAT table starts
- * @index_table_expansion_addr: Addr in sys/shared mem where expansion index
- *  table starts
- * @index_table_addr_shared: index_table_addr in shared mem (if not, then sys)
- * @index_table_expansion_addr_shared: index_table_expansion_addr in
- *  shared mem (if not, then sys)
- * @public_addr_info: Public IP addresses info suitable to the IPA H/W version
- *                    IPA H/W >= 4.0 - PDN config table offset in SMEM
- *                    IPA H/W < 4.0  - The public IP address
- */
-struct ipahal_imm_cmd_ip_v4_nat_init {
-	struct ipahal_imm_cmd_nat_ipv6ct_init_common table_init;
-	u64 index_table_addr;
-	u64 index_table_expansion_addr;
-	bool index_table_addr_shared;
-	bool index_table_expansion_addr_shared;
-	u32 public_addr_info;
-};
-
-/*
- * struct ipahal_imm_cmd_ip_v6_ct_init - IP_V6_CONN_TRACK_INIT cmd payload
- * Inits IPv6CT block. Initiate IPv6CT table with it dimensions, location
- *  cache address and other related parameters.
- * @table_init: table initialization parameters
- */
-struct ipahal_imm_cmd_ip_v6_ct_init {
-	struct ipahal_imm_cmd_nat_ipv6ct_init_common table_init;
-};
-
-/*
- * struct ipahal_imm_cmd_ip_v4_routing_init - IP_V4_ROUTING_INIT cmd payload
- * Inits IPv4 routing table/structure - with the rules and other related params
- * @hash_rules_addr: Addr in sys mem where ipv4 hashable rt tbl starts
- * @nhash_rules_addr: Addr in sys mem where ipv4 non-hashable rt tbl starts
- * @hash_rules_size: Size in bytes of the hashable tbl to cpy to local mem
- * @hash_local_addr: Addr in shared mem where ipv4 hashable rt tbl should
- *  be copied to
- * @nhash_rules_size: Size in bytes of the non-hashable tbl to cpy to local mem
- * @nhash_local_addr: Addr in shared mem where ipv4 non-hashable rt tbl should
- *  be copied to
- */
-struct ipahal_imm_cmd_ip_v4_routing_init {
-	u64 hash_rules_addr;
-	u64 nhash_rules_addr;
-	u32 hash_rules_size;
-	u32 hash_local_addr;
-	u32 nhash_rules_size;
-	u32 nhash_local_addr;
-};
-
-/*
- * struct ipahal_imm_cmd_ip_v6_routing_init - IP_V6_ROUTING_INIT cmd payload
- * Inits IPv6 routing table/structure - with the rules and other related params
- * @hash_rules_addr: Addr in sys mem where ipv6 hashable rt tbl starts
- * @nhash_rules_addr: Addr in sys mem where ipv6 non-hashable rt tbl starts
- * @hash_rules_size: Size in bytes of the hashable tbl to cpy to local mem
- * @hash_local_addr: Addr in shared mem where ipv6 hashable rt tbl should
- *  be copied to
- * @nhash_rules_size: Size in bytes of the non-hashable tbl to cpy to local mem
- * @nhash_local_addr: Addr in shared mem where ipv6 non-hashable rt tbl should
- *  be copied to
- */
-struct ipahal_imm_cmd_ip_v6_routing_init {
-	u64 hash_rules_addr;
-	u64 nhash_rules_addr;
-	u32 hash_rules_size;
-	u32 hash_local_addr;
-	u32 nhash_rules_size;
-	u32 nhash_local_addr;
-};
-
-/*
- * struct ipahal_imm_cmd_hdr_init_local - HDR_INIT_LOCAL cmd payload
- * Inits hdr table within local mem with the hdrs and their length.
- * @hdr_table_addr: Word address in sys mem where the table starts (SRC)
- * @size_hdr_table: Size of the above (in bytes)
- * @hdr_addr: header address in IPA sram (used as DST for memory copy)
- * @rsvd: reserved
- */
-struct ipahal_imm_cmd_hdr_init_local {
-	u64 hdr_table_addr;
-	u32 size_hdr_table;
-	u32 hdr_addr;
-};
-
-/*
- * struct ipahal_imm_cmd_hdr_init_system - HDR_INIT_SYSTEM cmd payload
- * Inits hdr table within sys mem with the hdrs and their length.
- * @hdr_table_addr: Word address in system memory where the hdrs tbl starts.
- */
-struct ipahal_imm_cmd_hdr_init_system {
-	u64 hdr_table_addr;
-};
-
-/*
- * struct ipahal_imm_cmd_table_dma - TABLE_DMA cmd payload
- * Perform DMA operation on NAT and IPV6 connection tracking related mem
- * addresses. Copy data into different locations within IPv6CT and NAT
- * associated tbls. (For add/remove NAT rules)
- * @offset: offset in bytes from base addr to write 'data' to
- * @data: data to be written
- * @table_index: NAT tbl index. Defines the tbl on which to perform DMA op.
- * @base_addr: Base addr to which the DMA operation should be performed.
- */
-struct ipahal_imm_cmd_table_dma {
-	u32 offset;
-	u16 data;
-	u8 table_index;
-	u8 base_addr;
-};
-
-/*
- * struct ipahal_imm_cmd_ip_packet_init - IP_PACKET_INIT cmd payload
- * Configuration for specific IP pkt. Shall be called prior to an IP pkt
- *  data. Pkt will not go through IP pkt processing.
- * @destination_pipe_index: Destination pipe index  (in case routing
- *  is enabled, this field will overwrite the rt  rule)
- */
-struct ipahal_imm_cmd_ip_packet_init {
-	u32 destination_pipe_index;
-};
-
-/*
- * enum ipa_pipeline_clear_option - Values for pipeline clear waiting options
- * @IPAHAL_HPS_CLEAR: Wait for HPS clear. All queues except high priority queue
- *  shall not be serviced until HPS is clear of packets or immediate commands.
- *  The high priority Rx queue / Q6ZIP group shall still be serviced normally.
- *
- * @IPAHAL_SRC_GRP_CLEAR: Wait for originating source group to be clear
- *  (for no packet contexts allocated to the originating source group).
- *  The source group / Rx queue shall not be serviced until all previously
- *  allocated packet contexts are released. All other source groups/queues shall
- *  be serviced normally.
- *
- * @IPAHAL_FULL_PIPELINE_CLEAR: Wait for full pipeline to be clear.
- *  All groups / Rx queues shall not be serviced until IPA pipeline is fully
- *  clear. This should be used for debug only.
- */
-enum ipahal_pipeline_clear_option {
-	IPAHAL_HPS_CLEAR,
-	IPAHAL_SRC_GRP_CLEAR,
-	IPAHAL_FULL_PIPELINE_CLEAR
-};
-
-/*
- * struct ipahal_imm_cmd_register_write - REGISTER_WRITE cmd payload
- * Write value to register. Allows reg changes to be synced with data packet
- *  and other immediate commands. Can be used to access the sram
- * @offset: offset from IPA base address - Lower 16bit of the IPA reg addr
- * @value: value to write to register
- * @value_mask: mask specifying which value bits to write to the register
- * @skip_pipeline_clear: if to skip pipeline clear waiting (don't wait)
- * @pipeline_clear_option: options for pipeline clear waiting
- */
-struct ipahal_imm_cmd_register_write {
-	u32 offset;
-	u32 value;
-	u32 value_mask;
-	bool skip_pipeline_clear;
-	enum ipahal_pipeline_clear_option pipeline_clear_options;
-};
-
-/*
- * struct ipahal_imm_cmd_dma_shared_mem - DMA_SHARED_MEM cmd payload
- * Perform mem copy into or out of the SW area of IPA local mem
- * @system_addr: Address in system memory
- * @size: Size in bytes of data to copy. Expected size is up to 2K bytes
- * @local_addr: Address in IPA local memory
- * @clear_after_read: Clear local memory at the end of a read operation allows
- *  atomic read and clear if HPS is clear. Ignore for writes.
- * @is_read: Read operation from local memory? If not, then write.
- * @skip_pipeline_clear: if to skip pipeline clear waiting (don't wait)
- * @pipeline_clear_option: options for pipeline clear waiting
- */
-struct ipahal_imm_cmd_dma_shared_mem {
-	u64 system_addr;
-	u32 size;
-	u32 local_addr;
-	bool clear_after_read;
-	bool is_read;
-	bool skip_pipeline_clear;
-	enum ipahal_pipeline_clear_option pipeline_clear_options;
-};
-
-/*
- * struct ipahal_imm_cmd_ip_packet_tag_status - IP_PACKET_TAG_STATUS cmd payload
- * This cmd is used for to allow SW to track HW processing by setting a TAG
- *  value that is passed back to SW inside Packet Status information.
- *  TAG info will be provided as part of Packet Status info generated for
- *  the next pkt transferred over the pipe.
- *  This immediate command must be followed by a packet in the same transfer.
- * @tag: Tag that is provided back to SW
- */
-struct ipahal_imm_cmd_ip_packet_tag_status {
-	u64 tag;
-};
-
-/*
- * struct ipahal_imm_cmd_dma_task_32b_addr - IPA_DMA_TASK_32B_ADDR cmd payload
- * Used by clients using 32bit addresses. Used to perform DMA operation on
- *  multiple descriptors.
- *  The Opcode is dynamic, where it holds the number of buffer to process
- * @cmplt: Complete flag: If true, IPA interrupt SW when the entire
- *  DMA related data was completely xfered to its destination.
- * @eof: Enf Of Frame flag: If true, IPA assert the EOT to the
- *  dest client. This is used used for aggr sequence
- * @flsh: Flush flag: If true pkt will go through the IPA blocks but
- *  will not be xfered to dest client but rather will be discarded
- * @lock: Lock pipe flag: If true, IPA will stop processing descriptors
- *  from other EPs in the same src grp (RX queue)
- * @unlock: Unlock pipe flag: If true, IPA will stop exclusively
- *  servicing current EP out of the src EPs of the grp (RX queue)
- * @size1: Size of buffer1 data
- * @addr1: Pointer to buffer1 data
- * @packet_size: Total packet size. If a pkt send using multiple DMA_TASKs,
- *  only the first one needs to have this field set. It will be ignored
- *  in subsequent DMA_TASKs until the packet ends (EOT). First DMA_TASK
- *  must contain this field (2 or more buffers) or EOT.
- */
-struct ipahal_imm_cmd_dma_task_32b_addr {
-	bool cmplt;
-	bool eof;
-	bool flsh;
-	bool lock;
-	bool unlock;
-	u32 size1;
-	u32 addr1;
-	u32 packet_size;
-};
-
-/*
- * struct ipahal_imm_cmd_pyld - Immediate cmd payload information
- * @len: length of the buffer
- * @opcode: opcode of the immediate command
- * @data: buffer contains the immediate command payload. Buffer goes
- *  back to back with this structure
- */
-struct ipahal_imm_cmd_pyld {
-	u16 len;
-	u16 opcode;
-	u8 data[0];
-};
-
-
-/* Immediate command Function APIs */
-
-/*
- * ipahal_imm_cmd_name_str() - returns string that represent the imm cmd
- * @cmd_name: [in] Immediate command name
- */
-const char *ipahal_imm_cmd_name_str(enum ipahal_imm_cmd_name cmd_name);
-
-/*
- * ipahal_construct_imm_cmd() - Construct immdiate command
- * This function builds imm cmd bulk that can be be sent to IPA
- * The command will be allocated dynamically.
- * After done using it, call ipahal_destroy_imm_cmd() to release it
- */
-struct ipahal_imm_cmd_pyld *ipahal_construct_imm_cmd(
-	enum ipahal_imm_cmd_name cmd, const void *params, bool is_atomic_ctx);
-
-/*
- * ipahal_construct_nop_imm_cmd() - Construct immediate comamnd for NO-Op
- * Core driver may want functionality to inject NOP commands to IPA
- *  to ensure e.g., PIPLINE clear before someother operation.
- * The functionality given by this function can be reached by
- *  ipahal_construct_imm_cmd(). This function is helper to the core driver
- *  to reach this NOP functionlity easily.
- * @skip_pipline_clear: if to skip pipeline clear waiting (don't wait)
- * @pipline_clr_opt: options for pipeline clear waiting
- * @is_atomic_ctx: is called in atomic context or can sleep?
- */
-struct ipahal_imm_cmd_pyld *ipahal_construct_nop_imm_cmd(
-	bool skip_pipline_clear,
-	enum ipahal_pipeline_clear_option pipline_clr_opt,
-	bool is_atomic_ctx);
-
-/*
- * ipahal_destroy_imm_cmd() - Destroy/Release bulk that was built
- *  by the construction functions
- */
-static inline void ipahal_destroy_imm_cmd(struct ipahal_imm_cmd_pyld *pyld)
-{
-	kfree(pyld);
-}
-
-
-/* IPA Status packet Structures and Function APIs */
-
-/*
- * enum ipahal_pkt_status_opcode - Packet Status Opcode
- * @IPAHAL_STATUS_OPCODE_PACKET_2ND_PASS: Packet Status generated as part of
- *  IPA second processing pass for a packet (i.e. IPA XLAT processing for
- *  the translated packet).
- */
-enum ipahal_pkt_status_opcode {
-	IPAHAL_PKT_STATUS_OPCODE_PACKET = 0,
-	IPAHAL_PKT_STATUS_OPCODE_NEW_FRAG_RULE,
-	IPAHAL_PKT_STATUS_OPCODE_DROPPED_PACKET,
-	IPAHAL_PKT_STATUS_OPCODE_SUSPENDED_PACKET,
-	IPAHAL_PKT_STATUS_OPCODE_LOG,
-	IPAHAL_PKT_STATUS_OPCODE_DCMP,
-	IPAHAL_PKT_STATUS_OPCODE_PACKET_2ND_PASS,
-};
-
-/*
- * enum ipahal_pkt_status_exception - Packet Status exception type
- * @IPAHAL_PKT_STATUS_EXCEPTION_PACKET_LENGTH: formerly IHL exception.
- *
- * Note: IPTYPE, PACKET_LENGTH and PACKET_THRESHOLD exceptions means that
- *  partial / no IP processing took place and corresponding Status Mask
- *  fields should be ignored. Flt and rt info is not valid.
- *
- * NOTE:: Any change to this enum, need to change to
- *	ipahal_pkt_status_exception_to_str array as well.
- */
-enum ipahal_pkt_status_exception {
-	IPAHAL_PKT_STATUS_EXCEPTION_NONE = 0,
-	IPAHAL_PKT_STATUS_EXCEPTION_DEAGGR,
-	IPAHAL_PKT_STATUS_EXCEPTION_IPTYPE,
-	IPAHAL_PKT_STATUS_EXCEPTION_PACKET_LENGTH,
-	IPAHAL_PKT_STATUS_EXCEPTION_PACKET_THRESHOLD,
-	IPAHAL_PKT_STATUS_EXCEPTION_FRAG_RULE_MISS,
-	IPAHAL_PKT_STATUS_EXCEPTION_SW_FILT,
-	/*
-	 * NAT and IPv6CT have the same value at HW.
-	 * NAT for IPv4 and IPv6CT for IPv6 exceptions
-	 */
-	IPAHAL_PKT_STATUS_EXCEPTION_NAT,
-	IPAHAL_PKT_STATUS_EXCEPTION_IPV6CT,
-	IPAHAL_PKT_STATUS_EXCEPTION_CSUM,
-	IPAHAL_PKT_STATUS_EXCEPTION_MAX,
-};
-
-/*
- * enum ipahal_pkt_status_mask - Packet Status bitmask shift values of
- *  the contained flags. This bitmask indicates flags on the properties of
- *  the packet as well as IPA processing it may had.
- * @FRAG_PROCESS: Frag block processing flag: Was pkt processed by frag block?
- *  Also means the frag info is valid unless exception or first frag
- * @FILT_PROCESS: Flt block processing flag: Was pkt processed by flt block?
- *  Also means that flt info is valid.
- * @NAT_PROCESS: NAT block processing flag: Was pkt processed by NAT block?
- *  Also means that NAT info is valid, unless exception.
- * @ROUTE_PROCESS: Rt block processing flag: Was pkt processed by rt block?
- *  Also means that rt info is valid, unless exception.
- * @TAG_VALID: Flag specifying if TAG and TAG info valid?
- * @FRAGMENT: Flag specifying if pkt is IP fragment.
- * @FIRST_FRAGMENT: Flag specifying if pkt is first fragment. In this case, frag
- *  info is invalid
- * @V4: Flag specifying pkt is IPv4 or IPv6
- * @CKSUM_PROCESS: CSUM block processing flag: Was pkt processed by csum block?
- *  If so, csum trailer exists
- * @AGGR_PROCESS: Aggr block processing flag: Was pkt processed by aggr block?
- * @DEST_EOT: Flag specifying if EOT was asserted for the pkt on dest endp
- * @DEAGGR_PROCESS: Deaggr block processing flag: Was pkt processed by deaggr
- *  block?
- * @DEAGG_FIRST: Flag specifying if this is the first pkt in deaggr frame
- * @SRC_EOT: Flag specifying if EOT asserted by src endp when sending the buffer
- * @PREV_EOT: Flag specifying if EOT was sent just before the pkt as part of
- *  aggr hard-byte-limit
- * @BYTE_LIMIT: Flag specifying if pkt is over a configured byte limit.
- */
-enum ipahal_pkt_status_mask {
-	IPAHAL_PKT_STATUS_MASK_FRAG_PROCESS_SHFT = 0,
-	IPAHAL_PKT_STATUS_MASK_FILT_PROCESS_SHFT,
-	IPAHAL_PKT_STATUS_MASK_NAT_PROCESS_SHFT,
-	IPAHAL_PKT_STATUS_MASK_ROUTE_PROCESS_SHFT,
-	IPAHAL_PKT_STATUS_MASK_TAG_VALID_SHFT,
-	IPAHAL_PKT_STATUS_MASK_FRAGMENT_SHFT,
-	IPAHAL_PKT_STATUS_MASK_FIRST_FRAGMENT_SHFT,
-	IPAHAL_PKT_STATUS_MASK_V4_SHFT,
-	IPAHAL_PKT_STATUS_MASK_CKSUM_PROCESS_SHFT,
-	IPAHAL_PKT_STATUS_MASK_AGGR_PROCESS_SHFT,
-	IPAHAL_PKT_STATUS_MASK_DEST_EOT_SHFT,
-	IPAHAL_PKT_STATUS_MASK_DEAGGR_PROCESS_SHFT,
-	IPAHAL_PKT_STATUS_MASK_DEAGG_FIRST_SHFT,
-	IPAHAL_PKT_STATUS_MASK_SRC_EOT_SHFT,
-	IPAHAL_PKT_STATUS_MASK_PREV_EOT_SHFT,
-	IPAHAL_PKT_STATUS_MASK_BYTE_LIMIT_SHFT,
-};
-
-/*
- * Returns boolean value representing a property of the a packet.
- * @__flag_shft: The shift value of the flag of the status bitmask of
- * @__status: Pointer to abstracrted status structure
- *  the needed property. See enum ipahal_pkt_status_mask
- */
-#define IPAHAL_PKT_STATUS_MASK_FLAG_VAL(__flag_shft, __status) \
-	(((__status)->status_mask) & ((u32)0x1<<(__flag_shft)) ? true : false)
-
-/*
- * enum ipahal_pkt_status_nat_type - Type of NAT
- */
-enum ipahal_pkt_status_nat_type {
-	IPAHAL_PKT_STATUS_NAT_NONE,
-	IPAHAL_PKT_STATUS_NAT_SRC,
-	IPAHAL_PKT_STATUS_NAT_DST,
-};
-
-/*
- * struct ipahal_pkt_status - IPA status packet abstracted payload.
- *  This structure describes the status packet fields for the
- *   following statuses: IPA_STATUS_PACKET, IPA_STATUS_DROPPED_PACKET,
- *   IPA_STATUS_SUSPENDED_PACKET.
- *  Other statuses types has different status packet structure.
- * @tag_info: S/W defined value provided via immediate command
- * @status_opcode: The Type of the status (Opcode).
- * @exception: The first exception that took place.
- *  In case of exception, src endp and pkt len are always valid.
- * @status_mask: Bit mask for flags on several properties on the packet
- *  and processing it may passed at IPA. See enum ipahal_pkt_status_mask
- * @pkt_len: Pkt pyld len including hdr and retained hdr if used. Does
- *  not include padding or checksum trailer len.
- * @metadata: meta data value used by packet
- * @flt_local: Filter table location flag: Does matching flt rule belongs to
- *  flt tbl that resides in lcl memory? (if not, then system mem)
- * @flt_hash: Filter hash hit flag: Does matching flt rule was in hash tbl?
- * @flt_global: Global filter rule flag: Does matching flt rule belongs to
- *  the global flt tbl? (if not, then the per endp tables)
- * @flt_ret_hdr: Retain header in filter rule flag: Does matching flt rule
- *  specifies to retain header?
- *  Starting IPA4.5, this will be true only if packet has L2 header.
- * @flt_miss: Filtering miss flag: Was their a filtering rule miss?
- *   In case of miss, all flt info to be ignored
- * @rt_local: Route table location flag: Does matching rt rule belongs to
- *  rt tbl that resides in lcl memory? (if not, then system mem)
- * @rt_hash: Route hash hit flag: Does matching rt rule was in hash tbl?
- * @ucp: UC Processing flag
- * @rt_miss: Routing miss flag: Was their a routing rule miss?
- * @nat_hit: NAT hit flag: Was their NAT hit?
- * @nat_type: Defines the type of the NAT operation:
- * @time_of_day_ctr: running counter from IPA clock
- * @hdr_local: Header table location flag: In header insertion, was the header
- *  taken from the table resides in local memory? (If no, then system mem)
- * @frag_hit: Frag hit flag: Was their frag rule hit in H/W frag table?
- * @flt_rule_id: The ID of the matching filter rule (if no miss).
- *  This info can be combined with endp_src_idx to locate the exact rule.
- * @rt_rule_id: The ID of the matching rt rule. (if no miss). This info
- *  can be combined with rt_tbl_idx to locate the exact rule.
- * @nat_entry_idx: Index of the NAT entry used of NAT processing
- * @hdr_offset: Offset of used header in the header table
- * @endp_src_idx: Source end point index.
- * @endp_dest_idx: Destination end point index.
- *  Not valid in case of exception
- * @rt_tbl_idx: Index of rt tbl that contains the rule on which was a match
- * @seq_num: Per source endp unique packet sequence number
- * @frag_rule: Frag rule index in H/W frag table in case of frag hit
- */
-struct ipahal_pkt_status {
-	u64 tag_info;
-	enum ipahal_pkt_status_opcode status_opcode;
-	enum ipahal_pkt_status_exception exception;
-	u32 status_mask;
-	u32 pkt_len;
-	u32 metadata;
-	bool flt_local;
-	bool flt_hash;
-	bool flt_global;
-	bool flt_ret_hdr;
-	bool flt_miss;
-	bool rt_local;
-	bool rt_hash;
-	bool ucp;
-	bool rt_miss;
-	bool nat_hit;
-	enum ipahal_pkt_status_nat_type nat_type;
-	u32 time_of_day_ctr;
-	bool hdr_local;
-	bool frag_hit;
-	u16 flt_rule_id;
-	u16 rt_rule_id;
-	u16 nat_entry_idx;
-	u16 hdr_offset;
-	u8 endp_src_idx;
-	u8 endp_dest_idx;
-	u8 rt_tbl_idx;
-	u8 seq_num;
-	u8 frag_rule;
-};
-
-/*
- * ipahal_pkt_status_get_size() - Get H/W size of packet status
- */
-u32 ipahal_pkt_status_get_size(void);
-
-/*
- * ipahal_pkt_status_parse() - Parse Packet Status payload to abstracted form
- * @unparsed_status: Pointer to H/W format of the packet status as read from H/W
- * @status: Pointer to pre-allocated buffer where the parsed info will be stored
- */
-void ipahal_pkt_status_parse(const void *unparsed_status,
-	struct ipahal_pkt_status *status);
-
-/*
- * ipahal_pkt_status_exception_str() - returns string represents exception type
- * @exception: [in] The exception type
- */
-const char *ipahal_pkt_status_exception_str(
-	enum ipahal_pkt_status_exception exception);
-
-/*
- * ipahal_cp_hdr_to_hw_buff() - copy header to hardware buffer according to
- * base address and offset given.
- * @base: dma base address
- * @offset: offset from base address where the data will be copied
- * @hdr: the header to be copied
- * @hdr_len: the length of the header
- */
-void ipahal_cp_hdr_to_hw_buff(void *base, u32 offset, u8 *hdr, u32 hdr_len);
-
-/*
- * ipahal_cp_proc_ctx_to_hw_buff() - copy processing context to
- * base address and offset given.
- * @type: type of header processing context
- * @base: dma base address
- * @offset: offset from base address where the data will be copied
- * @hdr_len: the length of the header
- * @is_hdr_proc_ctx: header is located in phys_base (true) or hdr_base_addr
- * @phys_base: memory location in DDR
- * @hdr_base_addr: base address in table
- * @offset_entry: offset from hdr_base_addr in table
- * @l2tp_params: l2tp parameters
- * @is_64: Indicates whether header base address/dma base address is 64 bit.
- */
-int ipahal_cp_proc_ctx_to_hw_buff(enum ipa_hdr_proc_type type,
-		void *base, u32 offset, u32 hdr_len,
-		bool is_hdr_proc_ctx, dma_addr_t phys_base,
-		u64 hdr_base_addr,
-		struct ipa_hdr_offset_entry *offset_entry,
-		struct ipa_l2tp_hdr_proc_ctx_params l2tp_params, bool is_64);
-
-/*
- * ipahal_get_proc_ctx_needed_len() - calculates the needed length for addition
- * of header processing context according to the type of processing context
- * @type: header processing context type (no processing context,
- *	IPA_HDR_PROC_ETHII_TO_ETHII etc.)
- */
-int ipahal_get_proc_ctx_needed_len(enum ipa_hdr_proc_type type);
-
-int ipahal_init(enum ipa_hw_type ipa_hw_type, void __iomem *base,
-	struct device *ipa_pdev);
-void ipahal_destroy(void);
-void ipahal_free_dma_mem(struct ipa_mem_buffer *mem);
-
-#endif /* _IPAHAL_H_ */

ipa/ipa_v3/ipahal/ipahal_fltrt.c

@@ -1,4372 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2012-2019, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/ipa.h>
-#include <linux/errno.h>
-#include <linux/ipc_logging.h>
-#include <linux/debugfs.h>
-#include "ipahal.h"
-#include "ipahal_fltrt.h"
-#include "ipahal_fltrt_i.h"
-#include "ipahal_i.h"
-#include "../../ipa_common_i.h"
-
-/*
- * struct ipahal_fltrt_obj - Flt/Rt H/W information for specific IPA version
- * @support_hash: Is hashable tables supported
- * @tbl_width: Width of table in bytes
- * @sysaddr_alignment: System table address alignment
- * @lcladdr_alignment: Local table offset alignment
- * @blk_sz_alignment: Rules block size alignment
- * @rule_start_alignment: Rule start address alignment
- * @tbl_hdr_width: Width of the header structure in bytes
- * @tbl_addr_mask: Masking for Table address
- * @rule_max_prio: Max possible priority of a rule
- * @rule_min_prio: Min possible priority of a rule
- * @low_rule_id: Low value of Rule ID that can be used
- * @rule_id_bit_len: Rule is high (MSB) bit len
- * @rule_buf_size: Max size rule may utilize.
- * @write_val_to_hdr: Write address or offset to header entry
- * @create_flt_bitmap: Create bitmap in H/W format using given bitmap
- * @create_tbl_addr: Given raw table address, create H/W formated one
- * @parse_tbl_addr: Parse the given H/W address (hdr format)
- * @rt_generate_hw_rule: Generate RT rule in H/W format
- * @flt_generate_hw_rule: Generate FLT rule in H/W format
- * @flt_generate_eq: Generate flt equation attributes from rule attributes
- * @rt_parse_hw_rule: Parse rt rule read from H/W
- * @flt_parse_hw_rule: Parse flt rule read from H/W
- * @eq_bitfield: Array of the bit fields of the support equations.
- *	0xFF means the equation is not supported
- */
-struct ipahal_fltrt_obj {
-	bool support_hash;
-	u32 tbl_width;
-	u32 sysaddr_alignment;
-	u32 lcladdr_alignment;
-	u32 blk_sz_alignment;
-	u32 rule_start_alignment;
-	u32 tbl_hdr_width;
-	u32 tbl_addr_mask;
-	int rule_max_prio;
-	int rule_min_prio;
-	u32 low_rule_id;
-	u32 rule_id_bit_len;
-	u32 rule_buf_size;
-	u8* (*write_val_to_hdr)(u64 val, u8 *hdr);
-	u64 (*create_flt_bitmap)(u64 ep_bitmap);
-	u64 (*create_tbl_addr)(bool is_sys, u64 addr);
-	void (*parse_tbl_addr)(u64 hwaddr, u64 *addr, bool *is_sys);
-	int (*rt_generate_hw_rule)(struct ipahal_rt_rule_gen_params *params,
-		u32 *hw_len, u8 *buf);
-	int (*flt_generate_hw_rule)(struct ipahal_flt_rule_gen_params *params,
-		u32 *hw_len, u8 *buf);
-	int (*flt_generate_eq)(enum ipa_ip_type ipt,
-		const struct ipa_rule_attrib *attrib,
-		struct ipa_ipfltri_rule_eq *eq_atrb);
-	int (*rt_parse_hw_rule)(u8 *addr, struct ipahal_rt_rule_entry *rule);
-	int (*flt_parse_hw_rule)(u8 *addr, struct ipahal_flt_rule_entry *rule);
-	u8 eq_bitfield[IPA_EQ_MAX];
-};
-
-
-static u64 ipa_fltrt_create_flt_bitmap(u64 ep_bitmap)
-{
-	/* At IPA3, there global configuration is possible but not used */
-	return (ep_bitmap << 1) & ~0x1;
-}
-
-static u64 ipa_fltrt_create_tbl_addr(bool is_sys, u64 addr)
-{
-	if (is_sys) {
-		if (addr & IPA3_0_HW_TBL_SYSADDR_ALIGNMENT) {
-			IPAHAL_ERR(
-				"sys addr is not aligned accordingly addr=0x%pad\n",
-				&addr);
-			ipa_assert();
-			return 0;
-		}
-	} else {
-		if (addr & IPA3_0_HW_TBL_LCLADDR_ALIGNMENT) {
-			IPAHAL_ERR("addr/ofst isn't lcl addr aligned %llu\n",
-				addr);
-			ipa_assert();
-			return 0;
-		}
-		/*
-		 * for local tables (at sram) offsets is used as tables
-		 * addresses. offset need to be in 8B units
-		 * (local address aligned) and left shifted to its place.
-		 * Local bit need to be enabled.
-		 */
-		addr /= IPA3_0_HW_TBL_LCLADDR_ALIGNMENT + 1;
-		addr *= IPA3_0_HW_TBL_ADDR_MASK + 1;
-		addr += 1;
-	}
-
-	return addr;
-}
-
-static void ipa_fltrt_parse_tbl_addr(u64 hwaddr, u64 *addr, bool *is_sys)
-{
-	IPAHAL_DBG_LOW("Parsing hwaddr 0x%llx\n", hwaddr);
-
-	*is_sys = !(hwaddr & 0x1);
-	hwaddr &= (~0ULL - 1);
-	if (hwaddr & IPA3_0_HW_TBL_SYSADDR_ALIGNMENT) {
-		IPAHAL_ERR(
-			"sys addr is not aligned accordingly addr=0x%pad\n",
-			&hwaddr);
-		ipa_assert();
-		return;
-	}
-
-	if (!*is_sys) {
-		hwaddr /= IPA3_0_HW_TBL_ADDR_MASK + 1;
-		hwaddr *= IPA3_0_HW_TBL_LCLADDR_ALIGNMENT + 1;
-	}
-
-	*addr = hwaddr;
-}
-
-/* Update these tables of the number of equations changes */
-static const int ipa3_0_ofst_meq32[] = { IPA_OFFSET_MEQ32_0,
-					IPA_OFFSET_MEQ32_1};
-static const int ipa3_0_ofst_meq128[] = { IPA_OFFSET_MEQ128_0,
-					IPA_OFFSET_MEQ128_1};
-static const int ipa3_0_ihl_ofst_rng16[] = { IPA_IHL_OFFSET_RANGE16_0,
-					IPA_IHL_OFFSET_RANGE16_1};
-static const int ipa3_0_ihl_ofst_meq32[] = { IPA_IHL_OFFSET_MEQ32_0,
-					IPA_IHL_OFFSET_MEQ32_1};
-
-static int ipa_fltrt_generate_hw_rule_bdy(enum ipa_ip_type ipt,
-	const struct ipa_rule_attrib *attrib, u8 **buf, u16 *en_rule);
-static int ipa_fltrt_generate_hw_rule_bdy_from_eq(
-		const struct ipa_ipfltri_rule_eq *attrib, u8 **buf);
-static int ipa_flt_generate_eq_ip4(enum ipa_ip_type ip,
-		const struct ipa_rule_attrib *attrib,
-		struct ipa_ipfltri_rule_eq *eq_atrb);
-static int ipa_flt_generate_eq_ip6(enum ipa_ip_type ip,
-		const struct ipa_rule_attrib *attrib,
-		struct ipa_ipfltri_rule_eq *eq_atrb);
-static int ipa_flt_generate_eq(enum ipa_ip_type ipt,
-		const struct ipa_rule_attrib *attrib,
-		struct ipa_ipfltri_rule_eq *eq_atrb);
-static int ipa_rt_parse_hw_rule(u8 *addr,
-		struct ipahal_rt_rule_entry *rule);
-static int ipa_rt_parse_hw_rule_ipav4_5(u8 *addr,
-		struct ipahal_rt_rule_entry *rule);
-static int ipa_flt_parse_hw_rule(u8 *addr,
-		struct ipahal_flt_rule_entry *rule);
-static int ipa_flt_parse_hw_rule_ipav4(u8 *addr,
-		struct ipahal_flt_rule_entry *rule);
-static int ipa_flt_parse_hw_rule_ipav4_5(u8 *addr,
-	struct ipahal_flt_rule_entry *rule);
-
-#define IPA_IS_RAN_OUT_OF_EQ(__eq_array, __eq_index) \
-	(ARRAY_SIZE(__eq_array) <= (__eq_index))
-
-#define IPA_GET_RULE_EQ_BIT_PTRN(__eq) \
-	(BIT(ipahal_fltrt_objs[ipahal_ctx->hw_type].eq_bitfield[(__eq)]))
-
-#define IPA_IS_RULE_EQ_VALID(__eq) \
-	(ipahal_fltrt_objs[ipahal_ctx->hw_type].eq_bitfield[(__eq)] != 0xFF)
-
-/*
- * ipa_fltrt_rule_generation_err_check() - check basic validity on the rule
- *  attribs before starting building it
- *  checks if not not using ipv4 attribs on ipv6 and vice-versa
- * @ip: IP address type
- * @attrib: IPA rule attribute
- *
- * Return: 0 on success, -EPERM on failure
- */
-static int ipa_fltrt_rule_generation_err_check(
-	enum ipa_ip_type ipt, const struct ipa_rule_attrib *attrib)
-{
-	if (ipt == IPA_IP_v4) {
-		if (attrib->attrib_mask & IPA_FLT_NEXT_HDR ||
-		    attrib->attrib_mask & IPA_FLT_TC ||
-		    attrib->attrib_mask & IPA_FLT_FLOW_LABEL) {
-			IPAHAL_ERR_RL("v6 attrib's specified for v4 rule\n");
-			return -EPERM;
-		}
-	} else if (ipt == IPA_IP_v6) {
-		if (attrib->attrib_mask & IPA_FLT_TOS ||
-		    attrib->attrib_mask & IPA_FLT_PROTOCOL) {
-			IPAHAL_ERR_RL("v4 attrib's specified for v6 rule\n");
-			return -EPERM;
-		}
-	} else {
-		IPAHAL_ERR_RL("unsupported ip %d\n", ipt);
-		return -EPERM;
-	}
-
-	return 0;
-}
-
-static int ipa_rt_gen_hw_rule(struct ipahal_rt_rule_gen_params *params,
-	u32 *hw_len, u8 *buf)
-{
-	struct ipa3_0_rt_rule_hw_hdr *rule_hdr;
-	u8 *start;
-	u16 en_rule = 0;
-
-	start = buf;
-	rule_hdr = (struct ipa3_0_rt_rule_hw_hdr *)buf;
-
-	ipa_assert_on(params->dst_pipe_idx & ~0x1F);
-	rule_hdr->u.hdr.pipe_dest_idx = params->dst_pipe_idx;
-	switch (params->hdr_type) {
-	case IPAHAL_RT_RULE_HDR_PROC_CTX:
-		rule_hdr->u.hdr.system = !params->hdr_lcl;
-		rule_hdr->u.hdr.proc_ctx = 1;
-		ipa_assert_on(params->hdr_ofst & 31);
-		rule_hdr->u.hdr.hdr_offset = (params->hdr_ofst) >> 5;
-		break;
-	case IPAHAL_RT_RULE_HDR_RAW:
-		rule_hdr->u.hdr.system = !params->hdr_lcl;
-		rule_hdr->u.hdr.proc_ctx = 0;
-		ipa_assert_on(params->hdr_ofst & 3);
-		rule_hdr->u.hdr.hdr_offset = (params->hdr_ofst) >> 2;
-		break;
-	case IPAHAL_RT_RULE_HDR_NONE:
-		rule_hdr->u.hdr.system = !params->hdr_lcl;
-		rule_hdr->u.hdr.proc_ctx = 0;
-		rule_hdr->u.hdr.hdr_offset = 0;
-		break;
-	default:
-		IPAHAL_ERR("Invalid HDR type %d\n", params->hdr_type);
-		WARN_ON_RATELIMIT_IPA(1);
-		return -EINVAL;
-	}
-
-	ipa_assert_on(params->priority & ~0x3FF);
-	rule_hdr->u.hdr.priority = params->priority;
-	rule_hdr->u.hdr.retain_hdr = params->rule->retain_hdr ? 0x1 : 0x0;
-	ipa_assert_on(params->id & ~((1 << IPA3_0_RULE_ID_BIT_LEN) - 1));
-	ipa_assert_on(params->id == ((1 << IPA3_0_RULE_ID_BIT_LEN) - 1));
-	rule_hdr->u.hdr.rule_id = params->id;
-
-	buf += sizeof(struct ipa3_0_rt_rule_hw_hdr);
-
-	if (ipa_fltrt_generate_hw_rule_bdy(params->ipt, &params->rule->attrib,
-		&buf, &en_rule)) {
-		IPAHAL_ERR("fail to generate hw rule\n");
-		return -EPERM;
-	}
-	rule_hdr->u.hdr.en_rule = en_rule;
-
-	IPAHAL_DBG_LOW("en_rule 0x%x\n", en_rule);
-	ipa_write_64(rule_hdr->u.word, (u8 *)rule_hdr);
-
-	if (*hw_len == 0) {
-		*hw_len = buf - start;
-	} else if (*hw_len != (buf - start)) {
-		IPAHAL_ERR("hw_len differs b/w passed=0x%x calc=%td\n",
-			*hw_len, (buf - start));
-		return -EPERM;
-	}
-
-	return 0;
-}
-
-static int ipa_rt_gen_hw_rule_ipav4_5(struct ipahal_rt_rule_gen_params *params,
-	u32 *hw_len, u8 *buf)
-{
-	struct ipa4_5_rt_rule_hw_hdr *rule_hdr;
-	u8 *start;
-	u16 en_rule = 0;
-
-	start = buf;
-	rule_hdr = (struct ipa4_5_rt_rule_hw_hdr *)buf;
-
-	ipa_assert_on(params->dst_pipe_idx & ~0x1F);
-	rule_hdr->u.hdr.pipe_dest_idx = params->dst_pipe_idx;
-	switch (params->hdr_type) {
-	case IPAHAL_RT_RULE_HDR_PROC_CTX:
-		rule_hdr->u.hdr.system = !params->hdr_lcl;
-		rule_hdr->u.hdr.proc_ctx = 1;
-		ipa_assert_on(params->hdr_ofst & 31);
-		rule_hdr->u.hdr.hdr_offset = (params->hdr_ofst) >> 5;
-		break;
-	case IPAHAL_RT_RULE_HDR_RAW:
-		rule_hdr->u.hdr.system = !params->hdr_lcl;
-		rule_hdr->u.hdr.proc_ctx = 0;
-		ipa_assert_on(params->hdr_ofst & 3);
-		rule_hdr->u.hdr.hdr_offset = (params->hdr_ofst) >> 2;
-		break;
-	case IPAHAL_RT_RULE_HDR_NONE:
-		rule_hdr->u.hdr.system = !params->hdr_lcl;
-		rule_hdr->u.hdr.proc_ctx = 0;
-		rule_hdr->u.hdr.hdr_offset = 0;
-		break;
-	default:
-		IPAHAL_ERR("Invalid HDR type %d\n", params->hdr_type);
-		WARN_ON_RATELIMIT_IPA(1);
-		return -EINVAL;
-	}
-
-	ipa_assert_on(params->priority & ~0x3FF);
-	rule_hdr->u.hdr.priority = params->priority;
-	rule_hdr->u.hdr.retain_hdr = params->rule->retain_hdr ? 0x1 : 0x0;
-	ipa_assert_on(params->id & ~((1 << IPA3_0_RULE_ID_BIT_LEN) - 1));
-	ipa_assert_on(params->id == ((1 << IPA3_0_RULE_ID_BIT_LEN) - 1));
-	rule_hdr->u.hdr.rule_id = params->id;
-	rule_hdr->u.hdr.stats_cnt_idx_lsb = params->cnt_idx & 0x3F;
-	rule_hdr->u.hdr.stats_cnt_idx_msb = (params->cnt_idx & 0xC0) >> 6;
-
-	buf += sizeof(struct ipa4_5_rt_rule_hw_hdr);
-
-	if (ipa_fltrt_generate_hw_rule_bdy(params->ipt, &params->rule->attrib,
-		&buf, &en_rule)) {
-		IPAHAL_ERR("fail to generate hw rule\n");
-		return -EPERM;
-	}
-	rule_hdr->u.hdr.en_rule = en_rule;
-
-	IPAHAL_DBG_LOW("en_rule 0x%x\n", en_rule);
-	ipa_write_64(rule_hdr->u.word, (u8 *)rule_hdr);
-
-	if (*hw_len == 0) {
-		*hw_len = buf - start;
-	} else if (*hw_len != (buf - start)) {
-		IPAHAL_ERR("hw_len differs b/w passed=0x%x calc=%td\n",
-			*hw_len, (buf - start));
-		return -EPERM;
-	}
-
-	return 0;
-}
-
-static int ipa_flt_gen_hw_rule(struct ipahal_flt_rule_gen_params *params,
-	u32 *hw_len, u8 *buf)
-{
-	struct ipa3_0_flt_rule_hw_hdr *rule_hdr;
-	u8 *start;
-	u16 en_rule = 0;
-
-	start = buf;
-	rule_hdr = (struct ipa3_0_flt_rule_hw_hdr *)buf;
-
-	switch (params->rule->action) {
-	case IPA_PASS_TO_ROUTING:
-		rule_hdr->u.hdr.action = 0x0;
-		break;
-	case IPA_PASS_TO_SRC_NAT:
-		rule_hdr->u.hdr.action = 0x1;
-		break;
-	case IPA_PASS_TO_DST_NAT:
-		rule_hdr->u.hdr.action = 0x2;
-		break;
-	case IPA_PASS_TO_EXCEPTION:
-		rule_hdr->u.hdr.action = 0x3;
-		break;
-	default:
-		IPAHAL_ERR_RL("Invalid Rule Action %d\n", params->rule->action);
-		WARN_ON_RATELIMIT_IPA(1);
-		return -EINVAL;
-	}
-	ipa_assert_on(params->rt_tbl_idx & ~0x1F);
-	rule_hdr->u.hdr.rt_tbl_idx = params->rt_tbl_idx;
-	rule_hdr->u.hdr.retain_hdr = params->rule->retain_hdr ? 0x1 : 0x0;
-	rule_hdr->u.hdr.rsvd1 = 0;
-	rule_hdr->u.hdr.rsvd2 = 0;
-	rule_hdr->u.hdr.rsvd3 = 0;
-
-	ipa_assert_on(params->priority & ~0x3FF);
-	rule_hdr->u.hdr.priority = params->priority;
-	ipa_assert_on(params->id & ~((1 << IPA3_0_RULE_ID_BIT_LEN) - 1));
-	ipa_assert_on(params->id == ((1 << IPA3_0_RULE_ID_BIT_LEN) - 1));
-	rule_hdr->u.hdr.rule_id = params->id;
-
-	buf += sizeof(struct ipa3_0_flt_rule_hw_hdr);
-
-	if (params->rule->eq_attrib_type) {
-		if (ipa_fltrt_generate_hw_rule_bdy_from_eq(
-			&params->rule->eq_attrib, &buf)) {
-			IPAHAL_ERR_RL("fail to generate hw rule from eq\n");
-			return -EPERM;
-		}
-		en_rule = params->rule->eq_attrib.rule_eq_bitmap;
-	} else {
-		if (ipa_fltrt_generate_hw_rule_bdy(params->ipt,
-			&params->rule->attrib, &buf, &en_rule)) {
-			IPAHAL_ERR_RL("fail to generate hw rule\n");
-			return -EPERM;
-		}
-	}
-	rule_hdr->u.hdr.en_rule = en_rule;
-
-	IPAHAL_DBG_LOW("en_rule=0x%x, action=%d, rt_idx=%d, retain_hdr=%d\n",
-		en_rule,
-		rule_hdr->u.hdr.action,
-		rule_hdr->u.hdr.rt_tbl_idx,
-		rule_hdr->u.hdr.retain_hdr);
-	IPAHAL_DBG_LOW("priority=%d, rule_id=%d\n",
-		rule_hdr->u.hdr.priority,
-		rule_hdr->u.hdr.rule_id);
-
-	ipa_write_64(rule_hdr->u.word, (u8 *)rule_hdr);
-
-	if (*hw_len == 0) {
-		*hw_len = buf - start;
-	} else if (*hw_len != (buf - start)) {
-		IPAHAL_ERR_RL("hw_len differs b/w passed=0x%x calc=%td\n",
-			*hw_len, (buf - start));
-		return -EPERM;
-	}
-
-	return 0;
-}
-
-static int ipa_flt_gen_hw_rule_ipav4(struct ipahal_flt_rule_gen_params *params,
-	u32 *hw_len, u8 *buf)
-{
-	struct ipa4_0_flt_rule_hw_hdr *rule_hdr;
-	u8 *start;
-	u16 en_rule = 0;
-
-	start = buf;
-	rule_hdr = (struct ipa4_0_flt_rule_hw_hdr *)buf;
-
-	switch (params->rule->action) {
-	case IPA_PASS_TO_ROUTING:
-		rule_hdr->u.hdr.action = 0x0;
-		break;
-	case IPA_PASS_TO_SRC_NAT:
-		rule_hdr->u.hdr.action = 0x1;
-		break;
-	case IPA_PASS_TO_DST_NAT:
-		rule_hdr->u.hdr.action = 0x2;
-		break;
-	case IPA_PASS_TO_EXCEPTION:
-		rule_hdr->u.hdr.action = 0x3;
-		break;
-	default:
-		IPAHAL_ERR("Invalid Rule Action %d\n", params->rule->action);
-		WARN_ON_RATELIMIT_IPA(1);
-		return -EINVAL;
-	}
-
-	ipa_assert_on(params->rt_tbl_idx & ~0x1F);
-	rule_hdr->u.hdr.rt_tbl_idx = params->rt_tbl_idx;
-	rule_hdr->u.hdr.retain_hdr = params->rule->retain_hdr ? 0x1 : 0x0;
-
-	ipa_assert_on(params->rule->pdn_idx & ~0xF);
-	rule_hdr->u.hdr.pdn_idx = params->rule->pdn_idx;
-	rule_hdr->u.hdr.set_metadata = params->rule->set_metadata;
-	rule_hdr->u.hdr.rsvd2 = 0;
-	rule_hdr->u.hdr.rsvd3 = 0;
-
-	ipa_assert_on(params->priority & ~0x3FF);
-	rule_hdr->u.hdr.priority = params->priority;
-	ipa_assert_on(params->id & ~((1 << IPA3_0_RULE_ID_BIT_LEN) - 1));
-	ipa_assert_on(params->id == ((1 << IPA3_0_RULE_ID_BIT_LEN) - 1));
-	rule_hdr->u.hdr.rule_id = params->id;
-
-	buf += sizeof(struct ipa4_0_flt_rule_hw_hdr);
-
-	if (params->rule->eq_attrib_type) {
-		if (ipa_fltrt_generate_hw_rule_bdy_from_eq(
-			&params->rule->eq_attrib, &buf)) {
-			IPAHAL_ERR("fail to generate hw rule from eq\n");
-			return -EPERM;
-		}
-		en_rule = params->rule->eq_attrib.rule_eq_bitmap;
-	} else {
-		if (ipa_fltrt_generate_hw_rule_bdy(params->ipt,
-			&params->rule->attrib, &buf, &en_rule)) {
-			IPAHAL_ERR("fail to generate hw rule\n");
-			return -EPERM;
-		}
-	}
-	rule_hdr->u.hdr.en_rule = en_rule;
-
-	IPAHAL_DBG_LOW("en_rule=0x%x, action=%d, rt_idx=%d, retain_hdr=%d\n",
-		en_rule,
-		rule_hdr->u.hdr.action,
-		rule_hdr->u.hdr.rt_tbl_idx,
-		rule_hdr->u.hdr.retain_hdr);
-	IPAHAL_DBG_LOW("priority=%d, rule_id=%d, pdn=%d, set_metadata=%d\n",
-		rule_hdr->u.hdr.priority,
-		rule_hdr->u.hdr.rule_id,
-		rule_hdr->u.hdr.pdn_idx,
-		rule_hdr->u.hdr.set_metadata);
-
-	ipa_write_64(rule_hdr->u.word, (u8 *)rule_hdr);
-
-	if (*hw_len == 0) {
-		*hw_len = buf - start;
-	} else if (*hw_len != (buf - start)) {
-		IPAHAL_ERR("hw_len differs b/w passed=0x%x calc=%td\n",
-			*hw_len, (buf - start));
-		return -EPERM;
-	}
-
-	return 0;
-}
-
-static int ipa_flt_gen_hw_rule_ipav4_5(
-	struct ipahal_flt_rule_gen_params *params,
-	u32 *hw_len, u8 *buf)
-{
-	struct ipa4_5_flt_rule_hw_hdr *rule_hdr;
-	u8 *start;
-	u16 en_rule = 0;
-
-	start = buf;
-	rule_hdr = (struct ipa4_5_flt_rule_hw_hdr *)buf;
-
-	switch (params->rule->action) {
-	case IPA_PASS_TO_ROUTING:
-		rule_hdr->u.hdr.action = 0x0;
-		break;
-	case IPA_PASS_TO_SRC_NAT:
-		rule_hdr->u.hdr.action = 0x1;
-		break;
-	case IPA_PASS_TO_DST_NAT:
-		rule_hdr->u.hdr.action = 0x2;
-		break;
-	case IPA_PASS_TO_EXCEPTION:
-		rule_hdr->u.hdr.action = 0x3;
-		break;
-	default:
-		IPAHAL_ERR("Invalid Rule Action %d\n", params->rule->action);
-		WARN_ON_RATELIMIT_IPA(1);
-		return -EINVAL;
-	}
-
-	ipa_assert_on(params->rt_tbl_idx & ~0x1F);
-	rule_hdr->u.hdr.rt_tbl_idx = params->rt_tbl_idx;
-	rule_hdr->u.hdr.retain_hdr = params->rule->retain_hdr ? 0x1 : 0x0;
-
-	ipa_assert_on(params->rule->pdn_idx & ~0xF);
-	rule_hdr->u.hdr.pdn_idx = params->rule->pdn_idx;
-	rule_hdr->u.hdr.set_metadata = params->rule->set_metadata;
-	rule_hdr->u.hdr.rsvd2 = 0;
-
-	ipa_assert_on(params->priority & ~0x3FF);
-	rule_hdr->u.hdr.priority = params->priority;
-	ipa_assert_on(params->id & ~((1 << IPA3_0_RULE_ID_BIT_LEN) - 1));
-	ipa_assert_on(params->id == ((1 << IPA3_0_RULE_ID_BIT_LEN) - 1));
-	rule_hdr->u.hdr.rule_id = params->id;
-	rule_hdr->u.hdr.stats_cnt_idx_lsb = params->cnt_idx & 0x3F;
-	rule_hdr->u.hdr.stats_cnt_idx_msb = (params->cnt_idx & 0xC0) >> 6;
-
-	buf += sizeof(struct ipa4_5_flt_rule_hw_hdr);
-
-	if (params->rule->eq_attrib_type) {
-		if (ipa_fltrt_generate_hw_rule_bdy_from_eq(
-			&params->rule->eq_attrib, &buf)) {
-			IPAHAL_ERR("fail to generate hw rule from eq\n");
-			return -EPERM;
-		}
-		en_rule = params->rule->eq_attrib.rule_eq_bitmap;
-	} else {
-		if (ipa_fltrt_generate_hw_rule_bdy(params->ipt,
-			&params->rule->attrib, &buf, &en_rule)) {
-			IPAHAL_ERR("fail to generate hw rule\n");
-			return -EPERM;
-		}
-	}
-	rule_hdr->u.hdr.en_rule = en_rule;
-
-	IPAHAL_DBG_LOW("en_rule=0x%x, action=%d, rt_idx=%d, retain_hdr=%d\n",
-		en_rule,
-		rule_hdr->u.hdr.action,
-		rule_hdr->u.hdr.rt_tbl_idx,
-		rule_hdr->u.hdr.retain_hdr);
-	IPAHAL_DBG_LOW("priority=%d, rule_id=%d, pdn=%d, set_metadata=%d\n",
-		rule_hdr->u.hdr.priority,
-		rule_hdr->u.hdr.rule_id,
-		rule_hdr->u.hdr.pdn_idx,
-		rule_hdr->u.hdr.set_metadata);
-
-	ipa_write_64(rule_hdr->u.word, (u8 *)rule_hdr);
-
-	if (*hw_len == 0) {
-		*hw_len = buf - start;
-	} else if (*hw_len != (buf - start)) {
-		IPAHAL_ERR("hw_len differs b/w passed=0x%x calc=%td\n",
-			*hw_len, (buf - start));
-		return -EPERM;
-	}
-
-	return 0;
-}
-
-/*
- * This array contains the FLT/RT info for IPAv3 and later.
- * All the information on IPAv3 are statically defined below.
- * If information is missing regarding on some IPA version,
- *  the init function will fill it with the information from the previous
- *  IPA version.
- * Information is considered missing if all of the fields are 0.
- */
-static struct ipahal_fltrt_obj ipahal_fltrt_objs[IPA_HW_MAX] = {
-	/* IPAv3 */
-	[IPA_HW_v3_0] = {
-		true,
-		IPA3_0_HW_TBL_WIDTH,
-		IPA3_0_HW_TBL_SYSADDR_ALIGNMENT,
-		IPA3_0_HW_TBL_LCLADDR_ALIGNMENT,
-		IPA3_0_HW_TBL_BLK_SIZE_ALIGNMENT,
-		IPA3_0_HW_RULE_START_ALIGNMENT,
-		IPA3_0_HW_TBL_HDR_WIDTH,
-		IPA3_0_HW_TBL_ADDR_MASK,
-		IPA3_0_RULE_MAX_PRIORITY,
-		IPA3_0_RULE_MIN_PRIORITY,
-		IPA3_0_LOW_RULE_ID,
-		IPA3_0_RULE_ID_BIT_LEN,
-		IPA3_0_HW_RULE_BUF_SIZE,
-		ipa_write_64,
-		ipa_fltrt_create_flt_bitmap,
-		ipa_fltrt_create_tbl_addr,
-		ipa_fltrt_parse_tbl_addr,
-		ipa_rt_gen_hw_rule,
-		ipa_flt_gen_hw_rule,
-		ipa_flt_generate_eq,
-		ipa_rt_parse_hw_rule,
-		ipa_flt_parse_hw_rule,
-		{
-			[IPA_TOS_EQ]			= 0,
-			[IPA_PROTOCOL_EQ]		= 1,
-			[IPA_TC_EQ]			= 2,
-			[IPA_OFFSET_MEQ128_0]		= 3,
-			[IPA_OFFSET_MEQ128_1]		= 4,
-			[IPA_OFFSET_MEQ32_0]		= 5,
-			[IPA_OFFSET_MEQ32_1]		= 6,
-			[IPA_IHL_OFFSET_MEQ32_0]	= 7,
-			[IPA_IHL_OFFSET_MEQ32_1]	= 8,
-			[IPA_METADATA_COMPARE]		= 9,
-			[IPA_IHL_OFFSET_RANGE16_0]	= 10,
-			[IPA_IHL_OFFSET_RANGE16_1]	= 11,
-			[IPA_IHL_OFFSET_EQ_32]		= 12,
-			[IPA_IHL_OFFSET_EQ_16]		= 13,
-			[IPA_FL_EQ]			= 14,
-			[IPA_IS_FRAG]			= 15,
-			[IPA_IS_PURE_ACK]		= 0xFF,
-		},
-	},
-
-	/* IPAv4 */
-	[IPA_HW_v4_0] = {
-		true,
-		IPA3_0_HW_TBL_WIDTH,
-		IPA3_0_HW_TBL_SYSADDR_ALIGNMENT,
-		IPA3_0_HW_TBL_LCLADDR_ALIGNMENT,
-		IPA3_0_HW_TBL_BLK_SIZE_ALIGNMENT,
-		IPA3_0_HW_RULE_START_ALIGNMENT,
-		IPA3_0_HW_TBL_HDR_WIDTH,
-		IPA3_0_HW_TBL_ADDR_MASK,
-		IPA3_0_RULE_MAX_PRIORITY,
-		IPA3_0_RULE_MIN_PRIORITY,
-		IPA3_0_LOW_RULE_ID,
-		IPA3_0_RULE_ID_BIT_LEN,
-		IPA3_0_HW_RULE_BUF_SIZE,
-		ipa_write_64,
-		ipa_fltrt_create_flt_bitmap,
-		ipa_fltrt_create_tbl_addr,
-		ipa_fltrt_parse_tbl_addr,
-		ipa_rt_gen_hw_rule,
-		ipa_flt_gen_hw_rule_ipav4,
-		ipa_flt_generate_eq,
-		ipa_rt_parse_hw_rule,
-		ipa_flt_parse_hw_rule_ipav4,
-		{
-			[IPA_TOS_EQ]			= 0,
-			[IPA_PROTOCOL_EQ]		= 1,
-			[IPA_TC_EQ]			= 2,
-			[IPA_OFFSET_MEQ128_0]		= 3,
-			[IPA_OFFSET_MEQ128_1]		= 4,
-			[IPA_OFFSET_MEQ32_0]		= 5,
-			[IPA_OFFSET_MEQ32_1]		= 6,
-			[IPA_IHL_OFFSET_MEQ32_0]	= 7,
-			[IPA_IHL_OFFSET_MEQ32_1]	= 8,
-			[IPA_METADATA_COMPARE]		= 9,
-			[IPA_IHL_OFFSET_RANGE16_0]	= 10,
-			[IPA_IHL_OFFSET_RANGE16_1]	= 11,
-			[IPA_IHL_OFFSET_EQ_32]		= 12,
-			[IPA_IHL_OFFSET_EQ_16]		= 13,
-			[IPA_FL_EQ]			= 14,
-			[IPA_IS_FRAG]			= 15,
-			[IPA_IS_PURE_ACK]		= 0xFF,
-		},
-	},
-
-	/* IPAv4.2 */
-	[IPA_HW_v4_2] = {
-		false,
-		IPA3_0_HW_TBL_WIDTH,
-		IPA3_0_HW_TBL_SYSADDR_ALIGNMENT,
-		IPA3_0_HW_TBL_LCLADDR_ALIGNMENT,
-		IPA3_0_HW_TBL_BLK_SIZE_ALIGNMENT,
-		IPA3_0_HW_RULE_START_ALIGNMENT,
-		IPA3_0_HW_TBL_HDR_WIDTH,
-		IPA3_0_HW_TBL_ADDR_MASK,
-		IPA3_0_RULE_MAX_PRIORITY,
-		IPA3_0_RULE_MIN_PRIORITY,
-		IPA3_0_LOW_RULE_ID,
-		IPA3_0_RULE_ID_BIT_LEN,
-		IPA3_0_HW_RULE_BUF_SIZE,
-		ipa_write_64,
-		ipa_fltrt_create_flt_bitmap,
-		ipa_fltrt_create_tbl_addr,
-		ipa_fltrt_parse_tbl_addr,
-		ipa_rt_gen_hw_rule,
-		ipa_flt_gen_hw_rule_ipav4,
-		ipa_flt_generate_eq,
-		ipa_rt_parse_hw_rule,
-		ipa_flt_parse_hw_rule_ipav4,
-		{
-			[IPA_TOS_EQ]			= 0,
-			[IPA_PROTOCOL_EQ]		= 1,
-			[IPA_TC_EQ]			= 2,
-			[IPA_OFFSET_MEQ128_0]		= 3,
-			[IPA_OFFSET_MEQ128_1]		= 4,
-			[IPA_OFFSET_MEQ32_0]		= 5,
-			[IPA_OFFSET_MEQ32_1]		= 6,
-			[IPA_IHL_OFFSET_MEQ32_0]	= 7,
-			[IPA_IHL_OFFSET_MEQ32_1]	= 8,
-			[IPA_METADATA_COMPARE]		= 9,
-			[IPA_IHL_OFFSET_RANGE16_0]	= 10,
-			[IPA_IHL_OFFSET_RANGE16_1]	= 11,
-			[IPA_IHL_OFFSET_EQ_32]		= 12,
-			[IPA_IHL_OFFSET_EQ_16]		= 13,
-			[IPA_FL_EQ]			= 14,
-			[IPA_IS_FRAG]			= 15,
-			[IPA_IS_PURE_ACK]		= 0xFF,
-		},
-	},
-
-	/* IPAv4.5 */
-	[IPA_HW_v4_5] = {
-		true,
-		IPA3_0_HW_TBL_WIDTH,
-		IPA3_0_HW_TBL_SYSADDR_ALIGNMENT,
-		IPA3_0_HW_TBL_LCLADDR_ALIGNMENT,
-		IPA3_0_HW_TBL_BLK_SIZE_ALIGNMENT,
-		IPA3_0_HW_RULE_START_ALIGNMENT,
-		IPA3_0_HW_TBL_HDR_WIDTH,
-		IPA3_0_HW_TBL_ADDR_MASK,
-		IPA3_0_RULE_MAX_PRIORITY,
-		IPA3_0_RULE_MIN_PRIORITY,
-		IPA3_0_LOW_RULE_ID,
-		IPA3_0_RULE_ID_BIT_LEN,
-		IPA3_0_HW_RULE_BUF_SIZE,
-		ipa_write_64,
-		ipa_fltrt_create_flt_bitmap,
-		ipa_fltrt_create_tbl_addr,
-		ipa_fltrt_parse_tbl_addr,
-		ipa_rt_gen_hw_rule_ipav4_5,
-		ipa_flt_gen_hw_rule_ipav4_5,
-		ipa_flt_generate_eq,
-		ipa_rt_parse_hw_rule_ipav4_5,
-		ipa_flt_parse_hw_rule_ipav4_5,
-		{
-			[IPA_TOS_EQ]			= 0xFF,
-			[IPA_PROTOCOL_EQ]		= 1,
-			[IPA_TC_EQ]			= 2,
-			[IPA_OFFSET_MEQ128_0]		= 3,
-			[IPA_OFFSET_MEQ128_1]		= 4,
-			[IPA_OFFSET_MEQ32_0]		= 5,
-			[IPA_OFFSET_MEQ32_1]		= 6,
-			[IPA_IHL_OFFSET_MEQ32_0]	= 7,
-			[IPA_IHL_OFFSET_MEQ32_1]	= 8,
-			[IPA_METADATA_COMPARE]		= 9,
-			[IPA_IHL_OFFSET_RANGE16_0]	= 10,
-			[IPA_IHL_OFFSET_RANGE16_1]	= 11,
-			[IPA_IHL_OFFSET_EQ_32]		= 12,
-			[IPA_IHL_OFFSET_EQ_16]		= 13,
-			[IPA_FL_EQ]			= 14,
-			[IPA_IS_FRAG]			= 15,
-			[IPA_IS_PURE_ACK]		= 0,
-		},
-	},
-};
-
-static int ipa_flt_generate_eq(enum ipa_ip_type ipt,
-		const struct ipa_rule_attrib *attrib,
-		struct ipa_ipfltri_rule_eq *eq_atrb)
-{
-	if (ipa_fltrt_rule_generation_err_check(ipt, attrib))
-		return -EPERM;
-
-	if (ipt == IPA_IP_v4) {
-		if (ipa_flt_generate_eq_ip4(ipt, attrib, eq_atrb)) {
-			IPAHAL_ERR("failed to build ipv4 flt eq rule\n");
-			return -EPERM;
-		}
-	} else if (ipt == IPA_IP_v6) {
-		if (ipa_flt_generate_eq_ip6(ipt, attrib, eq_atrb)) {
-			IPAHAL_ERR("failed to build ipv6 flt eq rule\n");
-			return -EPERM;
-		}
-	} else {
-		IPAHAL_ERR("unsupported ip %d\n", ipt);
-		return  -EPERM;
-	}
-
-	/*
-	 * default "rule" means no attributes set -> map to
-	 * OFFSET_MEQ32_0 with mask of 0 and val of 0 and offset 0
-	 */
-	if (attrib->attrib_mask == 0) {
-		eq_atrb->rule_eq_bitmap = 0;
-		eq_atrb->rule_eq_bitmap |= IPA_GET_RULE_EQ_BIT_PTRN(
-			IPA_OFFSET_MEQ32_0);
-		eq_atrb->offset_meq_32[0].offset = 0;
-		eq_atrb->offset_meq_32[0].mask = 0;
-		eq_atrb->offset_meq_32[0].value = 0;
-	}
-
-	return 0;
-}
-
-static void ipa_fltrt_generate_mac_addr_hw_rule(u8 **extra, u8 **rest,
-	u8 hdr_mac_addr_offset,
-	const uint8_t mac_addr_mask[ETH_ALEN],
-	const uint8_t mac_addr[ETH_ALEN])
-{
-	int i;
-
-	*extra = ipa_write_8(hdr_mac_addr_offset, *extra);
-
-	/* LSB MASK and ADDR */
-	*rest = ipa_write_64(0, *rest);
-	*rest = ipa_write_64(0, *rest);
-
-	/* MSB MASK and ADDR */
-	*rest = ipa_write_16(0, *rest);
-	for (i = 5; i >= 0; i--)
-		*rest = ipa_write_8(mac_addr_mask[i], *rest);
-	*rest = ipa_write_16(0, *rest);
-	for (i = 5; i >= 0; i--)
-		*rest = ipa_write_8(mac_addr[i], *rest);
-}
-
-static int ipa_fltrt_generate_hw_rule_bdy_ip4(u16 *en_rule,
-	const struct ipa_rule_attrib *attrib,
-	u8 **extra_wrds, u8 **rest_wrds)
-{
-	u8 *extra = *extra_wrds;
-	u8 *rest = *rest_wrds;
-	u8 ofst_meq32 = 0;
-	u8 ihl_ofst_rng16 = 0;
-	u8 ihl_ofst_meq32 = 0;
-	u8 ofst_meq128 = 0;
-	int rc = 0;
-	bool tos_done = false;
-
-	if (attrib->attrib_mask & IPA_FLT_IS_PURE_ACK) {
-		if (!IPA_IS_RULE_EQ_VALID(IPA_IS_PURE_ACK)) {
-			IPAHAL_ERR("is_pure_ack eq not supported\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(IPA_IS_PURE_ACK);
-		extra = ipa_write_8(0, extra);
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_TOS && !tos_done) {
-		if (!IPA_IS_RULE_EQ_VALID(IPA_TOS_EQ)) {
-			IPAHAL_DBG("tos eq not supported\n");
-		} else {
-			*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(IPA_TOS_EQ);
-			extra = ipa_write_8(attrib->u.v4.tos, extra);
-			tos_done = true;
-		}
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_PROTOCOL) {
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(IPA_PROTOCOL_EQ);
-		extra = ipa_write_8(attrib->u.v4.protocol, extra);
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_MAC_DST_ADDR_ETHER_II) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq128, ofst_meq128)) {
-			IPAHAL_ERR("ran out of meq128 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ofst_meq128[ofst_meq128]);
-
-		/* -14 => offset of dst mac addr in Ethernet II hdr */
-		ipa_fltrt_generate_mac_addr_hw_rule(
-			&extra,
-			&rest,
-			-14,
-			attrib->dst_mac_addr_mask,
-			attrib->dst_mac_addr);
-
-		ofst_meq128++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_MAC_SRC_ADDR_ETHER_II) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq128, ofst_meq128)) {
-			IPAHAL_ERR("ran out of meq128 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ofst_meq128[ofst_meq128]);
-
-		/* -8 => offset of src mac addr in Ethernet II hdr */
-		ipa_fltrt_generate_mac_addr_hw_rule(
-			&extra,
-			&rest,
-			-8,
-			attrib->src_mac_addr_mask,
-			attrib->src_mac_addr);
-
-		ofst_meq128++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_MAC_DST_ADDR_802_3) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq128, ofst_meq128)) {
-			IPAHAL_ERR("ran out of meq128 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ofst_meq128[ofst_meq128]);
-
-		/* -22 => offset of dst mac addr in 802.3 hdr */
-		ipa_fltrt_generate_mac_addr_hw_rule(
-			&extra,
-			&rest,
-			-22,
-			attrib->dst_mac_addr_mask,
-			attrib->dst_mac_addr);
-
-		ofst_meq128++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_MAC_SRC_ADDR_802_3) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq128, ofst_meq128)) {
-			IPAHAL_ERR("ran out of meq128 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ofst_meq128[ofst_meq128]);
-
-		/* -16 => offset of src mac addr in 802.3 hdr */
-		ipa_fltrt_generate_mac_addr_hw_rule(
-			&extra,
-			&rest,
-			-16,
-			attrib->src_mac_addr_mask,
-			attrib->src_mac_addr);
-
-		ofst_meq128++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_TOS_MASKED) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq32, ofst_meq32)) {
-			IPAHAL_ERR("ran out of meq32 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ofst_meq32[ofst_meq32]);
-		/* 0 => Take the first word. offset of TOS in v4 header is 1 */
-		extra = ipa_write_8(0, extra);
-		rest = ipa_write_32((attrib->tos_mask << 16), rest);
-		rest = ipa_write_32((attrib->tos_value << 16), rest);
-		ofst_meq32++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_SRC_ADDR) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq32, ofst_meq32)) {
-			IPAHAL_ERR("ran out of meq32 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ofst_meq32[ofst_meq32]);
-		/* 12 => offset of src ip in v4 header */
-		extra = ipa_write_8(12, extra);
-		rest = ipa_write_32(attrib->u.v4.src_addr_mask, rest);
-		rest = ipa_write_32(attrib->u.v4.src_addr, rest);
-		ofst_meq32++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_DST_ADDR) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq32, ofst_meq32)) {
-			IPAHAL_ERR("ran out of meq32 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ofst_meq32[ofst_meq32]);
-		/* 16 => offset of dst ip in v4 header */
-		extra = ipa_write_8(16, extra);
-		rest = ipa_write_32(attrib->u.v4.dst_addr_mask, rest);
-		rest = ipa_write_32(attrib->u.v4.dst_addr, rest);
-		ofst_meq32++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_MAC_ETHER_TYPE) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq32, ofst_meq32)) {
-			IPAHAL_ERR("ran out of meq32 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ofst_meq32[ofst_meq32]);
-		/* -2 => offset of ether type in L2 hdr */
-		extra = ipa_write_8((u8)-2, extra);
-		rest = ipa_write_16(0, rest);
-		rest = ipa_write_16(htons(attrib->ether_type), rest);
-		rest = ipa_write_16(0, rest);
-		rest = ipa_write_16(htons(attrib->ether_type), rest);
-		ofst_meq32++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_TOS && !tos_done) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq32, ofst_meq32)) {
-			IPAHAL_DBG("ran out of meq32 eq\n");
-		} else {
-			*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-				ipa3_0_ofst_meq32[ofst_meq32]);
-			/*
-			 * 0 => Take the first word.
-			 * offset of TOS in v4 header is 1
-			 */
-			extra = ipa_write_8(0, extra);
-			rest = ipa_write_32(0xFF << 16, rest);
-			rest = ipa_write_32((attrib->u.v4.tos << 16), rest);
-			ofst_meq32++;
-			tos_done = true;
-		}
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_TYPE) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_meq32,
-			ihl_ofst_meq32)) {
-			IPAHAL_ERR("ran out of ihl_meq32 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32]);
-		/* 0  => offset of type after v4 header */
-		extra = ipa_write_8(0, extra);
-		rest = ipa_write_32(0xFF, rest);
-		rest = ipa_write_32(attrib->type, rest);
-		ihl_ofst_meq32++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_CODE) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_meq32,
-			ihl_ofst_meq32)) {
-			IPAHAL_ERR("ran out of ihl_meq32 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32]);
-		/* 1  => offset of code after v4 header */
-		extra = ipa_write_8(1, extra);
-		rest = ipa_write_32(0xFF, rest);
-		rest = ipa_write_32(attrib->code, rest);
-		ihl_ofst_meq32++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_SPI) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_meq32,
-			ihl_ofst_meq32)) {
-			IPAHAL_ERR("ran out of ihl_meq32 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32]);
-		/* 0  => offset of SPI after v4 header */
-		extra = ipa_write_8(0, extra);
-		rest = ipa_write_32(0xFFFFFFFF, rest);
-		rest = ipa_write_32(attrib->spi, rest);
-		ihl_ofst_meq32++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_MAC_DST_ADDR_L2TP) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_meq32,
-			ihl_ofst_meq32) || IPA_IS_RAN_OUT_OF_EQ(
-			ipa3_0_ihl_ofst_meq32, ihl_ofst_meq32 + 1)) {
-			IPAHAL_ERR("ran out of ihl_meq32 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32]);
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32 + 1]);
-		/* populate first ihl meq eq */
-		extra = ipa_write_8(8, extra);
-		rest = ipa_write_8(attrib->dst_mac_addr_mask[3], rest);
-		rest = ipa_write_8(attrib->dst_mac_addr_mask[2], rest);
-		rest = ipa_write_8(attrib->dst_mac_addr_mask[1], rest);
-		rest = ipa_write_8(attrib->dst_mac_addr_mask[0], rest);
-		rest = ipa_write_8(attrib->dst_mac_addr[3], rest);
-		rest = ipa_write_8(attrib->dst_mac_addr[2], rest);
-		rest = ipa_write_8(attrib->dst_mac_addr[1], rest);
-		rest = ipa_write_8(attrib->dst_mac_addr[0], rest);
-		/* populate second ihl meq eq */
-		extra = ipa_write_8(12, extra);
-		rest = ipa_write_16(0, rest);
-		rest = ipa_write_8(attrib->dst_mac_addr_mask[5], rest);
-		rest = ipa_write_8(attrib->dst_mac_addr_mask[4], rest);
-		rest = ipa_write_16(0, rest);
-		rest = ipa_write_8(attrib->dst_mac_addr[5], rest);
-		rest = ipa_write_8(attrib->dst_mac_addr[4], rest);
-		ihl_ofst_meq32 += 2;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_TCP_SYN) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_meq32,
-			ihl_ofst_meq32)) {
-			IPAHAL_ERR("ran out of ihl_meq32 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32]);
-		/* 12  => offset of SYN after v4 header */
-		extra = ipa_write_8(12, extra);
-		rest = ipa_write_32(0x20000, rest);
-		rest = ipa_write_32(0x20000, rest);
-		ihl_ofst_meq32++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_TOS && !tos_done) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_meq32,
-			ihl_ofst_meq32)) {
-			IPAHAL_DBG("ran out of ihl_meq32 eq\n");
-		} else {
-			*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-				ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32]);
-			/*
-			 * 0 => Take the first word. offset of TOS in
-			 * v4 header is 1. MSB bit asserted at IHL means
-			 * to ignore packet IHL and do offset inside IPA header
-			 */
-			extra = ipa_write_8(0x80, extra);
-			rest = ipa_write_32(0xFF << 16, rest);
-			rest = ipa_write_32((attrib->u.v4.tos << 16), rest);
-			ihl_ofst_meq32++;
-			tos_done = true;
-		}
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_META_DATA) {
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(IPA_METADATA_COMPARE);
-		rest = ipa_write_32(attrib->meta_data_mask, rest);
-		rest = ipa_write_32(attrib->meta_data, rest);
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_SRC_PORT_RANGE) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_rng16,
-				ihl_ofst_rng16)) {
-			IPAHAL_ERR("ran out of ihl_rng16 eq\n");
-			goto err;
-		}
-		if (attrib->src_port_hi < attrib->src_port_lo) {
-			IPAHAL_ERR("bad src port range param\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_rng16[ihl_ofst_rng16]);
-		/* 0  => offset of src port after v4 header */
-		extra = ipa_write_8(0, extra);
-		rest = ipa_write_16(attrib->src_port_hi, rest);
-		rest = ipa_write_16(attrib->src_port_lo, rest);
-		ihl_ofst_rng16++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_DST_PORT_RANGE) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_rng16,
-				ihl_ofst_rng16)) {
-			IPAHAL_ERR("ran out of ihl_rng16 eq\n");
-			goto err;
-		}
-		if (attrib->dst_port_hi < attrib->dst_port_lo) {
-			IPAHAL_ERR("bad dst port range param\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_rng16[ihl_ofst_rng16]);
-		/* 2  => offset of dst port after v4 header */
-		extra = ipa_write_8(2, extra);
-		rest = ipa_write_16(attrib->dst_port_hi, rest);
-		rest = ipa_write_16(attrib->dst_port_lo, rest);
-		ihl_ofst_rng16++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_SRC_PORT) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_rng16,
-				ihl_ofst_rng16)) {
-			IPAHAL_ERR("ran out of ihl_rng16 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_rng16[ihl_ofst_rng16]);
-		/* 0  => offset of src port after v4 header */
-		extra = ipa_write_8(0, extra);
-		rest = ipa_write_16(attrib->src_port, rest);
-		rest = ipa_write_16(attrib->src_port, rest);
-		ihl_ofst_rng16++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_DST_PORT) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_rng16,
-				ihl_ofst_rng16)) {
-			IPAHAL_ERR("ran out of ihl_rng16 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_rng16[ihl_ofst_rng16]);
-		/* 2  => offset of dst port after v4 header */
-		extra = ipa_write_8(2, extra);
-		rest = ipa_write_16(attrib->dst_port, rest);
-		rest = ipa_write_16(attrib->dst_port, rest);
-		ihl_ofst_rng16++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_FRAGMENT)
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(IPA_IS_FRAG);
-
-	if (attrib->attrib_mask & IPA_FLT_TOS && !tos_done) {
-		IPAHAL_ERR("could not find equation for tos\n");
-		goto err;
-	}
-
-	goto done;
-
-err:
-	rc = -EPERM;
-done:
-	*extra_wrds = extra;
-	*rest_wrds = rest;
-	return rc;
-}
-
-static int ipa_fltrt_generate_hw_rule_bdy_ip6(u16 *en_rule,
-	const struct ipa_rule_attrib *attrib,
-	u8 **extra_wrds, u8 **rest_wrds)
-{
-	u8 *extra = *extra_wrds;
-	u8 *rest = *rest_wrds;
-	u8 ofst_meq32 = 0;
-	u8 ihl_ofst_rng16 = 0;
-	u8 ihl_ofst_meq32 = 0;
-	u8 ofst_meq128 = 0;
-	int rc = 0;
-
-	/* v6 code below assumes no extension headers TODO: fix this */
-	if (attrib->attrib_mask & IPA_FLT_IS_PURE_ACK) {
-		if (!IPA_IS_RULE_EQ_VALID(IPA_IS_PURE_ACK)) {
-			IPAHAL_ERR("is_pure_ack eq not supported\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(IPA_IS_PURE_ACK);
-		extra = ipa_write_8(0, extra);
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_NEXT_HDR) {
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(IPA_PROTOCOL_EQ);
-		extra = ipa_write_8(attrib->u.v6.next_hdr, extra);
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_TC) {
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(IPA_TC_EQ);
-		extra = ipa_write_8(attrib->u.v6.tc, extra);
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_SRC_ADDR) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq128, ofst_meq128)) {
-			IPAHAL_ERR("ran out of meq128 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ofst_meq128[ofst_meq128]);
-		/* 8 => offset of src ip in v6 header */
-		extra = ipa_write_8(8, extra);
-		rest = ipa_write_32(attrib->u.v6.src_addr_mask[3], rest);
-		rest = ipa_write_32(attrib->u.v6.src_addr_mask[2], rest);
-		rest = ipa_write_32(attrib->u.v6.src_addr[3], rest);
-		rest = ipa_write_32(attrib->u.v6.src_addr[2], rest);
-		rest = ipa_write_32(attrib->u.v6.src_addr_mask[1], rest);
-		rest = ipa_write_32(attrib->u.v6.src_addr_mask[0], rest);
-		rest = ipa_write_32(attrib->u.v6.src_addr[1], rest);
-		rest = ipa_write_32(attrib->u.v6.src_addr[0], rest);
-		ofst_meq128++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_DST_ADDR) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq128, ofst_meq128)) {
-			IPAHAL_ERR("ran out of meq128 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ofst_meq128[ofst_meq128]);
-		/* 24 => offset of dst ip in v6 header */
-		extra = ipa_write_8(24, extra);
-		rest = ipa_write_32(attrib->u.v6.dst_addr_mask[3], rest);
-		rest = ipa_write_32(attrib->u.v6.dst_addr_mask[2], rest);
-		rest = ipa_write_32(attrib->u.v6.dst_addr[3], rest);
-		rest = ipa_write_32(attrib->u.v6.dst_addr[2], rest);
-		rest = ipa_write_32(attrib->u.v6.dst_addr_mask[1], rest);
-		rest = ipa_write_32(attrib->u.v6.dst_addr_mask[0], rest);
-		rest = ipa_write_32(attrib->u.v6.dst_addr[1], rest);
-		rest = ipa_write_32(attrib->u.v6.dst_addr[0], rest);
-		ofst_meq128++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_TOS_MASKED) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq128, ofst_meq128)) {
-			IPAHAL_ERR("ran out of meq128 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ofst_meq128[ofst_meq128]);
-		/* 0 => offset of TOS in v6 header */
-		extra = ipa_write_8(0, extra);
-		rest = ipa_write_64(0, rest);
-		rest = ipa_write_64(0, rest);
-		rest = ipa_write_32(0, rest);
-		rest = ipa_write_32((attrib->tos_mask << 20), rest);
-		rest = ipa_write_32(0, rest);
-		rest = ipa_write_32((attrib->tos_value << 20), rest);
-		ofst_meq128++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_MAC_DST_ADDR_ETHER_II) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq128, ofst_meq128)) {
-			IPAHAL_ERR("ran out of meq128 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ofst_meq128[ofst_meq128]);
-
-		/* -14 => offset of dst mac addr in Ethernet II hdr */
-		ipa_fltrt_generate_mac_addr_hw_rule(
-			&extra,
-			&rest,
-			-14,
-			attrib->dst_mac_addr_mask,
-			attrib->dst_mac_addr);
-
-		ofst_meq128++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_MAC_SRC_ADDR_ETHER_II) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq128, ofst_meq128)) {
-			IPAHAL_ERR("ran out of meq128 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ofst_meq128[ofst_meq128]);
-
-		/* -8 => offset of src mac addr in Ethernet II hdr */
-		ipa_fltrt_generate_mac_addr_hw_rule(
-			&extra,
-			&rest,
-			-8,
-			attrib->src_mac_addr_mask,
-			attrib->src_mac_addr);
-
-		ofst_meq128++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_MAC_DST_ADDR_802_3) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq128, ofst_meq128)) {
-			IPAHAL_ERR("ran out of meq128 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ofst_meq128[ofst_meq128]);
-
-		/* -22 => offset of dst mac addr in 802.3 hdr */
-		ipa_fltrt_generate_mac_addr_hw_rule(
-			&extra,
-			&rest,
-			-22,
-			attrib->dst_mac_addr_mask,
-			attrib->dst_mac_addr);
-
-		ofst_meq128++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_MAC_SRC_ADDR_802_3) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq128, ofst_meq128)) {
-			IPAHAL_ERR("ran out of meq128 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ofst_meq128[ofst_meq128]);
-
-		/* -16 => offset of src mac addr in 802.3 hdr */
-		ipa_fltrt_generate_mac_addr_hw_rule(
-			&extra,
-			&rest,
-			-16,
-			attrib->src_mac_addr_mask,
-			attrib->src_mac_addr);
-
-		ofst_meq128++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_MAC_ETHER_TYPE) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq32, ofst_meq32)) {
-			IPAHAL_ERR("ran out of meq32 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ofst_meq32[ofst_meq32]);
-		/* -2 => offset of ether type in L2 hdr */
-		extra = ipa_write_8((u8)-2, extra);
-		rest = ipa_write_16(0, rest);
-		rest = ipa_write_16(htons(attrib->ether_type), rest);
-		rest = ipa_write_16(0, rest);
-		rest = ipa_write_16(htons(attrib->ether_type), rest);
-		ofst_meq32++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_TYPE) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_meq32,
-			ihl_ofst_meq32)) {
-			IPAHAL_ERR("ran out of ihl_meq32 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32]);
-		/* 0  => offset of type after v6 header */
-		extra = ipa_write_8(0, extra);
-		rest = ipa_write_32(0xFF, rest);
-		rest = ipa_write_32(attrib->type, rest);
-		ihl_ofst_meq32++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_CODE) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_meq32,
-			ihl_ofst_meq32)) {
-			IPAHAL_ERR("ran out of ihl_meq32 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32]);
-		/* 1  => offset of code after v6 header */
-		extra = ipa_write_8(1, extra);
-		rest = ipa_write_32(0xFF, rest);
-		rest = ipa_write_32(attrib->code, rest);
-		ihl_ofst_meq32++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_SPI) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_meq32,
-			ihl_ofst_meq32)) {
-			IPAHAL_ERR("ran out of ihl_meq32 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32]);
-		/* 0  => offset of SPI after v6 header FIXME */
-		extra = ipa_write_8(0, extra);
-		rest = ipa_write_32(0xFFFFFFFF, rest);
-		rest = ipa_write_32(attrib->spi, rest);
-		ihl_ofst_meq32++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_MAC_DST_ADDR_L2TP) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_meq32,
-			ihl_ofst_meq32) || IPA_IS_RAN_OUT_OF_EQ(
-			ipa3_0_ihl_ofst_meq32, ihl_ofst_meq32 + 1)) {
-			IPAHAL_ERR("ran out of ihl_meq32 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32]);
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32 + 1]);
-		/* populate first ihl meq eq */
-		extra = ipa_write_8(8, extra);
-		rest = ipa_write_8(attrib->dst_mac_addr_mask[3], rest);
-		rest = ipa_write_8(attrib->dst_mac_addr_mask[2], rest);
-		rest = ipa_write_8(attrib->dst_mac_addr_mask[1], rest);
-		rest = ipa_write_8(attrib->dst_mac_addr_mask[0], rest);
-		rest = ipa_write_8(attrib->dst_mac_addr[3], rest);
-		rest = ipa_write_8(attrib->dst_mac_addr[2], rest);
-		rest = ipa_write_8(attrib->dst_mac_addr[1], rest);
-		rest = ipa_write_8(attrib->dst_mac_addr[0], rest);
-		/* populate second ihl meq eq */
-		extra = ipa_write_8(12, extra);
-		rest = ipa_write_16(0, rest);
-		rest = ipa_write_8(attrib->dst_mac_addr_mask[5], rest);
-		rest = ipa_write_8(attrib->dst_mac_addr_mask[4], rest);
-		rest = ipa_write_16(0, rest);
-		rest = ipa_write_8(attrib->dst_mac_addr[5], rest);
-		rest = ipa_write_8(attrib->dst_mac_addr[4], rest);
-		ihl_ofst_meq32 += 2;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_TCP_SYN) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_meq32,
-			ihl_ofst_meq32)) {
-			IPAHAL_ERR("ran out of ihl_meq32 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32]);
-		/* 12  => offset of SYN after v4 header */
-		extra = ipa_write_8(12, extra);
-		rest = ipa_write_32(0x20000, rest);
-		rest = ipa_write_32(0x20000, rest);
-		ihl_ofst_meq32++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_TCP_SYN_L2TP) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_meq32,
-			ihl_ofst_meq32) || IPA_IS_RAN_OUT_OF_EQ(
-			ipa3_0_ihl_ofst_meq32, ihl_ofst_meq32 + 1)) {
-			IPAHAL_ERR("ran out of ihl_meq32 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32]);
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32 + 1]);
-
-		/* populate TCP protocol eq */
-		if (attrib->ether_type == 0x0800) {
-			extra = ipa_write_8(30, extra);
-			rest = ipa_write_32(0xFF0000, rest);
-			rest = ipa_write_32(0x60000, rest);
-		} else {
-			extra = ipa_write_8(26, extra);
-			rest = ipa_write_32(0xFF00, rest);
-			rest = ipa_write_32(0x600, rest);
-		}
-
-		/* populate TCP SYN eq */
-		if (attrib->ether_type == 0x0800) {
-			extra = ipa_write_8(54, extra);
-			rest = ipa_write_32(0x20000, rest);
-			rest = ipa_write_32(0x20000, rest);
-		} else {
-			extra = ipa_write_8(74, extra);
-			rest = ipa_write_32(0x20000, rest);
-			rest = ipa_write_32(0x20000, rest);
-		}
-		ihl_ofst_meq32 += 2;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_L2TP_INNER_IP_TYPE) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_meq32,
-			ihl_ofst_meq32)) {
-			IPAHAL_ERR("ran out of ihl_meq32 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32]);
-		/* 22  => offset of IP type after v6 header */
-		extra = ipa_write_8(22, extra);
-		rest = ipa_write_32(0xF0000000, rest);
-		if (attrib->type == 0x40)
-			rest = ipa_write_32(0x40000000, rest);
-		else
-			rest = ipa_write_32(0x60000000, rest);
-		ihl_ofst_meq32++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_L2TP_INNER_IPV4_DST_ADDR) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_meq32,
-			ihl_ofst_meq32)) {
-			IPAHAL_ERR("ran out of ihl_meq32 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32]);
-		/* 38  => offset of inner IPv4 addr */
-		extra = ipa_write_8(38, extra);
-		rest = ipa_write_32(attrib->u.v4.dst_addr_mask, rest);
-		rest = ipa_write_32(attrib->u.v4.dst_addr, rest);
-		ihl_ofst_meq32++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_META_DATA) {
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(IPA_METADATA_COMPARE);
-		rest = ipa_write_32(attrib->meta_data_mask, rest);
-		rest = ipa_write_32(attrib->meta_data, rest);
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_SRC_PORT) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_rng16,
-				ihl_ofst_rng16)) {
-			IPAHAL_ERR("ran out of ihl_rng16 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_rng16[ihl_ofst_rng16]);
-		/* 0  => offset of src port after v6 header */
-		extra = ipa_write_8(0, extra);
-		rest = ipa_write_16(attrib->src_port, rest);
-		rest = ipa_write_16(attrib->src_port, rest);
-		ihl_ofst_rng16++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_DST_PORT) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_rng16,
-				ihl_ofst_rng16)) {
-			IPAHAL_ERR("ran out of ihl_rng16 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_rng16[ihl_ofst_rng16]);
-		/* 2  => offset of dst port after v6 header */
-		extra = ipa_write_8(2, extra);
-		rest = ipa_write_16(attrib->dst_port, rest);
-		rest = ipa_write_16(attrib->dst_port, rest);
-		ihl_ofst_rng16++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_SRC_PORT_RANGE) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_rng16,
-				ihl_ofst_rng16)) {
-			IPAHAL_ERR("ran out of ihl_rng16 eq\n");
-			goto err;
-		}
-		if (attrib->src_port_hi < attrib->src_port_lo) {
-			IPAHAL_ERR("bad src port range param\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_rng16[ihl_ofst_rng16]);
-		/* 0  => offset of src port after v6 header */
-		extra = ipa_write_8(0, extra);
-		rest = ipa_write_16(attrib->src_port_hi, rest);
-		rest = ipa_write_16(attrib->src_port_lo, rest);
-		ihl_ofst_rng16++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_DST_PORT_RANGE) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_rng16,
-				ihl_ofst_rng16)) {
-			IPAHAL_ERR("ran out of ihl_rng16 eq\n");
-			goto err;
-		}
-		if (attrib->dst_port_hi < attrib->dst_port_lo) {
-			IPAHAL_ERR("bad dst port range param\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_rng16[ihl_ofst_rng16]);
-		/* 2  => offset of dst port after v6 header */
-		extra = ipa_write_8(2, extra);
-		rest = ipa_write_16(attrib->dst_port_hi, rest);
-		rest = ipa_write_16(attrib->dst_port_lo, rest);
-		ihl_ofst_rng16++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_TCP_SYN_L2TP) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_rng16,
-				ihl_ofst_rng16)) {
-			IPAHAL_ERR("ran out of ihl_rng16 eq\n");
-			goto err;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_rng16[ihl_ofst_rng16]);
-		/* 20  => offset of Ethertype after v4 header */
-		if (attrib->ether_type == 0x0800) {
-			extra = ipa_write_8(21, extra);
-			rest = ipa_write_16(0x0045, rest);
-			rest = ipa_write_16(0x0045, rest);
-		} else {
-			extra = ipa_write_8(20, extra);
-			rest = ipa_write_16(attrib->ether_type, rest);
-			rest = ipa_write_16(attrib->ether_type, rest);
-		}
-		ihl_ofst_rng16++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_FLOW_LABEL) {
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(IPA_FL_EQ);
-		rest = ipa_write_32(attrib->u.v6.flow_label & 0xFFFFF,
-			rest);
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_FRAGMENT)
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(IPA_IS_FRAG);
-
-	goto done;
-
-err:
-	rc = -EPERM;
-done:
-	*extra_wrds = extra;
-	*rest_wrds = rest;
-	return rc;
-}
-
-static u8 *ipa_fltrt_copy_mem(u8 *src, u8 *dst, int cnt)
-{
-	while (cnt--)
-		*dst++ = *src++;
-
-	return dst;
-}
-
-/*
- * ipa_fltrt_generate_hw_rule_bdy() - generate HW rule body (w/o header)
- * @ip: IP address type
- * @attrib: IPA rule attribute
- * @buf: output buffer. Advance it after building the rule
- * @en_rule: enable rule
- *
- * Return codes:
- * 0: success
- * -EPERM: wrong input
- */
-static int ipa_fltrt_generate_hw_rule_bdy(enum ipa_ip_type ipt,
-	const struct ipa_rule_attrib *attrib, u8 **buf, u16 *en_rule)
-{
-	int sz;
-	int rc = 0;
-	u8 *extra_wrd_buf;
-	u8 *rest_wrd_buf;
-	u8 *extra_wrd_start;
-	u8 *rest_wrd_start;
-	u8 *extra_wrd_i;
-	u8 *rest_wrd_i;
-
-	sz = IPA3_0_HW_TBL_WIDTH * 2 + IPA3_0_HW_RULE_START_ALIGNMENT;
-	extra_wrd_buf = kzalloc(sz, GFP_KERNEL);
-	if (!extra_wrd_buf) {
-		rc = -ENOMEM;
-		goto fail_extra_alloc;
-	}
-
-	sz = IPA3_0_HW_RULE_BUF_SIZE + IPA3_0_HW_RULE_START_ALIGNMENT;
-	rest_wrd_buf = kzalloc(sz, GFP_KERNEL);
-	if (!rest_wrd_buf) {
-		rc = -ENOMEM;
-		goto fail_rest_alloc;
-	}
-
-	extra_wrd_start = extra_wrd_buf + IPA3_0_HW_RULE_START_ALIGNMENT;
-	extra_wrd_start = (u8 *)((long)extra_wrd_start &
-		~IPA3_0_HW_RULE_START_ALIGNMENT);
-
-	rest_wrd_start = rest_wrd_buf + IPA3_0_HW_RULE_START_ALIGNMENT;
-	rest_wrd_start = (u8 *)((long)rest_wrd_start &
-		~IPA3_0_HW_RULE_START_ALIGNMENT);
-
-	extra_wrd_i = extra_wrd_start;
-	rest_wrd_i = rest_wrd_start;
-
-	rc = ipa_fltrt_rule_generation_err_check(ipt, attrib);
-	if (rc) {
-		IPAHAL_ERR_RL("rule generation err check failed\n");
-		goto fail_err_check;
-	}
-
-	if (ipt == IPA_IP_v4) {
-		if (ipa_fltrt_generate_hw_rule_bdy_ip4(en_rule, attrib,
-			&extra_wrd_i, &rest_wrd_i)) {
-			IPAHAL_ERR_RL("failed to build ipv4 hw rule\n");
-			rc = -EPERM;
-			goto fail_err_check;
-		}
-
-	} else if (ipt == IPA_IP_v6) {
-		if (ipa_fltrt_generate_hw_rule_bdy_ip6(en_rule, attrib,
-			&extra_wrd_i, &rest_wrd_i)) {
-			IPAHAL_ERR_RL("failed to build ipv6 hw rule\n");
-			rc = -EPERM;
-			goto fail_err_check;
-		}
-	} else {
-		IPAHAL_ERR_RL("unsupported ip %d\n", ipt);
-		goto fail_err_check;
-	}
-
-	/*
-	 * default "rule" means no attributes set -> map to
-	 * OFFSET_MEQ32_0 with mask of 0 and val of 0 and offset 0
-	 */
-	if (attrib->attrib_mask == 0) {
-		IPAHAL_DBG_LOW("building default rule\n");
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(ipa3_0_ofst_meq32[0]);
-		extra_wrd_i = ipa_write_8(0, extra_wrd_i);  /* offset */
-		rest_wrd_i = ipa_write_32(0, rest_wrd_i);   /* mask */
-		rest_wrd_i = ipa_write_32(0, rest_wrd_i);   /* val */
-	}
-
-	IPAHAL_DBG_LOW("extra_word_1 0x%llx\n", *(u64 *)extra_wrd_start);
-	IPAHAL_DBG_LOW("extra_word_2 0x%llx\n",
-		*(u64 *)(extra_wrd_start + IPA3_0_HW_TBL_WIDTH));
-
-	extra_wrd_i = ipa_pad_to_64(extra_wrd_i);
-	sz = extra_wrd_i - extra_wrd_start;
-	IPAHAL_DBG_LOW("extra words params sz %d\n", sz);
-	*buf = ipa_fltrt_copy_mem(extra_wrd_start, *buf, sz);
-
-	rest_wrd_i = ipa_pad_to_64(rest_wrd_i);
-	sz = rest_wrd_i - rest_wrd_start;
-	IPAHAL_DBG_LOW("non extra words params sz %d\n", sz);
-	*buf = ipa_fltrt_copy_mem(rest_wrd_start, *buf, sz);
-
-fail_err_check:
-	kfree(rest_wrd_buf);
-fail_rest_alloc:
-	kfree(extra_wrd_buf);
-fail_extra_alloc:
-	return rc;
-}
-
-
-/**
- * ipa_fltrt_calc_extra_wrd_bytes()- Calculate the number of extra words for eq
- * @attrib: equation attribute
- *
- * Return value: 0 on success, negative otherwise
- */
-static int ipa_fltrt_calc_extra_wrd_bytes(
-	const struct ipa_ipfltri_rule_eq *attrib)
-{
-	int num = 0;
-
-	/*
-	 * tos_eq_present field has two meanings:
-	 * tos equation for IPA ver < 4.5 (as the field name reveals)
-	 * pure_ack equation for IPA ver >= 4.5
-	 * In both cases it needs one extra word.
-	 */
-	if (attrib->tos_eq_present)
-		num++;
-	if (attrib->protocol_eq_present)
-		num++;
-	if (attrib->tc_eq_present)
-		num++;
-	num += attrib->num_offset_meq_128;
-	num += attrib->num_offset_meq_32;
-	num += attrib->num_ihl_offset_meq_32;
-	num += attrib->num_ihl_offset_range_16;
-	if (attrib->ihl_offset_eq_32_present)
-		num++;
-	if (attrib->ihl_offset_eq_16_present)
-		num++;
-
-	IPAHAL_DBG_LOW("extra bytes number %d\n", num);
-
-	return num;
-}
-
-static int ipa_fltrt_generate_hw_rule_bdy_from_eq(
-		const struct ipa_ipfltri_rule_eq *attrib, u8 **buf)
-{
-	uint8_t num_offset_meq_32 = attrib->num_offset_meq_32;
-	uint8_t num_ihl_offset_range_16 = attrib->num_ihl_offset_range_16;
-	uint8_t num_ihl_offset_meq_32 = attrib->num_ihl_offset_meq_32;
-	uint8_t num_offset_meq_128 = attrib->num_offset_meq_128;
-	int i;
-	int extra_bytes;
-	u8 *extra;
-	u8 *rest;
-
-	extra_bytes = ipa_fltrt_calc_extra_wrd_bytes(attrib);
-	/* only 3 eq does not have extra word param, 13 out of 16 is the number
-	 * of equations that needs extra word param
-	 */
-	if (extra_bytes > 13) {
-		IPAHAL_ERR_RL("too much extra bytes\n");
-		return -EPERM;
-	} else if (extra_bytes > IPA3_0_HW_TBL_HDR_WIDTH) {
-		/* two extra words */
-		extra = *buf;
-		rest = *buf + IPA3_0_HW_TBL_HDR_WIDTH * 2;
-	} else if (extra_bytes > 0) {
-		/* single exra word */
-		extra = *buf;
-		rest = *buf + IPA3_0_HW_TBL_HDR_WIDTH;
-	} else {
-		/* no extra words */
-		extra = NULL;
-		rest = *buf;
-	}
-
-	/*
-	 * tos_eq_present field has two meanings:
-	 * tos equation for IPA ver < 4.5 (as the field name reveals)
-	 * pure_ack equation for IPA ver >= 4.5
-	 * In both cases it needs one extra word.
-	 */
-	if (attrib->tos_eq_present) {
-		if (IPA_IS_RULE_EQ_VALID(IPA_IS_PURE_ACK)) {
-			extra = ipa_write_8(0, extra);
-		} else if (IPA_IS_RULE_EQ_VALID(IPA_TOS_EQ)) {
-			extra = ipa_write_8(attrib->tos_eq, extra);
-		} else {
-			IPAHAL_ERR("no support for pure_ack and tos eqs\n");
-			return -EPERM;
-		}
-	}
-
-	if (attrib->protocol_eq_present)
-		extra = ipa_write_8(attrib->protocol_eq, extra);
-
-	if (attrib->tc_eq_present)
-		extra = ipa_write_8(attrib->tc_eq, extra);
-
-	if (num_offset_meq_128) {
-		extra = ipa_write_8(attrib->offset_meq_128[0].offset, extra);
-		for (i = 0; i < 8; i++)
-			rest = ipa_write_8(attrib->offset_meq_128[0].mask[i],
-				rest);
-		for (i = 0; i < 8; i++)
-			rest = ipa_write_8(attrib->offset_meq_128[0].value[i],
-				rest);
-		for (i = 8; i < 16; i++)
-			rest = ipa_write_8(attrib->offset_meq_128[0].mask[i],
-				rest);
-		for (i = 8; i < 16; i++)
-			rest = ipa_write_8(attrib->offset_meq_128[0].value[i],
-				rest);
-		num_offset_meq_128--;
-	}
-
-	if (num_offset_meq_128) {
-		extra = ipa_write_8(attrib->offset_meq_128[1].offset, extra);
-		for (i = 0; i < 8; i++)
-			rest = ipa_write_8(attrib->offset_meq_128[1].mask[i],
-				rest);
-		for (i = 0; i < 8; i++)
-			rest = ipa_write_8(attrib->offset_meq_128[1].value[i],
-				rest);
-		for (i = 8; i < 16; i++)
-			rest = ipa_write_8(attrib->offset_meq_128[1].mask[i],
-				rest);
-		for (i = 8; i < 16; i++)
-			rest = ipa_write_8(attrib->offset_meq_128[1].value[i],
-				rest);
-		num_offset_meq_128--;
-	}
-
-	if (num_offset_meq_32) {
-		extra = ipa_write_8(attrib->offset_meq_32[0].offset, extra);
-		rest = ipa_write_32(attrib->offset_meq_32[0].mask, rest);
-		rest = ipa_write_32(attrib->offset_meq_32[0].value, rest);
-		num_offset_meq_32--;
-	}
-
-	if (num_offset_meq_32) {
-		extra = ipa_write_8(attrib->offset_meq_32[1].offset, extra);
-		rest = ipa_write_32(attrib->offset_meq_32[1].mask, rest);
-		rest = ipa_write_32(attrib->offset_meq_32[1].value, rest);
-		num_offset_meq_32--;
-	}
-
-	if (num_ihl_offset_meq_32) {
-		extra = ipa_write_8(attrib->ihl_offset_meq_32[0].offset,
-		extra);
-
-		rest = ipa_write_32(attrib->ihl_offset_meq_32[0].mask, rest);
-		rest = ipa_write_32(attrib->ihl_offset_meq_32[0].value, rest);
-		num_ihl_offset_meq_32--;
-	}
-
-	if (num_ihl_offset_meq_32) {
-		extra = ipa_write_8(attrib->ihl_offset_meq_32[1].offset,
-		extra);
-
-		rest = ipa_write_32(attrib->ihl_offset_meq_32[1].mask, rest);
-		rest = ipa_write_32(attrib->ihl_offset_meq_32[1].value, rest);
-		num_ihl_offset_meq_32--;
-	}
-
-	if (attrib->metadata_meq32_present) {
-		rest = ipa_write_32(attrib->metadata_meq32.mask, rest);
-		rest = ipa_write_32(attrib->metadata_meq32.value, rest);
-	}
-
-	if (num_ihl_offset_range_16) {
-		extra = ipa_write_8(attrib->ihl_offset_range_16[0].offset,
-		extra);
-
-		rest = ipa_write_16(attrib->ihl_offset_range_16[0].range_high,
-				rest);
-		rest = ipa_write_16(attrib->ihl_offset_range_16[0].range_low,
-				rest);
-		num_ihl_offset_range_16--;
-	}
-
-	if (num_ihl_offset_range_16) {
-		extra = ipa_write_8(attrib->ihl_offset_range_16[1].offset,
-		extra);
-
-		rest = ipa_write_16(attrib->ihl_offset_range_16[1].range_high,
-				rest);
-		rest = ipa_write_16(attrib->ihl_offset_range_16[1].range_low,
-				rest);
-		num_ihl_offset_range_16--;
-	}
-
-	if (attrib->ihl_offset_eq_32_present) {
-		extra = ipa_write_8(attrib->ihl_offset_eq_32.offset, extra);
-		rest = ipa_write_32(attrib->ihl_offset_eq_32.value, rest);
-	}
-
-	if (attrib->ihl_offset_eq_16_present) {
-		extra = ipa_write_8(attrib->ihl_offset_eq_16.offset, extra);
-		rest = ipa_write_16(attrib->ihl_offset_eq_16.value, rest);
-		rest = ipa_write_16(0, rest);
-	}
-
-	if (attrib->fl_eq_present)
-		rest = ipa_write_32(attrib->fl_eq & 0xFFFFF, rest);
-
-	if (extra)
-		extra = ipa_pad_to_64(extra);
-	rest = ipa_pad_to_64(rest);
-	*buf = rest;
-
-	return 0;
-}
-
-static void ipa_flt_generate_mac_addr_eq(struct ipa_ipfltri_rule_eq *eq_atrb,
-	u8 hdr_mac_addr_offset,	const uint8_t mac_addr_mask[ETH_ALEN],
-	const uint8_t mac_addr[ETH_ALEN], u8 ofst_meq128)
-{
-	int i;
-
-	eq_atrb->offset_meq_128[ofst_meq128].offset = hdr_mac_addr_offset;
-
-	/* LSB MASK and ADDR */
-	memset(eq_atrb->offset_meq_128[ofst_meq128].mask, 0, 8);
-	memset(eq_atrb->offset_meq_128[ofst_meq128].value, 0, 8);
-
-	/* MSB MASK and ADDR */
-	memset(eq_atrb->offset_meq_128[ofst_meq128].mask + 8, 0, 2);
-	for (i = 0; i <= 5; i++)
-		eq_atrb->offset_meq_128[ofst_meq128].mask[15 - i] =
-			mac_addr_mask[i];
-
-	memset(eq_atrb->offset_meq_128[ofst_meq128].value + 8, 0, 2);
-	for (i = 0; i <= 5; i++)
-		eq_atrb->offset_meq_128[ofst_meq128].value[15 - i] =
-			mac_addr[i];
-}
-
-static int ipa_flt_generate_eq_ip4(enum ipa_ip_type ip,
-		const struct ipa_rule_attrib *attrib,
-		struct ipa_ipfltri_rule_eq *eq_atrb)
-{
-	u8 ofst_meq32 = 0;
-	u8 ihl_ofst_rng16 = 0;
-	u8 ihl_ofst_meq32 = 0;
-	u8 ofst_meq128 = 0;
-	u16 eq_bitmap = 0;
-	u16 *en_rule = &eq_bitmap;
-	bool tos_done = false;
-
-	if (attrib->attrib_mask & IPA_FLT_IS_PURE_ACK) {
-		if (!IPA_IS_RULE_EQ_VALID(IPA_IS_PURE_ACK)) {
-			IPAHAL_ERR("is_pure_ack eq not supported\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(IPA_IS_PURE_ACK);
-		/*
-		 * Starting IPA 4.5, where PURE ACK equation supported
-		 * and TOS equation support removed, field tos_eq_present
-		 * represent pure_ack presence.
-		 */
-		eq_atrb->tos_eq_present = 1;
-		eq_atrb->tos_eq = 0;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_TOS && !tos_done) {
-		if (!IPA_IS_RULE_EQ_VALID(IPA_TOS_EQ)) {
-			IPAHAL_DBG("tos eq not supported\n");
-		} else {
-			*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(IPA_TOS_EQ);
-			eq_atrb->tos_eq_present = 1;
-			eq_atrb->tos_eq = attrib->u.v4.tos;
-		}
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_PROTOCOL) {
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(IPA_PROTOCOL_EQ);
-		eq_atrb->protocol_eq_present = 1;
-		eq_atrb->protocol_eq = attrib->u.v4.protocol;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_MAC_DST_ADDR_ETHER_II) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq128, ofst_meq128)) {
-			IPAHAL_ERR("ran out of meq128 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ofst_meq128[ofst_meq128]);
-
-		/* -14 => offset of dst mac addr in Ethernet II hdr */
-		ipa_flt_generate_mac_addr_eq(eq_atrb, -14,
-			attrib->dst_mac_addr_mask, attrib->dst_mac_addr,
-			ofst_meq128);
-
-		ofst_meq128++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_MAC_SRC_ADDR_ETHER_II) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq128, ofst_meq128)) {
-			IPAHAL_ERR("ran out of meq128 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ofst_meq128[ofst_meq128]);
-
-		/* -8 => offset of src mac addr in Ethernet II hdr */
-		ipa_flt_generate_mac_addr_eq(eq_atrb, -8,
-			attrib->src_mac_addr_mask, attrib->src_mac_addr,
-			ofst_meq128);
-
-		ofst_meq128++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_MAC_DST_ADDR_802_3) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq128, ofst_meq128)) {
-			IPAHAL_ERR("ran out of meq128 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ofst_meq128[ofst_meq128]);
-
-		/* -22 => offset of dst mac addr in 802.3 hdr */
-		ipa_flt_generate_mac_addr_eq(eq_atrb, -22,
-			attrib->dst_mac_addr_mask, attrib->dst_mac_addr,
-			ofst_meq128);
-
-		ofst_meq128++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_MAC_SRC_ADDR_802_3) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq128, ofst_meq128)) {
-			IPAHAL_ERR("ran out of meq128 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ofst_meq128[ofst_meq128]);
-
-		/* -16 => offset of src mac addr in 802.3 hdr */
-		ipa_flt_generate_mac_addr_eq(eq_atrb, -16,
-			attrib->src_mac_addr_mask, attrib->src_mac_addr,
-			ofst_meq128);
-
-		ofst_meq128++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_MAC_DST_ADDR_L2TP) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_meq32,
-			ihl_ofst_meq32) || IPA_IS_RAN_OUT_OF_EQ(
-			ipa3_0_ihl_ofst_meq32, ihl_ofst_meq32 + 1)) {
-			IPAHAL_ERR("ran out of ihl_meq32 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32]);
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32 + 1]);
-		/* populate the first ihl meq 32 eq */
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].offset = 8;
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].mask =
-			(attrib->dst_mac_addr_mask[3] & 0xFF) |
-			((attrib->dst_mac_addr_mask[2] << 8) & 0xFF00) |
-			((attrib->dst_mac_addr_mask[1] << 16) & 0xFF0000) |
-			((attrib->dst_mac_addr_mask[0] << 24) & 0xFF000000);
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].value =
-			(attrib->dst_mac_addr[3] & 0xFF) |
-			((attrib->dst_mac_addr[2] << 8) & 0xFF00) |
-			((attrib->dst_mac_addr[1] << 16) & 0xFF0000) |
-			((attrib->dst_mac_addr[0] << 24) & 0xFF000000);
-		/* populate the second ihl meq 32 eq */
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32 + 1].offset = 12;
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32 + 1].mask =
-			((attrib->dst_mac_addr_mask[5] << 16) & 0xFF0000) |
-			((attrib->dst_mac_addr_mask[4] << 24) & 0xFF000000);
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32 + 1].value =
-			((attrib->dst_mac_addr[5] << 16) & 0xFF0000) |
-			((attrib->dst_mac_addr[4] << 24) & 0xFF000000);
-		ihl_ofst_meq32 += 2;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_TCP_SYN) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_meq32,
-			ihl_ofst_meq32)) {
-			IPAHAL_ERR("ran out of ihl_meq32 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32]);
-		/* 12  => offset of SYN after v4 header */
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].offset = 12;
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].mask = 0x20000;
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].value = 0x20000;
-		ihl_ofst_meq32++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_TOS_MASKED) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq32, ofst_meq32)) {
-			IPAHAL_ERR("ran out of meq32 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ofst_meq32[ofst_meq32]);
-		eq_atrb->offset_meq_32[ofst_meq32].offset = 0;
-		eq_atrb->offset_meq_32[ofst_meq32].mask =
-			attrib->tos_mask << 16;
-		eq_atrb->offset_meq_32[ofst_meq32].value =
-			attrib->tos_value << 16;
-		ofst_meq32++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_SRC_ADDR) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq32, ofst_meq32)) {
-			IPAHAL_ERR("ran out of meq32 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ofst_meq32[ofst_meq32]);
-		eq_atrb->offset_meq_32[ofst_meq32].offset = 12;
-		eq_atrb->offset_meq_32[ofst_meq32].mask =
-			attrib->u.v4.src_addr_mask;
-		eq_atrb->offset_meq_32[ofst_meq32].value =
-			attrib->u.v4.src_addr;
-		ofst_meq32++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_DST_ADDR) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq32, ofst_meq32)) {
-			IPAHAL_ERR("ran out of meq32 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ofst_meq32[ofst_meq32]);
-		eq_atrb->offset_meq_32[ofst_meq32].offset = 16;
-		eq_atrb->offset_meq_32[ofst_meq32].mask =
-			attrib->u.v4.dst_addr_mask;
-		eq_atrb->offset_meq_32[ofst_meq32].value =
-			attrib->u.v4.dst_addr;
-		ofst_meq32++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_MAC_ETHER_TYPE) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq32, ofst_meq32)) {
-			IPAHAL_ERR("ran out of meq32 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ofst_meq32[ofst_meq32]);
-		eq_atrb->offset_meq_32[ofst_meq32].offset = -2;
-		eq_atrb->offset_meq_32[ofst_meq32].mask =
-			htons(attrib->ether_type);
-		eq_atrb->offset_meq_32[ofst_meq32].value =
-			htons(attrib->ether_type);
-		ofst_meq32++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_TOS && !tos_done) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq32, ofst_meq32)) {
-			IPAHAL_DBG("ran out of meq32 eq\n");
-		} else {
-			*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-				ipa3_0_ofst_meq32[ofst_meq32]);
-			/*
-			 * offset 0 => Take the first word.
-			 * offset of TOS in v4 header is 1
-			 */
-			eq_atrb->offset_meq_32[ofst_meq32].offset = 0;
-			eq_atrb->offset_meq_32[ofst_meq32].mask =
-				0xFF << 16;
-			eq_atrb->offset_meq_32[ofst_meq32].value =
-				attrib->u.v4.tos << 16;
-			ofst_meq32++;
-			tos_done = true;
-		}
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_TYPE) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_meq32,
-			ihl_ofst_meq32)) {
-			IPAHAL_ERR("ran out of ihl_meq32 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32]);
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].offset = 0;
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].mask = 0xFF;
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].value =
-			attrib->type;
-		ihl_ofst_meq32++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_CODE) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_meq32,
-			ihl_ofst_meq32)) {
-			IPAHAL_ERR("ran out of ihl_meq32 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32]);
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].offset = 1;
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].mask = 0xFF;
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].value =
-			attrib->code;
-		ihl_ofst_meq32++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_SPI) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_meq32,
-			ihl_ofst_meq32)) {
-			IPAHAL_ERR("ran out of ihl_meq32 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32]);
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].offset = 0;
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].mask =
-			0xFFFFFFFF;
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].value =
-			attrib->spi;
-		ihl_ofst_meq32++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_TOS && !tos_done) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_meq32,
-			ihl_ofst_meq32)) {
-			IPAHAL_DBG("ran out of ihl_meq32 eq\n");
-		} else {
-			*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-				ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32]);
-			/*
-			 * 0 => Take the first word. offset of TOS in
-			 * v4 header is 1. MSB bit asserted at IHL means
-			 * to ignore packet IHL and do offset inside IPA header
-			 */
-			eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].offset =
-				0x80;
-			eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].mask =
-				0xFF << 16;
-			eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].value =
-				attrib->u.v4.tos << 16;
-			ihl_ofst_meq32++;
-			tos_done = true;
-		}
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_META_DATA) {
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			IPA_METADATA_COMPARE);
-		eq_atrb->metadata_meq32_present = 1;
-		eq_atrb->metadata_meq32.offset = 0;
-		eq_atrb->metadata_meq32.mask = attrib->meta_data_mask;
-		eq_atrb->metadata_meq32.value = attrib->meta_data;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_SRC_PORT_RANGE) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_rng16,
-				ihl_ofst_rng16)) {
-			IPAHAL_ERR("ran out of ihl_rng16 eq\n");
-			return -EPERM;
-		}
-		if (attrib->src_port_hi < attrib->src_port_lo) {
-			IPAHAL_ERR("bad src port range param\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_rng16[ihl_ofst_rng16]);
-		eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].offset = 0;
-		eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].range_low
-			= attrib->src_port_lo;
-		eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].range_high
-			= attrib->src_port_hi;
-		ihl_ofst_rng16++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_DST_PORT_RANGE) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_rng16,
-				ihl_ofst_rng16)) {
-			IPAHAL_ERR("ran out of ihl_rng16 eq\n");
-			return -EPERM;
-		}
-		if (attrib->dst_port_hi < attrib->dst_port_lo) {
-			IPAHAL_ERR("bad dst port range param\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_rng16[ihl_ofst_rng16]);
-		eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].offset = 2;
-		eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].range_low
-			= attrib->dst_port_lo;
-		eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].range_high
-			= attrib->dst_port_hi;
-		ihl_ofst_rng16++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_SRC_PORT) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_rng16,
-				ihl_ofst_rng16)) {
-			IPAHAL_ERR("ran out of ihl_rng16 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_rng16[ihl_ofst_rng16]);
-		eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].offset = 0;
-		eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].range_low
-			= attrib->src_port;
-		eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].range_high
-			= attrib->src_port;
-		ihl_ofst_rng16++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_DST_PORT) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_rng16,
-				ihl_ofst_rng16)) {
-			IPAHAL_ERR("ran out of ihl_rng16 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_rng16[ihl_ofst_rng16]);
-		eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].offset = 2;
-		eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].range_low
-			= attrib->dst_port;
-		eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].range_high
-			= attrib->dst_port;
-		ihl_ofst_rng16++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_FRAGMENT) {
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(IPA_IS_FRAG);
-		eq_atrb->ipv4_frag_eq_present = 1;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_TOS && !tos_done) {
-		IPAHAL_ERR("could not find equation for tos\n");
-		return -EPERM;
-	}
-
-	eq_atrb->rule_eq_bitmap = *en_rule;
-	eq_atrb->num_offset_meq_32 = ofst_meq32;
-	eq_atrb->num_ihl_offset_range_16 = ihl_ofst_rng16;
-	eq_atrb->num_ihl_offset_meq_32 = ihl_ofst_meq32;
-	eq_atrb->num_offset_meq_128 = ofst_meq128;
-
-	return 0;
-}
-
-static int ipa_flt_generate_eq_ip6(enum ipa_ip_type ip,
-		const struct ipa_rule_attrib *attrib,
-		struct ipa_ipfltri_rule_eq *eq_atrb)
-{
-	u8 ofst_meq32 = 0;
-	u8 ihl_ofst_rng16 = 0;
-	u8 ihl_ofst_meq32 = 0;
-	u8 ofst_meq128 = 0;
-	u16 eq_bitmap = 0;
-	u16 *en_rule = &eq_bitmap;
-
-	if (attrib->attrib_mask & IPA_FLT_IS_PURE_ACK) {
-		if (!IPA_IS_RULE_EQ_VALID(IPA_IS_PURE_ACK)) {
-			IPAHAL_ERR("is_pure_ack eq not supported\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(IPA_IS_PURE_ACK);
-		/*
-		 * Starting IPA 4.5, where PURE ACK equation supported
-		 * and TOS equation support removed, field tos_eq_present
-		 * represent pure_ack presenence.
-		 */
-		eq_atrb->tos_eq_present = 1;
-		eq_atrb->tos_eq = 0;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_NEXT_HDR) {
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			IPA_PROTOCOL_EQ);
-		eq_atrb->protocol_eq_present = 1;
-		eq_atrb->protocol_eq = attrib->u.v6.next_hdr;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_TC) {
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			IPA_TC_EQ);
-		eq_atrb->tc_eq_present = 1;
-		eq_atrb->tc_eq = attrib->u.v6.tc;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_SRC_ADDR) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq128, ofst_meq128)) {
-			IPAHAL_ERR_RL("ran out of meq128 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ofst_meq128[ofst_meq128]);
-		/* use the same word order as in ipa v2 */
-		eq_atrb->offset_meq_128[ofst_meq128].offset = 8;
-		*(u32 *)(eq_atrb->offset_meq_128[ofst_meq128].mask + 0)
-			= attrib->u.v6.src_addr_mask[0];
-		*(u32 *)(eq_atrb->offset_meq_128[ofst_meq128].mask + 4)
-			= attrib->u.v6.src_addr_mask[1];
-		*(u32 *)(eq_atrb->offset_meq_128[ofst_meq128].mask + 8)
-			= attrib->u.v6.src_addr_mask[2];
-		*(u32 *)(eq_atrb->offset_meq_128[ofst_meq128].mask + 12)
-			= attrib->u.v6.src_addr_mask[3];
-		*(u32 *)(eq_atrb->offset_meq_128[ofst_meq128].value + 0)
-			= attrib->u.v6.src_addr[0];
-		*(u32 *)(eq_atrb->offset_meq_128[ofst_meq128].value + 4)
-			= attrib->u.v6.src_addr[1];
-		*(u32 *)(eq_atrb->offset_meq_128[ofst_meq128].value + 8)
-			= attrib->u.v6.src_addr[2];
-		*(u32 *)(eq_atrb->offset_meq_128[ofst_meq128].value +
-				12) = attrib->u.v6.src_addr[3];
-		ofst_meq128++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_DST_ADDR) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq128, ofst_meq128)) {
-			IPAHAL_ERR_RL("ran out of meq128 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ofst_meq128[ofst_meq128]);
-		eq_atrb->offset_meq_128[ofst_meq128].offset = 24;
-		/* use the same word order as in ipa v2 */
-		*(u32 *)(eq_atrb->offset_meq_128[ofst_meq128].mask + 0)
-			= attrib->u.v6.dst_addr_mask[0];
-		*(u32 *)(eq_atrb->offset_meq_128[ofst_meq128].mask + 4)
-			= attrib->u.v6.dst_addr_mask[1];
-		*(u32 *)(eq_atrb->offset_meq_128[ofst_meq128].mask + 8)
-			= attrib->u.v6.dst_addr_mask[2];
-		*(u32 *)(eq_atrb->offset_meq_128[ofst_meq128].mask + 12)
-			= attrib->u.v6.dst_addr_mask[3];
-		*(u32 *)(eq_atrb->offset_meq_128[ofst_meq128].value + 0)
-			= attrib->u.v6.dst_addr[0];
-		*(u32 *)(eq_atrb->offset_meq_128[ofst_meq128].value + 4)
-			= attrib->u.v6.dst_addr[1];
-		*(u32 *)(eq_atrb->offset_meq_128[ofst_meq128].value + 8)
-			= attrib->u.v6.dst_addr[2];
-		*(u32 *)(eq_atrb->offset_meq_128[ofst_meq128].value +
-				12) = attrib->u.v6.dst_addr[3];
-		ofst_meq128++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_TOS_MASKED) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq128, ofst_meq128)) {
-			IPAHAL_ERR_RL("ran out of meq128 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ofst_meq128[ofst_meq128]);
-		eq_atrb->offset_meq_128[ofst_meq128].offset = 0;
-		memset(eq_atrb->offset_meq_128[ofst_meq128].mask, 0, 12);
-		*(u32 *)(eq_atrb->offset_meq_128[ofst_meq128].mask + 12)
-			= attrib->tos_mask << 20;
-		memset(eq_atrb->offset_meq_128[ofst_meq128].value, 0, 12);
-		*(u32 *)(eq_atrb->offset_meq_128[ofst_meq128].value +
-				12) = attrib->tos_value << 20;
-		ofst_meq128++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_MAC_DST_ADDR_ETHER_II) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq128, ofst_meq128)) {
-			IPAHAL_ERR_RL("ran out of meq128 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ofst_meq128[ofst_meq128]);
-
-		/* -14 => offset of dst mac addr in Ethernet II hdr */
-		ipa_flt_generate_mac_addr_eq(eq_atrb, -14,
-			attrib->dst_mac_addr_mask, attrib->dst_mac_addr,
-			ofst_meq128);
-
-		ofst_meq128++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_MAC_SRC_ADDR_ETHER_II) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq128, ofst_meq128)) {
-			IPAHAL_ERR_RL("ran out of meq128 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ofst_meq128[ofst_meq128]);
-
-		/* -8 => offset of src mac addr in Ethernet II hdr */
-		ipa_flt_generate_mac_addr_eq(eq_atrb, -8,
-			attrib->src_mac_addr_mask, attrib->src_mac_addr,
-			ofst_meq128);
-
-		ofst_meq128++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_MAC_DST_ADDR_802_3) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq128, ofst_meq128)) {
-			IPAHAL_ERR_RL("ran out of meq128 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ofst_meq128[ofst_meq128]);
-
-		/* -22 => offset of dst mac addr in 802.3 hdr */
-		ipa_flt_generate_mac_addr_eq(eq_atrb, -22,
-			attrib->dst_mac_addr_mask, attrib->dst_mac_addr,
-			ofst_meq128);
-
-		ofst_meq128++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_MAC_SRC_ADDR_802_3) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq128, ofst_meq128)) {
-			IPAHAL_ERR_RL("ran out of meq128 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ofst_meq128[ofst_meq128]);
-
-		/* -16 => offset of src mac addr in 802.3 hdr */
-		ipa_flt_generate_mac_addr_eq(eq_atrb, -16,
-			attrib->src_mac_addr_mask, attrib->src_mac_addr,
-			ofst_meq128);
-
-		ofst_meq128++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_MAC_DST_ADDR_L2TP) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_meq32,
-			ihl_ofst_meq32) || IPA_IS_RAN_OUT_OF_EQ(
-			ipa3_0_ihl_ofst_meq32, ihl_ofst_meq32 + 1)) {
-			IPAHAL_ERR_RL("ran out of ihl_meq32 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32]);
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32 + 1]);
-		/* populate the first ihl meq 32 eq */
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].offset = 8;
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].mask =
-			(attrib->dst_mac_addr_mask[3] & 0xFF) |
-			((attrib->dst_mac_addr_mask[2] << 8) & 0xFF00) |
-			((attrib->dst_mac_addr_mask[1] << 16) & 0xFF0000) |
-			((attrib->dst_mac_addr_mask[0] << 24) & 0xFF000000);
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].value =
-			(attrib->dst_mac_addr[3] & 0xFF) |
-			((attrib->dst_mac_addr[2] << 8) & 0xFF00) |
-			((attrib->dst_mac_addr[1] << 16) & 0xFF0000) |
-			((attrib->dst_mac_addr[0] << 24) & 0xFF000000);
-		/* populate the second ihl meq 32 eq */
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32 + 1].offset = 12;
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32 + 1].mask =
-			((attrib->dst_mac_addr_mask[5] << 16) & 0xFF0000) |
-			((attrib->dst_mac_addr_mask[4] << 24) & 0xFF000000);
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32 + 1].value =
-			((attrib->dst_mac_addr[5] << 16) & 0xFF0000) |
-			((attrib->dst_mac_addr[4] << 24) & 0xFF000000);
-		ihl_ofst_meq32 += 2;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_TCP_SYN) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_meq32,
-			ihl_ofst_meq32)) {
-			IPAHAL_ERR_RL("ran out of ihl_meq32 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32]);
-		/* 12  => offset of SYN after v4 header */
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].offset = 12;
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].mask = 0x20000;
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].value = 0x20000;
-		ihl_ofst_meq32++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_TCP_SYN_L2TP) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_meq32,
-			ihl_ofst_meq32) || IPA_IS_RAN_OUT_OF_EQ(
-			ipa3_0_ihl_ofst_meq32, ihl_ofst_meq32 + 1)) {
-			IPAHAL_ERR_RL("ran out of ihl_meq32 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32]);
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32 + 1]);
-
-		/* populate TCP protocol eq */
-		if (attrib->ether_type == 0x0800) {
-			eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].offset = 30;
-			eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].mask =
-				0xFF0000;
-			eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].value =
-				0x60000;
-		} else {
-			eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].offset = 26;
-			eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].mask =
-				0xFF00;
-			eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].value =
-				0x600;
-		}
-
-		/* populate TCP SYN eq */
-		if (attrib->ether_type == 0x0800) {
-			eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].offset = 54;
-			eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].mask =
-				0x20000;
-			eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].value =
-				0x20000;
-		} else {
-			eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].offset = 74;
-			eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].mask =
-				0x20000;
-			eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].value =
-				0x20000;
-		}
-		ihl_ofst_meq32 += 2;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_L2TP_INNER_IP_TYPE) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_meq32,
-			ihl_ofst_meq32)) {
-			IPAHAL_ERR("ran out of ihl_meq32 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32]);
-		/* 22  => offset of inner IP type after v6 header */
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].offset = 22;
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].mask =
-			0xF0000000;
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].value =
-			(u32)attrib->type << 24;
-		ihl_ofst_meq32++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_L2TP_INNER_IPV4_DST_ADDR) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_meq32,
-			ihl_ofst_meq32)) {
-			IPAHAL_ERR("ran out of ihl_meq32 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32]);
-		/* 38  => offset of inner IPv4 addr */
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].offset = 38;
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].mask =
-			attrib->u.v4.dst_addr_mask;
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].value =
-			attrib->u.v4.dst_addr;
-		ihl_ofst_meq32++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_MAC_ETHER_TYPE) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ofst_meq32, ofst_meq32)) {
-			IPAHAL_ERR_RL("ran out of meq32 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ofst_meq32[ofst_meq32]);
-		eq_atrb->offset_meq_32[ofst_meq32].offset = -2;
-		eq_atrb->offset_meq_32[ofst_meq32].mask =
-			htons(attrib->ether_type);
-		eq_atrb->offset_meq_32[ofst_meq32].value =
-			htons(attrib->ether_type);
-		ofst_meq32++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_TYPE) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_meq32,
-			ihl_ofst_meq32)) {
-			IPAHAL_ERR_RL("ran out of ihl_meq32 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32]);
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].offset = 0;
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].mask = 0xFF;
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].value =
-			attrib->type;
-		ihl_ofst_meq32++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_CODE) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_meq32,
-			ihl_ofst_meq32)) {
-			IPAHAL_ERR_RL("ran out of ihl_meq32 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32]);
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].offset = 1;
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].mask = 0xFF;
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].value =
-			attrib->code;
-		ihl_ofst_meq32++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_SPI) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_meq32,
-			ihl_ofst_meq32)) {
-			IPAHAL_ERR_RL("ran out of ihl_meq32 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_meq32[ihl_ofst_meq32]);
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].offset = 0;
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].mask =
-			0xFFFFFFFF;
-		eq_atrb->ihl_offset_meq_32[ihl_ofst_meq32].value =
-			attrib->spi;
-		ihl_ofst_meq32++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_META_DATA) {
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			IPA_METADATA_COMPARE);
-		eq_atrb->metadata_meq32_present = 1;
-		eq_atrb->metadata_meq32.offset = 0;
-		eq_atrb->metadata_meq32.mask = attrib->meta_data_mask;
-		eq_atrb->metadata_meq32.value = attrib->meta_data;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_SRC_PORT) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_rng16,
-				ihl_ofst_rng16)) {
-			IPAHAL_ERR_RL("ran out of ihl_rng16 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_rng16[ihl_ofst_rng16]);
-		eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].offset = 0;
-		eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].range_low
-			= attrib->src_port;
-		eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].range_high
-			= attrib->src_port;
-		ihl_ofst_rng16++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_DST_PORT) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_rng16,
-				ihl_ofst_rng16)) {
-			IPAHAL_ERR_RL("ran out of ihl_rng16 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_rng16[ihl_ofst_rng16]);
-		eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].offset = 2;
-		eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].range_low
-			= attrib->dst_port;
-		eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].range_high
-			= attrib->dst_port;
-		ihl_ofst_rng16++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_SRC_PORT_RANGE) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_rng16,
-				ihl_ofst_rng16)) {
-			IPAHAL_ERR_RL("ran out of ihl_rng16 eq\n");
-			return -EPERM;
-		}
-		if (attrib->src_port_hi < attrib->src_port_lo) {
-			IPAHAL_ERR_RL("bad src port range param\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_rng16[ihl_ofst_rng16]);
-		eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].offset = 0;
-		eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].range_low
-			= attrib->src_port_lo;
-		eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].range_high
-			= attrib->src_port_hi;
-		ihl_ofst_rng16++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_DST_PORT_RANGE) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_rng16,
-				ihl_ofst_rng16)) {
-			IPAHAL_ERR_RL("ran out of ihl_rng16 eq\n");
-			return -EPERM;
-		}
-		if (attrib->dst_port_hi < attrib->dst_port_lo) {
-			IPAHAL_ERR_RL("bad dst port range param\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_rng16[ihl_ofst_rng16]);
-		eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].offset = 2;
-		eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].range_low
-			= attrib->dst_port_lo;
-		eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].range_high
-			= attrib->dst_port_hi;
-		ihl_ofst_rng16++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_TCP_SYN_L2TP) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_rng16,
-				ihl_ofst_rng16)) {
-			IPAHAL_ERR("ran out of ihl_rng16 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_rng16[ihl_ofst_rng16]);
-		if (attrib->ether_type == 0x0800) {
-			eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].offset
-				= 21;
-			eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].range_low
-				= 0x0045;
-			eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].range_high
-				= 0x0045;
-		} else {
-			eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].offset =
-				20;
-			eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].range_low
-				= attrib->ether_type;
-			eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].range_high
-				= attrib->ether_type;
-		}
-		ihl_ofst_rng16++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_TCP_SYN_L2TP) {
-		if (IPA_IS_RAN_OUT_OF_EQ(ipa3_0_ihl_ofst_rng16,
-				ihl_ofst_rng16)) {
-			IPAHAL_ERR_RL("ran out of ihl_rng16 eq\n");
-			return -EPERM;
-		}
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			ipa3_0_ihl_ofst_rng16[ihl_ofst_rng16]);
-		if (attrib->ether_type == 0x0800) {
-			eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].offset
-				= 21;
-			eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].range_low
-				= 0x0045;
-			eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].range_high
-				= 0x0045;
-		} else {
-			eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].offset =
-				20;
-			eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].range_low
-				= attrib->ether_type;
-			eq_atrb->ihl_offset_range_16[ihl_ofst_rng16].range_high
-				= attrib->ether_type;
-		}
-		ihl_ofst_rng16++;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_FLOW_LABEL) {
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(IPA_FL_EQ);
-		eq_atrb->fl_eq_present = 1;
-		eq_atrb->fl_eq = attrib->u.v6.flow_label;
-	}
-
-	if (attrib->attrib_mask & IPA_FLT_FRAGMENT) {
-		*en_rule |= IPA_GET_RULE_EQ_BIT_PTRN(
-			IPA_IS_FRAG);
-		eq_atrb->ipv4_frag_eq_present = 1;
-	}
-
-	eq_atrb->rule_eq_bitmap = *en_rule;
-	eq_atrb->num_offset_meq_32 = ofst_meq32;
-	eq_atrb->num_ihl_offset_range_16 = ihl_ofst_rng16;
-	eq_atrb->num_ihl_offset_meq_32 = ihl_ofst_meq32;
-	eq_atrb->num_offset_meq_128 = ofst_meq128;
-
-	return 0;
-}
-
-static int ipa_fltrt_parse_hw_rule_eq(u8 *addr, u32 hdr_sz,
-	struct ipa_ipfltri_rule_eq *atrb, u32 *rule_size)
-{
-	u16 eq_bitmap;
-	int extra_bytes;
-	u8 *extra;
-	u8 *rest;
-	int i;
-	u8 dummy_extra_wrd;
-
-	if (!addr || !atrb || !rule_size) {
-		IPAHAL_ERR("Input error: addr=%pK atrb=%pK rule_size=%pK\n",
-			addr, atrb, rule_size);
-		return -EINVAL;
-	}
-
-	eq_bitmap = atrb->rule_eq_bitmap;
-
-	IPAHAL_DBG_LOW("eq_bitmap=0x%x\n", eq_bitmap);
-
-	if (IPA_IS_RULE_EQ_VALID(IPA_IS_PURE_ACK) &&
-		(eq_bitmap & IPA_GET_RULE_EQ_BIT_PTRN(IPA_IS_PURE_ACK))) {
-		/*
-		 * tos_eq_present field represents pure_ack when pure
-		 * ack equation valid (started IPA 4.5). In this case
-		 * tos equation should not be supported.
-		 */
-		atrb->tos_eq_present = true;
-	}
-	if (IPA_IS_RULE_EQ_VALID(IPA_TOS_EQ) &&
-		(eq_bitmap & IPA_GET_RULE_EQ_BIT_PTRN(IPA_TOS_EQ))) {
-		atrb->tos_eq_present = true;
-	}
-	if (eq_bitmap & IPA_GET_RULE_EQ_BIT_PTRN(IPA_PROTOCOL_EQ))
-		atrb->protocol_eq_present = true;
-	if (eq_bitmap & IPA_GET_RULE_EQ_BIT_PTRN(IPA_TC_EQ))
-		atrb->tc_eq_present = true;
-	if (eq_bitmap & IPA_GET_RULE_EQ_BIT_PTRN(IPA_OFFSET_MEQ128_0))
-		atrb->num_offset_meq_128++;
-	if (eq_bitmap & IPA_GET_RULE_EQ_BIT_PTRN(IPA_OFFSET_MEQ128_1))
-		atrb->num_offset_meq_128++;
-	if (eq_bitmap & IPA_GET_RULE_EQ_BIT_PTRN(IPA_OFFSET_MEQ32_0))
-		atrb->num_offset_meq_32++;
-	if (eq_bitmap & IPA_GET_RULE_EQ_BIT_PTRN(IPA_OFFSET_MEQ32_1))
-		atrb->num_offset_meq_32++;
-	if (eq_bitmap & IPA_GET_RULE_EQ_BIT_PTRN(IPA_IHL_OFFSET_MEQ32_0))
-		atrb->num_ihl_offset_meq_32++;
-	if (eq_bitmap & IPA_GET_RULE_EQ_BIT_PTRN(IPA_IHL_OFFSET_MEQ32_1))
-		atrb->num_ihl_offset_meq_32++;
-	if (eq_bitmap & IPA_GET_RULE_EQ_BIT_PTRN(IPA_METADATA_COMPARE))
-		atrb->metadata_meq32_present = true;
-	if (eq_bitmap & IPA_GET_RULE_EQ_BIT_PTRN(IPA_IHL_OFFSET_RANGE16_0))
-		atrb->num_ihl_offset_range_16++;
-	if (eq_bitmap & IPA_GET_RULE_EQ_BIT_PTRN(IPA_IHL_OFFSET_RANGE16_1))
-		atrb->num_ihl_offset_range_16++;
-	if (eq_bitmap & IPA_GET_RULE_EQ_BIT_PTRN(IPA_IHL_OFFSET_EQ_32))
-		atrb->ihl_offset_eq_32_present = true;
-	if (eq_bitmap & IPA_GET_RULE_EQ_BIT_PTRN(IPA_IHL_OFFSET_EQ_16))
-		atrb->ihl_offset_eq_16_present = true;
-	if (eq_bitmap & IPA_GET_RULE_EQ_BIT_PTRN(IPA_FL_EQ))
-		atrb->fl_eq_present = true;
-	if (eq_bitmap & IPA_GET_RULE_EQ_BIT_PTRN(IPA_IS_FRAG))
-		atrb->ipv4_frag_eq_present = true;
-
-	extra_bytes = ipa_fltrt_calc_extra_wrd_bytes(atrb);
-	/* only 3 eq does not have extra word param, 13 out of 16 is the number
-	 * of equations that needs extra word param
-	 */
-	if (extra_bytes > 13) {
-		IPAHAL_ERR("too much extra bytes\n");
-		return -EPERM;
-	} else if (extra_bytes > IPA3_0_HW_TBL_HDR_WIDTH) {
-		/* two extra words */
-		extra = addr + hdr_sz;
-		rest = extra + IPA3_0_HW_TBL_HDR_WIDTH * 2;
-	} else if (extra_bytes > 0) {
-		/* single extra word */
-		extra = addr + hdr_sz;
-		rest = extra + IPA3_0_HW_TBL_HDR_WIDTH;
-	} else {
-		/* no extra words */
-		dummy_extra_wrd = 0;
-		extra = &dummy_extra_wrd;
-		rest = addr + hdr_sz;
-	}
-	IPAHAL_DBG_LOW("addr=0x%pK extra=0x%pK rest=0x%pK\n",
-		addr, extra, rest);
-
-	if (IPA_IS_RULE_EQ_VALID(IPA_TOS_EQ) && atrb->tos_eq_present)
-		atrb->tos_eq = *extra++;
-	if (IPA_IS_RULE_EQ_VALID(IPA_IS_PURE_ACK) && atrb->tos_eq_present) {
-		atrb->tos_eq = 0;
-		extra++;
-	}
-	if (atrb->protocol_eq_present)
-		atrb->protocol_eq = *extra++;
-	if (atrb->tc_eq_present)
-		atrb->tc_eq = *extra++;
-
-	if (atrb->num_offset_meq_128 > 0) {
-		atrb->offset_meq_128[0].offset = *extra++;
-		for (i = 0; i < 8; i++)
-			atrb->offset_meq_128[0].mask[i] = *rest++;
-		for (i = 0; i < 8; i++)
-			atrb->offset_meq_128[0].value[i] = *rest++;
-		for (i = 8; i < 16; i++)
-			atrb->offset_meq_128[0].mask[i] = *rest++;
-		for (i = 8; i < 16; i++)
-			atrb->offset_meq_128[0].value[i] = *rest++;
-	}
-	if (atrb->num_offset_meq_128 > 1) {
-		atrb->offset_meq_128[1].offset = *extra++;
-		for (i = 0; i < 8; i++)
-			atrb->offset_meq_128[1].mask[i] = *rest++;
-		for (i = 0; i < 8; i++)
-			atrb->offset_meq_128[1].value[i] = *rest++;
-		for (i = 8; i < 16; i++)
-			atrb->offset_meq_128[1].mask[i] = *rest++;
-		for (i = 8; i < 16; i++)
-			atrb->offset_meq_128[1].value[i] = *rest++;
-	}
-
-	if (atrb->num_offset_meq_32 > 0) {
-		atrb->offset_meq_32[0].offset = *extra++;
-		atrb->offset_meq_32[0].mask = *((u32 *)rest);
-		rest += 4;
-		atrb->offset_meq_32[0].value = *((u32 *)rest);
-		rest += 4;
-	}
-	if (atrb->num_offset_meq_32 > 1) {
-		atrb->offset_meq_32[1].offset = *extra++;
-		atrb->offset_meq_32[1].mask = *((u32 *)rest);
-		rest += 4;
-		atrb->offset_meq_32[1].value = *((u32 *)rest);
-		rest += 4;
-	}
-
-	if (atrb->num_ihl_offset_meq_32 > 0) {
-		atrb->ihl_offset_meq_32[0].offset = *extra++;
-		atrb->ihl_offset_meq_32[0].mask = *((u32 *)rest);
-		rest += 4;
-		atrb->ihl_offset_meq_32[0].value = *((u32 *)rest);
-		rest += 4;
-	}
-	if (atrb->num_ihl_offset_meq_32 > 1) {
-		atrb->ihl_offset_meq_32[1].offset = *extra++;
-		atrb->ihl_offset_meq_32[1].mask = *((u32 *)rest);
-		rest += 4;
-		atrb->ihl_offset_meq_32[1].value = *((u32 *)rest);
-		rest += 4;
-	}
-
-	if (atrb->metadata_meq32_present) {
-		atrb->metadata_meq32.mask = *((u32 *)rest);
-		rest += 4;
-		atrb->metadata_meq32.value = *((u32 *)rest);
-		rest += 4;
-	}
-
-	if (atrb->num_ihl_offset_range_16 > 0) {
-		atrb->ihl_offset_range_16[0].offset = *extra++;
-		atrb->ihl_offset_range_16[0].range_high = *((u16 *)rest);
-		rest += 2;
-		atrb->ihl_offset_range_16[0].range_low = *((u16 *)rest);
-		rest += 2;
-	}
-	if (atrb->num_ihl_offset_range_16 > 1) {
-		atrb->ihl_offset_range_16[1].offset = *extra++;
-		atrb->ihl_offset_range_16[1].range_high = *((u16 *)rest);
-		rest += 2;
-		atrb->ihl_offset_range_16[1].range_low = *((u16 *)rest);
-		rest += 2;
-	}
-
-	if (atrb->ihl_offset_eq_32_present) {
-		atrb->ihl_offset_eq_32.offset = *extra++;
-		atrb->ihl_offset_eq_32.value = *((u32 *)rest);
-		rest += 4;
-	}
-
-	if (atrb->ihl_offset_eq_16_present) {
-		atrb->ihl_offset_eq_16.offset = *extra++;
-		atrb->ihl_offset_eq_16.value = *((u16 *)rest);
-		rest += 4;
-	}
-
-	if (atrb->fl_eq_present) {
-		atrb->fl_eq = *((u32 *)rest);
-		atrb->fl_eq &= 0xfffff;
-		rest += 4;
-	}
-
-	IPAHAL_DBG_LOW("before rule alignment rest=0x%pK\n", rest);
-	rest = (u8 *)(((unsigned long)rest + IPA3_0_HW_RULE_START_ALIGNMENT) &
-		~IPA3_0_HW_RULE_START_ALIGNMENT);
-	IPAHAL_DBG_LOW("after rule alignment  rest=0x%pK\n", rest);
-
-	*rule_size = rest - addr;
-	IPAHAL_DBG_LOW("rule_size=0x%x\n", *rule_size);
-
-	return 0;
-}
-
-static int ipa_rt_parse_hw_rule(u8 *addr, struct ipahal_rt_rule_entry *rule)
-{
-	struct ipa3_0_rt_rule_hw_hdr *rule_hdr;
-	struct ipa_ipfltri_rule_eq *atrb;
-
-	IPAHAL_DBG_LOW("Entry\n");
-
-	rule_hdr = (struct ipa3_0_rt_rule_hw_hdr *)addr;
-	atrb = &rule->eq_attrib;
-
-	IPAHAL_DBG_LOW("read hdr 0x%llx\n", rule_hdr->u.word);
-
-	if (rule_hdr->u.word == 0) {
-		/* table terminator - empty table */
-		rule->rule_size = 0;
-		return 0;
-	}
-
-	rule->dst_pipe_idx = rule_hdr->u.hdr.pipe_dest_idx;
-	if (rule_hdr->u.hdr.proc_ctx) {
-		rule->hdr_type = IPAHAL_RT_RULE_HDR_PROC_CTX;
-		rule->hdr_ofst = (rule_hdr->u.hdr.hdr_offset) << 5;
-	} else {
-		rule->hdr_type = IPAHAL_RT_RULE_HDR_RAW;
-		rule->hdr_ofst = (rule_hdr->u.hdr.hdr_offset) << 2;
-	}
-	rule->hdr_lcl = !rule_hdr->u.hdr.system;
-
-	rule->priority = rule_hdr->u.hdr.priority;
-	rule->retain_hdr = rule_hdr->u.hdr.retain_hdr;
-	rule->id = rule_hdr->u.hdr.rule_id;
-
-	atrb->rule_eq_bitmap = rule_hdr->u.hdr.en_rule;
-	return ipa_fltrt_parse_hw_rule_eq(addr, sizeof(*rule_hdr),
-		atrb, &rule->rule_size);
-}
-
-static int ipa_rt_parse_hw_rule_ipav4_5(u8 *addr,
-	struct ipahal_rt_rule_entry *rule)
-{
-	struct ipa4_5_rt_rule_hw_hdr *rule_hdr;
-	struct ipa_ipfltri_rule_eq *atrb;
-
-	IPAHAL_DBG_LOW("Entry\n");
-
-	rule_hdr = (struct ipa4_5_rt_rule_hw_hdr *)addr;
-	atrb = &rule->eq_attrib;
-
-	IPAHAL_DBG_LOW("read hdr 0x%llx\n", rule_hdr->u.word);
-
-	if (rule_hdr->u.word == 0) {
-		/* table termintator - empty table */
-		rule->rule_size = 0;
-		return 0;
-	}
-
-	rule->dst_pipe_idx = rule_hdr->u.hdr.pipe_dest_idx;
-	if (rule_hdr->u.hdr.proc_ctx) {
-		rule->hdr_type = IPAHAL_RT_RULE_HDR_PROC_CTX;
-		rule->hdr_ofst = (rule_hdr->u.hdr.hdr_offset) << 5;
-	} else {
-		rule->hdr_type = IPAHAL_RT_RULE_HDR_RAW;
-		rule->hdr_ofst = (rule_hdr->u.hdr.hdr_offset) << 2;
-	}
-	rule->hdr_lcl = !rule_hdr->u.hdr.system;
-
-	rule->priority = rule_hdr->u.hdr.priority;
-	rule->retain_hdr = rule_hdr->u.hdr.retain_hdr;
-	rule->cnt_idx = rule_hdr->u.hdr.stats_cnt_idx_lsb |
-		(rule_hdr->u.hdr.stats_cnt_idx_msb) << 6;
-	rule->id = rule_hdr->u.hdr.rule_id;
-
-	atrb->rule_eq_bitmap = rule_hdr->u.hdr.en_rule;
-	return ipa_fltrt_parse_hw_rule_eq(addr, sizeof(*rule_hdr),
-		atrb, &rule->rule_size);
-}
-
-static int ipa_flt_parse_hw_rule(u8 *addr, struct ipahal_flt_rule_entry *rule)
-{
-	struct ipa3_0_flt_rule_hw_hdr *rule_hdr;
-	struct ipa_ipfltri_rule_eq *atrb;
-
-	IPAHAL_DBG_LOW("Entry\n");
-
-	rule_hdr = (struct ipa3_0_flt_rule_hw_hdr *)addr;
-	atrb = &rule->rule.eq_attrib;
-
-	if (rule_hdr->u.word == 0) {
-		/* table termintator - empty table */
-		rule->rule_size = 0;
-		return 0;
-	}
-
-	switch (rule_hdr->u.hdr.action) {
-	case 0x0:
-		rule->rule.action = IPA_PASS_TO_ROUTING;
-		break;
-	case 0x1:
-		rule->rule.action = IPA_PASS_TO_SRC_NAT;
-		break;
-	case 0x2:
-		rule->rule.action = IPA_PASS_TO_DST_NAT;
-		break;
-	case 0x3:
-		rule->rule.action = IPA_PASS_TO_EXCEPTION;
-		break;
-	default:
-		IPAHAL_ERR("Invalid Rule Action %d\n", rule_hdr->u.hdr.action);
-		WARN_ON_RATELIMIT_IPA(1);
-		rule->rule.action = rule_hdr->u.hdr.action;
-	}
-
-	rule->rule.rt_tbl_idx = rule_hdr->u.hdr.rt_tbl_idx;
-	rule->rule.retain_hdr = rule_hdr->u.hdr.retain_hdr;
-	rule->priority = rule_hdr->u.hdr.priority;
-	rule->id = rule_hdr->u.hdr.rule_id;
-
-	atrb->rule_eq_bitmap = rule_hdr->u.hdr.en_rule;
-	rule->rule.eq_attrib_type = 1;
-	return ipa_fltrt_parse_hw_rule_eq(addr, sizeof(*rule_hdr),
-		atrb, &rule->rule_size);
-}
-
-static int ipa_flt_parse_hw_rule_ipav4(u8 *addr,
-	struct ipahal_flt_rule_entry *rule)
-{
-	struct ipa4_0_flt_rule_hw_hdr *rule_hdr;
-	struct ipa_ipfltri_rule_eq *atrb;
-
-	IPAHAL_DBG_LOW("Entry\n");
-
-	rule_hdr = (struct ipa4_0_flt_rule_hw_hdr *)addr;
-	atrb = &rule->rule.eq_attrib;
-
-	if (rule_hdr->u.word == 0) {
-		/* table termintator - empty table */
-		rule->rule_size = 0;
-		return 0;
-	}
-
-	switch (rule_hdr->u.hdr.action) {
-	case 0x0:
-		rule->rule.action = IPA_PASS_TO_ROUTING;
-		break;
-	case 0x1:
-		rule->rule.action = IPA_PASS_TO_SRC_NAT;
-		break;
-	case 0x2:
-		rule->rule.action = IPA_PASS_TO_DST_NAT;
-		break;
-	case 0x3:
-		rule->rule.action = IPA_PASS_TO_EXCEPTION;
-		break;
-	default:
-		IPAHAL_ERR("Invalid Rule Action %d\n", rule_hdr->u.hdr.action);
-		WARN_ON_RATELIMIT_IPA(1);
-		rule->rule.action = rule_hdr->u.hdr.action;
-	}
-
-	rule->rule.rt_tbl_idx = rule_hdr->u.hdr.rt_tbl_idx;
-	rule->rule.retain_hdr = rule_hdr->u.hdr.retain_hdr;
-	rule->priority = rule_hdr->u.hdr.priority;
-	rule->id = rule_hdr->u.hdr.rule_id;
-	rule->rule.pdn_idx = rule_hdr->u.hdr.pdn_idx;
-	rule->rule.set_metadata = rule_hdr->u.hdr.set_metadata;
-
-	atrb->rule_eq_bitmap = rule_hdr->u.hdr.en_rule;
-	rule->rule.eq_attrib_type = 1;
-	return ipa_fltrt_parse_hw_rule_eq(addr, sizeof(*rule_hdr),
-		atrb, &rule->rule_size);
-}
-
-static int ipa_flt_parse_hw_rule_ipav4_5(u8 *addr,
-	struct ipahal_flt_rule_entry *rule)
-{
-	struct ipa4_5_flt_rule_hw_hdr *rule_hdr;
-	struct ipa_ipfltri_rule_eq *atrb;
-
-	IPAHAL_DBG_LOW("Entry\n");
-
-	rule_hdr = (struct ipa4_5_flt_rule_hw_hdr *)addr;
-	atrb = &rule->rule.eq_attrib;
-
-	if (rule_hdr->u.word == 0) {
-		/* table termintator - empty table */
-		rule->rule_size = 0;
-		return 0;
-	}
-
-	switch (rule_hdr->u.hdr.action) {
-	case 0x0:
-		rule->rule.action = IPA_PASS_TO_ROUTING;
-		break;
-	case 0x1:
-		rule->rule.action = IPA_PASS_TO_SRC_NAT;
-		break;
-	case 0x2:
-		rule->rule.action = IPA_PASS_TO_DST_NAT;
-		break;
-	case 0x3:
-		rule->rule.action = IPA_PASS_TO_EXCEPTION;
-		break;
-	default:
-		IPAHAL_ERR("Invalid Rule Action %d\n", rule_hdr->u.hdr.action);
-		WARN_ON_RATELIMIT_IPA(1);
-		rule->rule.action = rule_hdr->u.hdr.action;
-	}
-
-	rule->rule.rt_tbl_idx = rule_hdr->u.hdr.rt_tbl_idx;
-	rule->rule.retain_hdr = rule_hdr->u.hdr.retain_hdr;
-	rule->priority = rule_hdr->u.hdr.priority;
-	rule->id = rule_hdr->u.hdr.rule_id;
-	rule->rule.pdn_idx = rule_hdr->u.hdr.pdn_idx;
-	rule->rule.set_metadata = rule_hdr->u.hdr.set_metadata;
-	rule->cnt_idx = rule_hdr->u.hdr.stats_cnt_idx_lsb |
-		(rule_hdr->u.hdr.stats_cnt_idx_msb) << 6;
-
-	atrb->rule_eq_bitmap = rule_hdr->u.hdr.en_rule;
-	rule->rule.eq_attrib_type = 1;
-	return ipa_fltrt_parse_hw_rule_eq(addr, sizeof(*rule_hdr),
-		atrb, &rule->rule_size);
-}
-
-/*
- * ipahal_fltrt_init() - Build the FLT/RT information table
- *  See ipahal_fltrt_objs[] comments
- *
- * Note: As global variables are initialized with zero, any un-overridden
- *  register entry will be zero. By this we recognize them.
- */
-int ipahal_fltrt_init(enum ipa_hw_type ipa_hw_type)
-{
-	struct ipahal_fltrt_obj zero_obj;
-	int i;
-	struct ipa_mem_buffer *mem;
-	int rc = -EFAULT;
-	u32 eq_bits;
-	u8 *eq_bitfield;
-
-	IPAHAL_DBG("Entry - HW_TYPE=%d\n", ipa_hw_type);
-
-	if (ipa_hw_type >= IPA_HW_MAX) {
-		IPAHAL_ERR("Invalid H/W type\n");
-		return -EFAULT;
-	}
-
-	memset(&zero_obj, 0, sizeof(zero_obj));
-	for (i = IPA_HW_v3_0 ; i < ipa_hw_type ; i++) {
-		if (!memcmp(&ipahal_fltrt_objs[i+1], &zero_obj,
-			sizeof(struct ipahal_fltrt_obj))) {
-			memcpy(&ipahal_fltrt_objs[i+1],
-				&ipahal_fltrt_objs[i],
-				sizeof(struct ipahal_fltrt_obj));
-		} else {
-			/*
-			 * explicitly overridden FLT RT info
-			 * Check validity
-			 */
-			if (!ipahal_fltrt_objs[i+1].tbl_width) {
-				IPAHAL_ERR(
-				 "Zero tbl width ipaver=%d\n",
-				 i+1);
-				WARN_ON(1);
-			}
-			if (!ipahal_fltrt_objs[i+1].sysaddr_alignment) {
-				IPAHAL_ERR(
-				  "No tbl sysaddr alignment ipaver=%d\n",
-				  i+1);
-				WARN_ON(1);
-			}
-			if (!ipahal_fltrt_objs[i+1].lcladdr_alignment) {
-				IPAHAL_ERR(
-				  "No tbl lcladdr alignment ipaver=%d\n",
-				  i+1);
-				WARN_ON(1);
-			}
-			if (!ipahal_fltrt_objs[i+1].blk_sz_alignment) {
-				IPAHAL_ERR(
-				  "No blk sz alignment ipaver=%d\n",
-				  i+1);
-				WARN_ON(1);
-			}
-			if (!ipahal_fltrt_objs[i+1].rule_start_alignment) {
-				IPAHAL_ERR(
-				  "No rule start alignment ipaver=%d\n",
-				  i+1);
-				WARN_ON(1);
-			}
-			if (!ipahal_fltrt_objs[i+1].tbl_hdr_width) {
-				IPAHAL_ERR(
-				 "Zero tbl hdr width ipaver=%d\n",
-				 i+1);
-				WARN_ON(1);
-			}
-			if (!ipahal_fltrt_objs[i+1].tbl_addr_mask) {
-				IPAHAL_ERR(
-				 "Zero tbl hdr width ipaver=%d\n",
-				 i+1);
-				WARN_ON(1);
-			}
-			if (ipahal_fltrt_objs[i+1].rule_id_bit_len < 2) {
-				IPAHAL_ERR(
-				 "Too little bits for rule_id ipaver=%d\n",
-				 i+1);
-				WARN_ON(1);
-			}
-			if (!ipahal_fltrt_objs[i+1].rule_buf_size) {
-				IPAHAL_ERR(
-				 "zero rule buf size ipaver=%d\n",
-				 i+1);
-				WARN_ON(1);
-			}
-			if (!ipahal_fltrt_objs[i+1].write_val_to_hdr) {
-				IPAHAL_ERR(
-				  "No write_val_to_hdr CB ipaver=%d\n",
-				  i+1);
-				WARN_ON(1);
-			}
-			if (!ipahal_fltrt_objs[i+1].create_flt_bitmap) {
-				IPAHAL_ERR(
-				  "No create_flt_bitmap CB ipaver=%d\n",
-				  i+1);
-				WARN_ON(1);
-			}
-			if (!ipahal_fltrt_objs[i+1].create_tbl_addr) {
-				IPAHAL_ERR(
-				  "No create_tbl_addr CB ipaver=%d\n",
-				  i+1);
-				WARN_ON(1);
-			}
-			if (!ipahal_fltrt_objs[i+1].parse_tbl_addr) {
-				IPAHAL_ERR(
-				  "No parse_tbl_addr CB ipaver=%d\n",
-				  i+1);
-				WARN_ON(1);
-			}
-			if (!ipahal_fltrt_objs[i+1].rt_generate_hw_rule) {
-				IPAHAL_ERR(
-				  "No rt_generate_hw_rule CB ipaver=%d\n",
-				  i+1);
-				WARN_ON(1);
-			}
-			if (!ipahal_fltrt_objs[i+1].flt_generate_hw_rule) {
-				IPAHAL_ERR(
-				  "No flt_generate_hw_rule CB ipaver=%d\n",
-				  i+1);
-				WARN_ON(1);
-			}
-			if (!ipahal_fltrt_objs[i+1].flt_generate_eq) {
-				IPAHAL_ERR(
-				  "No flt_generate_eq CB ipaver=%d\n",
-				  i+1);
-				WARN_ON(1);
-			}
-			if (!ipahal_fltrt_objs[i+1].rt_parse_hw_rule) {
-				IPAHAL_ERR(
-				  "No rt_parse_hw_rule CB ipaver=%d\n",
-				  i+1);
-				WARN_ON(1);
-			}
-			if (!ipahal_fltrt_objs[i+1].flt_parse_hw_rule) {
-				IPAHAL_ERR(
-				  "No flt_parse_hw_rule CB ipaver=%d\n",
-				  i+1);
-				WARN_ON(1);
-			}
-		}
-	}
-
-	eq_bits = 0;
-	eq_bitfield = ipahal_fltrt_objs[ipa_hw_type].eq_bitfield;
-	for (i = 0; i < IPA_EQ_MAX; i++) {
-		if (!IPA_IS_RULE_EQ_VALID(i))
-			continue;
-
-		if (eq_bits & IPA_GET_RULE_EQ_BIT_PTRN(eq_bitfield[i])) {
-			IPAHAL_ERR("more than eq with same bit. eq=%d\n", i);
-			WARN_ON(1);
-			return -EFAULT;
-		}
-		eq_bits |= IPA_GET_RULE_EQ_BIT_PTRN(eq_bitfield[i]);
-	}
-
-	mem = &ipahal_ctx->empty_fltrt_tbl;
-
-	/* setup an empty  table in system memory; This will
-	 * be used, for example, to delete a rt tbl safely
-	 */
-	mem->size = ipahal_fltrt_objs[ipa_hw_type].tbl_width;
-	mem->base = dma_alloc_coherent(ipahal_ctx->ipa_pdev, mem->size,
-		&mem->phys_base, GFP_KERNEL);
-	if (!mem->base) {
-		IPAHAL_ERR("DMA buff alloc fail %d bytes for empty tbl\n",
-			mem->size);
-		return -ENOMEM;
-	}
-
-	if (mem->phys_base &
-		ipahal_fltrt_objs[ipa_hw_type].sysaddr_alignment) {
-		IPAHAL_ERR("Empty table buf is not address aligned 0x%pad\n",
-			&mem->phys_base);
-		rc = -EFAULT;
-		goto clear_empty_tbl;
-	}
-
-	memset(mem->base, 0, mem->size);
-	IPAHAL_DBG("empty table allocated in system memory");
-
-	return 0;
-
-clear_empty_tbl:
-	dma_free_coherent(ipahal_ctx->ipa_pdev, mem->size, mem->base,
-		mem->phys_base);
-	return rc;
-}
-
-void ipahal_fltrt_destroy(void)
-{
-	IPAHAL_DBG("Entry\n");
-
-	if (ipahal_ctx && ipahal_ctx->empty_fltrt_tbl.base)
-		dma_free_coherent(ipahal_ctx->ipa_pdev,
-			ipahal_ctx->empty_fltrt_tbl.size,
-			ipahal_ctx->empty_fltrt_tbl.base,
-			ipahal_ctx->empty_fltrt_tbl.phys_base);
-}
-
-/* Get the H/W table (flt/rt) header width */
-u32 ipahal_get_hw_tbl_hdr_width(void)
-{
-	return ipahal_fltrt_objs[ipahal_ctx->hw_type].tbl_hdr_width;
-}
-
-/* Get the H/W local table (SRAM) address alignment
- * Tables headers references to local tables via offsets in SRAM
- * This function return the alignment of the offset that IPA expects
- */
-u32 ipahal_get_lcl_tbl_addr_alignment(void)
-{
-	return ipahal_fltrt_objs[ipahal_ctx->hw_type].lcladdr_alignment;
-}
-
-/*
- * Rule priority is used to distinguish rules order
- * at the integrated table consisting from hashable and
- * non-hashable tables. Max priority are rules that once are
- * scanned by IPA, IPA will not look for further rules and use it.
- */
-int ipahal_get_rule_max_priority(void)
-{
-	return ipahal_fltrt_objs[ipahal_ctx->hw_type].rule_max_prio;
-}
-
-/* Given a priority, calc and return the next lower one if it is in
- * legal range.
- */
-int ipahal_rule_decrease_priority(int *prio)
-{
-	struct ipahal_fltrt_obj *obj;
-
-	obj = &ipahal_fltrt_objs[ipahal_ctx->hw_type];
-
-	if (!prio) {
-		IPAHAL_ERR("Invalid Input\n");
-		return -EINVAL;
-	}
-
-	/* Priority logic is reverse. 0 priority considred max priority */
-	if (*prio > obj->rule_min_prio || *prio < obj->rule_max_prio) {
-		IPAHAL_ERR("Invalid given priority %d\n", *prio);
-		return -EINVAL;
-	}
-
-	*prio += 1;
-
-	if (*prio > obj->rule_min_prio) {
-		IPAHAL_ERR("Cannot decrease priority. Already on min\n");
-		*prio -= 1;
-		return -EFAULT;
-	}
-
-	return 0;
-}
-
-/* Does the given ID represents rule miss?
- * Rule miss ID, is always the max ID possible in the bit-pattern
- */
-bool ipahal_is_rule_miss_id(u32 id)
-{
-	return (id ==
-		((1U << ipahal_fltrt_objs[ipahal_ctx->hw_type].rule_id_bit_len)
-		-1));
-}
-
-/* Get rule ID with high bit only asserted
- * Used e.g. to create groups of IDs according to this bit
- */
-u32 ipahal_get_rule_id_hi_bit(void)
-{
-	return BIT(ipahal_fltrt_objs[ipahal_ctx->hw_type].rule_id_bit_len - 1);
-}
-
-/* Get the low value possible to be used for rule-id */
-u32 ipahal_get_low_rule_id(void)
-{
-	return  ipahal_fltrt_objs[ipahal_ctx->hw_type].low_rule_id;
-}
-
-/*
- * Is the given counter id valid
- */
-bool ipahal_is_rule_cnt_id_valid(u8 cnt_id)
-{
-	if (cnt_id < 0 || cnt_id > IPA_FLT_RT_HW_COUNTER)
-		return false;
-	return true;
-}
-
-
-/*
- * low value possible for counter hdl id
- */
-u32 ipahal_get_low_hdl_id(void)
-{
-	return IPA4_5_LOW_CNT_ID;
-}
-
-/*
- * max counter hdl id for stats
- */
-u32 ipahal_get_high_hdl_id(void)
-{
-	return IPA_MAX_FLT_RT_CNT_INDEX;
-}
-
-/*
- * ipahal_rt_generate_empty_img() - Generate empty route image
- *  Creates routing header buffer for the given tables number.
- *  For each table, make it point to the empty table on DDR.
- * @tbls_num: Number of tables. For each will have an entry in the header
- * @hash_hdr_size: SRAM buf size of the hash tbls hdr. Used for space check
- * @nhash_hdr_size: SRAM buf size of the nhash tbls hdr. Used for space check
- * @mem: mem object that points to DMA mem representing the hdr structure
- * @atomic: should DMA allocation be executed with atomic flag
- */
-int ipahal_rt_generate_empty_img(u32 tbls_num, u32 hash_hdr_size,
-	u32 nhash_hdr_size, struct ipa_mem_buffer *mem, bool atomic)
-{
-	int i;
-	u64 addr;
-	struct ipahal_fltrt_obj *obj;
-	int flag;
-
-	IPAHAL_DBG("Entry\n");
-
-	flag = atomic ? GFP_ATOMIC : GFP_KERNEL;
-	obj = &ipahal_fltrt_objs[ipahal_ctx->hw_type];
-
-	if (!tbls_num || !nhash_hdr_size || !mem) {
-		IPAHAL_ERR("Input Error: tbls_num=%d nhash_hdr_sz=%d mem=%pK\n",
-			tbls_num, nhash_hdr_size, mem);
-		return -EINVAL;
-	}
-	if (obj->support_hash && !hash_hdr_size) {
-		IPAHAL_ERR("Input Error: hash_hdr_sz=%d\n", hash_hdr_size);
-		return -EINVAL;
-	}
-
-	if (nhash_hdr_size < (tbls_num * obj->tbl_hdr_width)) {
-		IPAHAL_ERR("No enough spc at non-hash hdr blk for all tbls\n");
-		WARN_ON(1);
-		return -EINVAL;
-	}
-	if (obj->support_hash &&
-		(hash_hdr_size < (tbls_num * obj->tbl_hdr_width))) {
-		IPAHAL_ERR("No enough spc at hash hdr blk for all tbls\n");
-		WARN_ON(1);
-		return -EINVAL;
-	}
-
-	mem->size = tbls_num * obj->tbl_hdr_width;
-	mem->base = dma_alloc_coherent(ipahal_ctx->ipa_pdev, mem->size,
-		&mem->phys_base, flag);
-	if (!mem->base) {
-		IPAHAL_ERR("fail to alloc DMA buff of size %d\n", mem->size);
-		return -ENOMEM;
-	}
-
-	addr = obj->create_tbl_addr(true,
-		ipahal_ctx->empty_fltrt_tbl.phys_base);
-	for (i = 0; i < tbls_num; i++)
-		obj->write_val_to_hdr(addr,
-			mem->base + i * obj->tbl_hdr_width);
-
-	return 0;
-}
-
-/*
- * ipahal_flt_generate_empty_img() - Generate empty filter image
- *  Creates filter header buffer for the given tables number.
- *  For each table, make it point to the empty table on DDR.
- * @tbls_num: Number of tables. For each will have an entry in the header
- * @hash_hdr_size: SRAM buf size of the hash tbls hdr. Used for space check
- * @nhash_hdr_size: SRAM buf size of the nhash tbls hdr. Used for space check
- * @ep_bitmap: Bitmap representing the EP that has flt tables. The format
- *  should be: bit0->EP0, bit1->EP1
- *  If bitmap is zero -> create tbl without bitmap entry
- * @mem: mem object that points to DMA mem representing the hdr structure
- * @atomic: should DMA allocation be executed with atomic flag
- */
-int ipahal_flt_generate_empty_img(u32 tbls_num, u32 hash_hdr_size,
-	u32 nhash_hdr_size, u64 ep_bitmap, struct ipa_mem_buffer *mem,
-	bool atomic)
-{
-	int flt_spc;
-	u64 flt_bitmap;
-	int i;
-	u64 addr;
-	struct ipahal_fltrt_obj *obj;
-	int flag;
-
-	IPAHAL_DBG("Entry - ep_bitmap 0x%llx\n", ep_bitmap);
-
-	flag = atomic ? GFP_ATOMIC : GFP_KERNEL;
-	obj = &ipahal_fltrt_objs[ipahal_ctx->hw_type];
-
-	if (!tbls_num || !nhash_hdr_size || !mem) {
-		IPAHAL_ERR("Input Error: tbls_num=%d nhash_hdr_sz=%d mem=%pK\n",
-			tbls_num, nhash_hdr_size, mem);
-		return -EINVAL;
-	}
-	if (obj->support_hash && !hash_hdr_size) {
-		IPAHAL_ERR("Input Error: hash_hdr_sz=%d\n", hash_hdr_size);
-		return -EINVAL;
-	}
-
-	if (obj->support_hash) {
-		flt_spc = hash_hdr_size;
-		/* bitmap word */
-		if (ep_bitmap)
-			flt_spc -= obj->tbl_hdr_width;
-		flt_spc /= obj->tbl_hdr_width;
-		if (tbls_num > flt_spc)  {
-			IPAHAL_ERR("space for hash flt hdr is too small\n");
-			WARN_ON(1);
-			return -EPERM;
-		}
-	}
-
-	flt_spc = nhash_hdr_size;
-	/* bitmap word */
-	if (ep_bitmap)
-		flt_spc -= obj->tbl_hdr_width;
-	flt_spc /= obj->tbl_hdr_width;
-	if (tbls_num > flt_spc)  {
-		IPAHAL_ERR("space for non-hash flt hdr is too small\n");
-		WARN_ON(1);
-		return -EPERM;
-	}
-
-	mem->size = tbls_num * obj->tbl_hdr_width;
-	if (ep_bitmap)
-		mem->size += obj->tbl_hdr_width;
-	mem->base = dma_alloc_coherent(ipahal_ctx->ipa_pdev, mem->size,
-		&mem->phys_base, flag);
-	if (!mem->base) {
-		IPAHAL_ERR("fail to alloc DMA buff of size %d\n", mem->size);
-		return -ENOMEM;
-	}
-
-	if (ep_bitmap) {
-		flt_bitmap = obj->create_flt_bitmap(ep_bitmap);
-		IPAHAL_DBG("flt bitmap 0x%llx\n", flt_bitmap);
-		obj->write_val_to_hdr(flt_bitmap, mem->base);
-	}
-
-	addr = obj->create_tbl_addr(true,
-		ipahal_ctx->empty_fltrt_tbl.phys_base);
-
-	if (ep_bitmap) {
-		for (i = 1; i <= tbls_num; i++)
-			obj->write_val_to_hdr(addr,
-				mem->base + i * obj->tbl_hdr_width);
-	} else {
-		for (i = 0; i < tbls_num; i++)
-			obj->write_val_to_hdr(addr,
-				mem->base + i * obj->tbl_hdr_width);
-	}
-
-	return 0;
-}
-
-/*
- * ipa_fltrt_alloc_init_tbl_hdr() - allocate and initialize buffers for
- *  flt/rt tables headers to be filled into sram. Init each table to point
- *  to empty system table
- * @params: Allocate IN and OUT params
- *
- * Return: 0 on success, negative on failure
- */
-static int ipa_fltrt_alloc_init_tbl_hdr(
-	struct ipahal_fltrt_alloc_imgs_params *params)
-{
-	u64 addr;
-	int i;
-	struct ipahal_fltrt_obj *obj;
-	gfp_t flag = GFP_KERNEL;
-
-	obj = &ipahal_fltrt_objs[ipahal_ctx->hw_type];
-
-	if (!params) {
-		IPAHAL_ERR_RL("Input error: params=%pK\n", params);
-		return -EINVAL;
-	}
-
-	params->nhash_hdr.size = params->tbls_num * obj->tbl_hdr_width;
-alloc:
-	params->nhash_hdr.base = dma_alloc_coherent(ipahal_ctx->ipa_pdev,
-		params->nhash_hdr.size,
-		&params->nhash_hdr.phys_base, flag);
-	if (!params->nhash_hdr.base) {
-		if (flag == GFP_KERNEL) {
-			flag = GFP_ATOMIC;
-			goto alloc;
-		}
-		IPAHAL_ERR_RL("fail to alloc DMA buff of size %d\n",
-			params->nhash_hdr.size);
-		goto nhash_alloc_fail;
-	}
-
-	if (obj->support_hash) {
-		params->hash_hdr.size = params->tbls_num * obj->tbl_hdr_width;
-		params->hash_hdr.base = dma_alloc_coherent(ipahal_ctx->ipa_pdev,
-			params->hash_hdr.size, &params->hash_hdr.phys_base,
-			GFP_KERNEL);
-		if (!params->hash_hdr.base) {
-			IPAHAL_ERR_RL("fail to alloc DMA buff of size %d\n",
-				params->hash_hdr.size);
-			goto hash_alloc_fail;
-		}
-	}
-
-	addr = obj->create_tbl_addr(true,
-		ipahal_ctx->empty_fltrt_tbl.phys_base);
-	for (i = 0; i < params->tbls_num; i++) {
-		obj->write_val_to_hdr(addr,
-			params->nhash_hdr.base + i * obj->tbl_hdr_width);
-		if (obj->support_hash)
-			obj->write_val_to_hdr(addr,
-				params->hash_hdr.base +
-				i * obj->tbl_hdr_width);
-	}
-
-	return 0;
-
-hash_alloc_fail:
-	ipahal_free_dma_mem(&params->nhash_hdr);
-nhash_alloc_fail:
-	return -ENOMEM;
-}
-
-/*
- * ipa_fltrt_alloc_lcl_bdy() - allocate and initialize buffers for
- *  local flt/rt tables bodies to be filled into sram
- * @params: Allocate IN and OUT params
- *
- * Return: 0 on success, negative on failure
- */
-static int ipa_fltrt_alloc_lcl_bdy(
-	struct ipahal_fltrt_alloc_imgs_params *params)
-{
-	struct ipahal_fltrt_obj *obj;
-
-	obj = &ipahal_fltrt_objs[ipahal_ctx->hw_type];
-
-	/* The HAL allocates larger sizes than the given effective ones
-	 * for alignments and border indications
-	 */
-	IPAHAL_DBG_LOW("lcl tbl bdy total effective sizes: hash=%u nhash=%u\n",
-		params->total_sz_lcl_hash_tbls,
-		params->total_sz_lcl_nhash_tbls);
-
-	IPAHAL_DBG_LOW("lcl tbl bdy count: hash=%u nhash=%u\n",
-		params->num_lcl_hash_tbls,
-		params->num_lcl_nhash_tbls);
-
-	/* Align the sizes to coop with termination word
-	 *  and H/W local table start offset alignment
-	 */
-	if (params->nhash_bdy.size) {
-		params->nhash_bdy.size = params->total_sz_lcl_nhash_tbls;
-		/* for table terminator */
-		params->nhash_bdy.size += obj->tbl_width *
-			params->num_lcl_nhash_tbls;
-		/* align the start of local rule-set */
-		params->nhash_bdy.size += obj->lcladdr_alignment *
-			params->num_lcl_nhash_tbls;
-		/* SRAM block size alignment */
-		params->nhash_bdy.size += obj->blk_sz_alignment;
-		params->nhash_bdy.size &= ~(obj->blk_sz_alignment);
-
-		IPAHAL_DBG_LOW("nhash lcl tbl bdy total h/w size = %u\n",
-			params->nhash_bdy.size);
-
-		params->nhash_bdy.base = dma_zalloc_coherent(
-			ipahal_ctx->ipa_pdev, params->nhash_bdy.size,
-			&params->nhash_bdy.phys_base, GFP_KERNEL);
-		if (!params->nhash_bdy.base) {
-			IPAHAL_ERR("fail to alloc DMA buff of size %d\n",
-				params->nhash_bdy.size);
-			return -ENOMEM;
-		}
-	}
-
-	if (!obj->support_hash && params->hash_bdy.size) {
-		IPAHAL_ERR("No HAL Hash tbls support - Will be ignored\n");
-		WARN_ON(1);
-	}
-
-	if (obj->support_hash && params->hash_bdy.size) {
-		params->hash_bdy.size = params->total_sz_lcl_hash_tbls;
-		/* for table terminator */
-		params->hash_bdy.size += obj->tbl_width *
-			params->num_lcl_hash_tbls;
-		/* align the start of local rule-set */
-		params->hash_bdy.size += obj->lcladdr_alignment *
-			params->num_lcl_hash_tbls;
-		/* SRAM block size alignment */
-		params->hash_bdy.size += obj->blk_sz_alignment;
-		params->hash_bdy.size &= ~(obj->blk_sz_alignment);
-
-		IPAHAL_DBG_LOW("hash lcl tbl bdy total h/w size = %u\n",
-			params->hash_bdy.size);
-
-		params->hash_bdy.base = dma_zalloc_coherent(
-			ipahal_ctx->ipa_pdev, params->hash_bdy.size,
-			&params->hash_bdy.phys_base, GFP_KERNEL);
-		if (!params->hash_bdy.base) {
-			IPAHAL_ERR("fail to alloc DMA buff of size %d\n",
-				params->hash_bdy.size);
-			goto hash_bdy_fail;
-		}
-	}
-
-	return 0;
-
-hash_bdy_fail:
-	if (params->nhash_bdy.size)
-		ipahal_free_dma_mem(&params->nhash_bdy);
-
-	return -ENOMEM;
-}
-
-/*
- * ipahal_fltrt_allocate_hw_tbl_imgs() - Allocate tbl images DMA structures
- *  Used usually during commit.
- *  Allocates header structures and init them to point to empty DDR table
- *  Allocate body strucutres for local bodies tables
- * @params: Parameters for IN and OUT regard the allocation.
- */
-int ipahal_fltrt_allocate_hw_tbl_imgs(
-	struct ipahal_fltrt_alloc_imgs_params *params)
-{
-	IPAHAL_DBG_LOW("Entry\n");
-
-	/* Input validation */
-	if (!params) {
-		IPAHAL_ERR_RL("Input err: no params\n");
-		return -EINVAL;
-	}
-	if (params->ipt >= IPA_IP_MAX) {
-		IPAHAL_ERR_RL("Input err: Invalid ip type %d\n", params->ipt);
-		return -EINVAL;
-	}
-
-	if (ipa_fltrt_alloc_init_tbl_hdr(params)) {
-		IPAHAL_ERR_RL("fail to alloc and init tbl hdr\n");
-		return -ENOMEM;
-	}
-
-	if (ipa_fltrt_alloc_lcl_bdy(params)) {
-		IPAHAL_ERR_RL("fail to alloc tbl bodies\n");
-		goto bdy_alloc_fail;
-	}
-
-	return 0;
-
-bdy_alloc_fail:
-	ipahal_free_dma_mem(&params->nhash_hdr);
-	if (params->hash_hdr.size)
-		ipahal_free_dma_mem(&params->hash_hdr);
-	return -ENOMEM;
-}
-
-/*
- * ipahal_fltrt_allocate_hw_sys_tbl() - Allocate DMA mem for H/W flt/rt sys tbl
- * @tbl_mem: IN/OUT param. size for effective table size. Pointer, for the
- *  allocated memory.
- *
- * The size is adapted for needed alignments/borders.
- */
-int ipahal_fltrt_allocate_hw_sys_tbl(struct ipa_mem_buffer *tbl_mem)
-{
-	struct ipahal_fltrt_obj *obj;
-
-	IPAHAL_DBG_LOW("Entry\n");
-
-	if (!tbl_mem) {
-		IPAHAL_ERR("Input err\n");
-		return -EINVAL;
-	}
-
-	if (!tbl_mem->size) {
-		IPAHAL_ERR("Input err: zero table size\n");
-		return -EINVAL;
-	}
-
-	obj = &ipahal_fltrt_objs[ipahal_ctx->hw_type];
-
-	/* add word for rule-set terminator */
-	tbl_mem->size += obj->tbl_width;
-
-	tbl_mem->base = dma_alloc_coherent(ipahal_ctx->ipa_pdev, tbl_mem->size,
-		&tbl_mem->phys_base, GFP_KERNEL);
-	if (!tbl_mem->base) {
-		IPAHAL_ERR("fail to alloc DMA buf of size %d\n",
-			tbl_mem->size);
-		return -ENOMEM;
-	}
-	if (tbl_mem->phys_base & obj->sysaddr_alignment) {
-		IPAHAL_ERR("sys rt tbl address is not aligned\n");
-		goto align_err;
-	}
-
-	memset(tbl_mem->base, 0, tbl_mem->size);
-
-	return 0;
-
-align_err:
-	ipahal_free_dma_mem(tbl_mem);
-	return -EPERM;
-}
-
-/*
- * ipahal_fltrt_write_addr_to_hdr() - Fill table header with table address
- *  Given table addr/offset, adapt it to IPA H/W format and write it
- *  to given header index.
- * @addr: Address or offset to be used
- * @hdr_base: base address of header structure to write the address
- * @hdr_idx: index of the address in the header structure
- * @is_sys: Is it system address or local offset
- */
-int ipahal_fltrt_write_addr_to_hdr(u64 addr, void *hdr_base, u32 hdr_idx,
-	bool is_sys)
-{
-	struct ipahal_fltrt_obj *obj;
-	u64 hwaddr;
-	u8 *hdr;
-
-	IPAHAL_DBG_LOW("Entry\n");
-
-	obj = &ipahal_fltrt_objs[ipahal_ctx->hw_type];
-
-	if (!addr || !hdr_base) {
-		IPAHAL_ERR("Input err: addr=0x%llx hdr_base=%pK\n",
-			addr, hdr_base);
-		return -EINVAL;
-	}
-
-	hdr = (u8 *)hdr_base;
-	hdr += hdr_idx * obj->tbl_hdr_width;
-	hwaddr = obj->create_tbl_addr(is_sys, addr);
-	obj->write_val_to_hdr(hwaddr, hdr);
-
-	return 0;
-}
-
-/*
- * ipahal_fltrt_read_addr_from_hdr() - Given sram address, read it's
- *  content (physical address or offset) and parse it.
- * @hdr_base: base sram address of the header structure.
- * @hdr_idx: index of the header entry line in the header structure.
- * @addr: The parsed address - Out parameter
- * @is_sys: Is this system or local address - Out parameter
- */
-int ipahal_fltrt_read_addr_from_hdr(void *hdr_base, u32 hdr_idx, u64 *addr,
-	bool *is_sys)
-{
-	struct ipahal_fltrt_obj *obj;
-	u64 hwaddr;
-	u8 *hdr;
-
-	IPAHAL_DBG_LOW("Entry\n");
-
-	obj = &ipahal_fltrt_objs[ipahal_ctx->hw_type];
-
-	if (!addr || !hdr_base || !is_sys) {
-		IPAHAL_ERR("Input err: addr=%pK hdr_base=%pK is_sys=%pK\n",
-			addr, hdr_base, is_sys);
-		return -EINVAL;
-	}
-
-	hdr = (u8 *)hdr_base;
-	hdr += hdr_idx * obj->tbl_hdr_width;
-	hwaddr = *((u64 *)hdr);
-	obj->parse_tbl_addr(hwaddr, addr, is_sys);
-	return 0;
-}
-
-/*
- * ipahal_rt_generate_hw_rule() - generates the routing hardware rule
- * @params: Params for the rule creation.
- * @hw_len: Size of the H/W rule to be returned
- * @buf: Buffer to build the rule in. If buf is NULL, then the rule will
- *  be built in internal temp buf. This is used e.g. to get the rule size
- *  only.
- */
-int ipahal_rt_generate_hw_rule(struct ipahal_rt_rule_gen_params *params,
-	u32 *hw_len, u8 *buf)
-{
-	struct ipahal_fltrt_obj *obj;
-	u8 *tmp = NULL;
-	int rc;
-
-	IPAHAL_DBG_LOW("Entry\n");
-
-	if (!params || !hw_len) {
-		IPAHAL_ERR("Input err: params=%pK hw_len=%pK\n",
-			params, hw_len);
-		return -EINVAL;
-	}
-	if (!params->rule) {
-		IPAHAL_ERR("Input err: invalid rule\n");
-		return -EINVAL;
-	}
-	if (params->ipt >= IPA_IP_MAX) {
-		IPAHAL_ERR("Input err: Invalid ip type %d\n", params->ipt);
-		return -EINVAL;
-	}
-
-	obj = &ipahal_fltrt_objs[ipahal_ctx->hw_type];
-
-	if (buf == NULL) {
-		tmp = kzalloc(obj->rule_buf_size, GFP_KERNEL);
-		if (!tmp)
-			return -ENOMEM;
-		buf = tmp;
-	} else {
-		if ((long)buf & obj->rule_start_alignment) {
-			IPAHAL_ERR("buff is not rule start aligned\n");
-			return -EPERM;
-		}
-	}
-
-	rc = obj->rt_generate_hw_rule(params, hw_len, buf);
-	if (!tmp && !rc) {
-		/* write the rule-set terminator */
-		memset(buf + *hw_len, 0, obj->tbl_width);
-	}
-
-	kfree(tmp);
-
-	return rc;
-}
-
-/*
- * ipahal_flt_generate_hw_rule() - generates the filtering hardware rule.
- * @params: Params for the rule creation.
- * @hw_len: Size of the H/W rule to be returned
- * @buf: Buffer to build the rule in. If buf is NULL, then the rule will
- *  be built in internal temp buf. This is used e.g. to get the rule size
- *  only.
- */
-int ipahal_flt_generate_hw_rule(struct ipahal_flt_rule_gen_params *params,
-	u32 *hw_len, u8 *buf)
-{
-	struct ipahal_fltrt_obj *obj;
-	u8 *tmp = NULL;
-	int rc;
-
-	IPAHAL_DBG_LOW("Entry\n");
-
-	if (!params || !hw_len) {
-		IPAHAL_ERR("Input err: params=%pK hw_len=%pK\n",
-			params, hw_len);
-		return -EINVAL;
-	}
-	if (!params->rule) {
-		IPAHAL_ERR("Input err: invalid rule\n");
-		return -EINVAL;
-	}
-	if (params->ipt >= IPA_IP_MAX) {
-		IPAHAL_ERR("Input err: Invalid ip type %d\n", params->ipt);
-		return -EINVAL;
-	}
-
-	obj = &ipahal_fltrt_objs[ipahal_ctx->hw_type];
-
-	if (buf == NULL) {
-		tmp = kzalloc(obj->rule_buf_size, GFP_KERNEL);
-		if (!tmp) {
-			IPAHAL_ERR("failed to alloc %u bytes\n",
-				obj->rule_buf_size);
-			return -ENOMEM;
-		}
-		buf = tmp;
-	} else
-		if ((long)buf & obj->rule_start_alignment) {
-			IPAHAL_ERR("buff is not rule rule start aligned\n");
-			return -EPERM;
-		}
-
-	rc = obj->flt_generate_hw_rule(params, hw_len, buf);
-	if (!tmp && !rc) {
-		/* write the rule-set terminator */
-		memset(buf + *hw_len, 0, obj->tbl_width);
-	}
-
-	kfree(tmp);
-
-	return rc;
-
-}
-
-/*
- * ipahal_flt_generate_equation() - generate flt rule in equation form
- *  Will build equation form flt rule from given info.
- * @ipt: IP family
- * @attrib: Rule attribute to be generated
- * @eq_atrb: Equation form generated rule
- * Note: Usage example: Pass the generated form to other sub-systems
- *  for inter-subsystems rules exchange.
- */
-int ipahal_flt_generate_equation(enum ipa_ip_type ipt,
-		const struct ipa_rule_attrib *attrib,
-		struct ipa_ipfltri_rule_eq *eq_atrb)
-{
-	IPAHAL_DBG_LOW("Entry\n");
-
-	if (ipt >= IPA_IP_MAX) {
-		IPAHAL_ERR_RL("Input err: Invalid ip type %d\n", ipt);
-		return -EINVAL;
-	}
-
-	if (!attrib || !eq_atrb) {
-		IPAHAL_ERR_RL("Input err: attrib=%pK eq_atrb=%pK\n",
-			attrib, eq_atrb);
-		return -EINVAL;
-	}
-
-	return ipahal_fltrt_objs[ipahal_ctx->hw_type].flt_generate_eq(ipt,
-		attrib, eq_atrb);
-
-}
-
-/*
- * ipahal_rt_parse_hw_rule() - Parse H/W formated rt rule
- *  Given the rule address, read the rule info from H/W and parse it.
- * @rule_addr: Rule address (virtual memory)
- * @rule: Out parameter for parsed rule info
- */
-int ipahal_rt_parse_hw_rule(u8 *rule_addr,
-	struct ipahal_rt_rule_entry *rule)
-{
-	IPAHAL_DBG_LOW("Entry\n");
-
-	if (!rule_addr || !rule) {
-		IPAHAL_ERR("Input err: rule_addr=%pK rule=%pK\n",
-			rule_addr, rule);
-		return -EINVAL;
-	}
-
-	return ipahal_fltrt_objs[ipahal_ctx->hw_type].rt_parse_hw_rule(
-		rule_addr, rule);
-}
-
-/*
- * ipahal_flt_parse_hw_rule() - Parse H/W formated flt rule
- *  Given the rule address, read the rule info from H/W and parse it.
- * @rule_addr: Rule address (virtual memory)
- * @rule: Out parameter for parsed rule info
- */
-int ipahal_flt_parse_hw_rule(u8 *rule_addr,
-	struct ipahal_flt_rule_entry *rule)
-{
-	IPAHAL_DBG_LOW("Entry\n");
-
-	if (!rule_addr || !rule) {
-		IPAHAL_ERR("Input err: rule_addr=%pK rule=%pK\n",
-			rule_addr, rule);
-		return -EINVAL;
-	}
-
-	return ipahal_fltrt_objs[ipahal_ctx->hw_type].flt_parse_hw_rule(
-		rule_addr, rule);
-}
-

ipa/ipa_v3/ipahal/ipahal_fltrt.h

@@ -1,308 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2016-2019, The Linux Foundation. All rights reserved.
- */
-
-#ifndef _IPAHAL_FLTRT_H_
-#define _IPAHAL_FLTRT_H_
-
-/*
- * struct ipahal_fltrt_alloc_imgs_params - Params for tbls imgs allocations
- *  The allocation logic will allocate DMA memory representing the header.
- *  If the bodies are local (SRAM) the allocation will allocate
- *  a DMA buffers that would contain the content of these local tables in raw
- * @ipt: IP version type
- * @tbls_num: Number of tables to represent by the header
- * @num_lcl_hash_tbls: Number of local (sram) hashable tables
- * @num_lcl_nhash_tbls: Number of local (sram) non-hashable tables
- * @total_sz_lcl_hash_tbls: Total size of local hashable tables
- * @total_sz_lcl_nhash_tbls: Total size of local non-hashable tables
- * @hash_hdr/nhash_hdr: OUT params for the header structures
- * @hash_bdy/nhash_bdy: OUT params for the local body structures
- */
-struct ipahal_fltrt_alloc_imgs_params {
-	enum ipa_ip_type ipt;
-	u32 tbls_num;
-	u32 num_lcl_hash_tbls;
-	u32 num_lcl_nhash_tbls;
-	u32 total_sz_lcl_hash_tbls;
-	u32 total_sz_lcl_nhash_tbls;
-
-	/* OUT PARAMS */
-	struct ipa_mem_buffer hash_hdr;
-	struct ipa_mem_buffer nhash_hdr;
-	struct ipa_mem_buffer hash_bdy;
-	struct ipa_mem_buffer nhash_bdy;
-};
-
-/*
- * enum ipahal_rt_rule_hdr_type - Header type used in rt rules
- * @IPAHAL_RT_RULE_HDR_NONE: No header is used
- * @IPAHAL_RT_RULE_HDR_RAW: Raw header is used
- * @IPAHAL_RT_RULE_HDR_PROC_CTX: Header Processing context is used
- */
-enum ipahal_rt_rule_hdr_type {
-	IPAHAL_RT_RULE_HDR_NONE,
-	IPAHAL_RT_RULE_HDR_RAW,
-	IPAHAL_RT_RULE_HDR_PROC_CTX,
-};
-
-/*
- * struct ipahal_rt_rule_gen_params - Params for generating rt rule
- * @ipt: IP family version
- * @dst_pipe_idx: Destination pipe index
- * @hdr_type: Header type to be used
- * @hdr_lcl: Does header on local or system table?
- * @hdr_ofst: Offset of the header in the header table
- * @priority: Rule priority
- * @id: Rule ID
- * @cnt_idx: Stats counter index
- * @rule: Rule info
- */
-struct ipahal_rt_rule_gen_params {
-	enum ipa_ip_type ipt;
-	int dst_pipe_idx;
-	enum ipahal_rt_rule_hdr_type hdr_type;
-	bool hdr_lcl;
-	u32 hdr_ofst;
-	u32 priority;
-	u32 id;
-	u8 cnt_idx;
-	const struct ipa_rt_rule_i *rule;
-};
-
-/*
- * struct ipahal_rt_rule_entry - Rt rule info parsed from H/W
- * @dst_pipe_idx: Destination pipe index
- * @hdr_lcl: Does the references header located in sram or system mem?
- * @hdr_ofst: Offset of the header in the header table
- * @hdr_type: Header type to be used
- * @priority: Rule priority
- * @retain_hdr: to retain the removed header in header removal
- * @id: Rule ID
- * @cnt_idx: stats counter index
- * @eq_attrib: Equations and their params in the rule
- * @rule_size: Rule size in memory
- */
-struct ipahal_rt_rule_entry {
-	int dst_pipe_idx;
-	bool hdr_lcl;
-	u32 hdr_ofst;
-	enum ipahal_rt_rule_hdr_type hdr_type;
-	u32 priority;
-	bool retain_hdr;
-	u32 id;
-	u8 cnt_idx;
-	struct ipa_ipfltri_rule_eq eq_attrib;
-	u32 rule_size;
-};
-
-/*
- * struct ipahal_flt_rule_gen_params - Params for generating flt rule
- * @ipt: IP family version
- * @rt_tbl_idx: Routing table the rule pointing to
- * @priority: Rule priority
- * @id: Rule ID
- * @cnt_idx: Stats counter index
- * @rule: Rule info
- */
-struct ipahal_flt_rule_gen_params {
-	enum ipa_ip_type ipt;
-	u32 rt_tbl_idx;
-	u32 priority;
-	u32 id;
-	u8 cnt_idx;
-	const struct ipa_flt_rule_i *rule;
-};
-
-/*
- * struct ipahal_flt_rule_entry - Flt rule info parsed from H/W
- * @rule: Rule info
- * @priority: Rule priority
- * @id: Rule ID
- * @cnt_idx: stats counter index
- * @rule_size: Rule size in memory
- */
-struct ipahal_flt_rule_entry {
-	struct ipa_flt_rule_i rule;
-	u32 priority;
-	u32 id;
-	u8 cnt_idx;
-	u32 rule_size;
-};
-
-/* Get the H/W table (flt/rt) header width */
-u32 ipahal_get_hw_tbl_hdr_width(void);
-
-/* Get the H/W local table (SRAM) address alignment
- * Tables headers references to local tables via offsets in SRAM
- * This function return the alignment of the offset that IPA expects
- */
-u32 ipahal_get_lcl_tbl_addr_alignment(void);
-
-/*
- * Rule priority is used to distinguish rules order
- * at the integrated table consisting from hashable and
- * non-hashable tables. Max priority are rules that once are
- * scanned by IPA, IPA will not look for further rules and use it.
- */
-int ipahal_get_rule_max_priority(void);
-
-/* Given a priority, calc and return the next lower one if it is in
- * legal range.
- */
-int ipahal_rule_decrease_priority(int *prio);
-
-/* Does the given ID represents rule miss? */
-bool ipahal_is_rule_miss_id(u32 id);
-
-/* Get rule ID with high bit only asserted
- * Used e.g. to create groups of IDs according to this bit
- */
-u32 ipahal_get_rule_id_hi_bit(void);
-
-/* Get the low value possible to be used for rule-id */
-u32 ipahal_get_low_rule_id(void);
-
-/*
- * low value possible for counter hdl id
- */
-u32 ipahal_get_low_hdl_id(void);
-
-/*
- * max counter hdl id for stats
- */
-u32 ipahal_get_high_hdl_id(void);
-
-/* used for query check and associated with rt/flt rules */
-bool ipahal_is_rule_cnt_id_valid(u8 cnt_id);
-
-/* max rule id for stats */
-bool ipahal_get_max_stats_rule_id(void);
-
-/*
- * ipahal_rt_generate_empty_img() - Generate empty route image
- *  Creates routing header buffer for the given tables number.
- * For each table, make it point to the empty table on DDR.
- * @tbls_num: Number of tables. For each will have an entry in the header
- * @hash_hdr_size: SRAM buf size of the hash tbls hdr. Used for space check
- * @nhash_hdr_size: SRAM buf size of the nhash tbls hdr. Used for space check
- * @mem: mem object that points to DMA mem representing the hdr structure
- * @atomic: should DMA allocation be executed with atomic flag
- */
-int ipahal_rt_generate_empty_img(u32 tbls_num, u32 hash_hdr_size,
-	u32 nhash_hdr_size, struct ipa_mem_buffer *mem, bool atomic);
-
-/*
- * ipahal_flt_generate_empty_img() - Generate empty filter image
- *  Creates filter header buffer for the given tables number.
- *  For each table, make it point to the empty table on DDR.
- * @tbls_num: Number of tables. For each will have an entry in the header
- * @hash_hdr_size: SRAM buf size of the hash tbls hdr. Used for space check
- * @nhash_hdr_size: SRAM buf size of the nhash tbls hdr. Used for space check
- * @ep_bitmap: Bitmap representing the EP that has flt tables. The format
- *  should be: bit0->EP0, bit1->EP1
- * @mem: mem object that points to DMA mem representing the hdr structure
- * @atomic: should DMA allocation be executed with atomic flag
- */
-int ipahal_flt_generate_empty_img(u32 tbls_num, u32 hash_hdr_size,
-	u32 nhash_hdr_size, u64 ep_bitmap, struct ipa_mem_buffer *mem,
-	bool atomic);
-
-/*
- * ipahal_fltrt_allocate_hw_tbl_imgs() - Allocate tbl images DMA structures
- *  Used usually during commit.
- *  Allocates header structures and init them to point to empty DDR table
- *  Allocate body strucutres for local bodies tables
- * @params: Parameters for IN and OUT regard the allocation.
- */
-int ipahal_fltrt_allocate_hw_tbl_imgs(
-	struct ipahal_fltrt_alloc_imgs_params *params);
-
-/*
- * ipahal_fltrt_allocate_hw_sys_tbl() - Allocate DMA mem for H/W flt/rt sys tbl
- * @tbl_mem: IN/OUT param. size for effective table size. Pointer, for the
- *  allocated memory.
- *
- * The size is adapted for needed alignments/borders.
- */
-int ipahal_fltrt_allocate_hw_sys_tbl(struct ipa_mem_buffer *tbl_mem);
-
-/*
- * ipahal_fltrt_write_addr_to_hdr() - Fill table header with table address
- *  Given table addr/offset, adapt it to IPA H/W format and write it
- *  to given header index.
- * @addr: Address or offset to be used
- * @hdr_base: base address of header structure to write the address
- * @hdr_idx: index of the address in the header structure
- * @is_sys: Is it system address or local offset
- */
-int ipahal_fltrt_write_addr_to_hdr(u64 addr, void *hdr_base, u32 hdr_idx,
-	bool is_sys);
-
-/*
- * ipahal_fltrt_read_addr_from_hdr() - Given sram address, read it's
- *  content (physical address or offset) and parse it.
- * @hdr_base: base sram address of the header structure.
- * @hdr_idx: index of the header entry line in the header structure.
- * @addr: The parsed address - Out parameter
- * @is_sys: Is this system or local address - Out parameter
- */
-int ipahal_fltrt_read_addr_from_hdr(void *hdr_base, u32 hdr_idx, u64 *addr,
-	bool *is_sys);
-
-/*
- * ipahal_rt_generate_hw_rule() - generates the routing hardware rule.
- * @params: Params for the rule creation.
- * @hw_len: Size of the H/W rule to be returned
- * @buf: Buffer to build the rule in. If buf is NULL, then the rule will
- *  be built in internal temp buf. This is used e.g. to get the rule size
- *  only.
- */
-int ipahal_rt_generate_hw_rule(struct ipahal_rt_rule_gen_params *params,
-	u32 *hw_len, u8 *buf);
-
-/*
- * ipahal_flt_generate_hw_rule() - generates the filtering hardware rule.
- * @params: Params for the rule creation.
- * @hw_len: Size of the H/W rule to be returned
- * @buf: Buffer to build the rule in. If buf is NULL, then the rule will
- *  be built in internal temp buf. This is used e.g. to get the rule size
- *  only.
- */
-int ipahal_flt_generate_hw_rule(struct ipahal_flt_rule_gen_params *params,
-	u32 *hw_len, u8 *buf);
-
-/*
- * ipahal_flt_generate_equation() - generate flt rule in equation form
- *  Will build equation form flt rule from given info.
- * @ipt: IP family
- * @attrib: Rule attribute to be generated
- * @eq_atrb: Equation form generated rule
- * Note: Usage example: Pass the generated form to other sub-systems
- *  for inter-subsystems rules exchange.
- */
-int ipahal_flt_generate_equation(enum ipa_ip_type ipt,
-		const struct ipa_rule_attrib *attrib,
-		struct ipa_ipfltri_rule_eq *eq_atrb);
-
-/*
- * ipahal_rt_parse_hw_rule() - Parse H/W formated rt rule
- *  Given the rule address, read the rule info from H/W and parse it.
- * @rule_addr: Rule address (virtual memory)
- * @rule: Out parameter for parsed rule info
- */
-int ipahal_rt_parse_hw_rule(u8 *rule_addr,
-	struct ipahal_rt_rule_entry *rule);
-
-/*
- * ipahal_flt_parse_hw_rule() - Parse H/W formated flt rule
- *  Given the rule address, read the rule info from H/W and parse it.
- * @rule_addr: Rule address (virtual memory)
- * @rule: Out parameter for parsed rule info
- */
-int ipahal_flt_parse_hw_rule(u8 *rule_addr,
-	struct ipahal_flt_rule_entry *rule);
-
-
-#endif /* _IPAHAL_FLTRT_H_ */

ipa/ipa_v3/ipahal/ipahal_fltrt_i.h

@@ -1,257 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
- */
-
-#ifndef _IPAHAL_FLTRT_I_H_
-#define _IPAHAL_FLTRT_I_H_
-
-/*
- * enum ipa_fltrt_equations - RULE equations
- *  These are names values to the equations that can be used
- *  The HAL layer holds mapping between these names and H/W
- *  presentation.
- */
-enum ipa_fltrt_equations {
-	IPA_TOS_EQ,
-	IPA_PROTOCOL_EQ,
-	IPA_TC_EQ,
-	IPA_OFFSET_MEQ128_0,
-	IPA_OFFSET_MEQ128_1,
-	IPA_OFFSET_MEQ32_0,
-	IPA_OFFSET_MEQ32_1,
-	IPA_IHL_OFFSET_MEQ32_0,
-	IPA_IHL_OFFSET_MEQ32_1,
-	IPA_METADATA_COMPARE,
-	IPA_IHL_OFFSET_RANGE16_0,
-	IPA_IHL_OFFSET_RANGE16_1,
-	IPA_IHL_OFFSET_EQ_32,
-	IPA_IHL_OFFSET_EQ_16,
-	IPA_FL_EQ,
-	IPA_IS_FRAG,
-	IPA_IS_PURE_ACK,
-	IPA_EQ_MAX,
-};
-
-/* Width and Alignment values for H/W structures.
- * Specific for IPA version.
- */
-#define IPA3_0_HW_TBL_SYSADDR_ALIGNMENT (127)
-#define IPA3_0_HW_TBL_LCLADDR_ALIGNMENT (7)
-#define IPA3_0_HW_TBL_BLK_SIZE_ALIGNMENT (127)
-#define IPA3_0_HW_TBL_WIDTH (8)
-#define IPA3_0_HW_TBL_HDR_WIDTH (8)
-#define IPA3_0_HW_TBL_ADDR_MASK (127)
-#define IPA3_0_HW_RULE_BUF_SIZE (256)
-#define IPA3_0_HW_RULE_START_ALIGNMENT (7)
-
-
-/*
- * Rules Priority.
- * Needed due to rules classification to hashable and non-hashable.
- * Higher priority is lower in number. i.e. 0 is highest priority
- */
-#define IPA3_0_RULE_MAX_PRIORITY (0)
-#define IPA3_0_RULE_MIN_PRIORITY (1023)
-
-/*
- * RULE ID, bit length (e.g. 10 bits).
- */
-#define IPA3_0_RULE_ID_BIT_LEN (10)
-#define IPA3_0_LOW_RULE_ID (1)
-
-/*
- * COUNTER ID, LOW COUNTER ID.
- */
-#define IPA4_5_LOW_CNT_ID (1)
-
-/**
- * struct ipa3_0_rt_rule_hw_hdr - HW header of IPA routing rule
- * @word: routing rule header properties
- * @en_rule: enable rule - Equation bit fields
- * @pipe_dest_idx: destination pipe index
- * @system: Is referenced header is lcl or sys memory
- * @hdr_offset: header offset
- * @proc_ctx: whether hdr_offset points to header table or to
- *	header processing context table
- * @priority: Rule priority. Added to distinguish rules order
- *  at the integrated table consisting from hashable and
- *  non-hashable parts
- * @rsvd1: reserved bits
- * @retain_hdr: added to add back to the packet the header removed
- *  as part of header removal. This will be done as part of
- *  header insertion block.
- * @rule_id: rule ID that will be returned in the packet status
- * @rsvd2: reserved bits
- */
-struct ipa3_0_rt_rule_hw_hdr {
-	union {
-		u64 word;
-		struct {
-			u64 en_rule:16;
-			u64 pipe_dest_idx:5;
-			u64 system:1;
-			u64 hdr_offset:9;
-			u64 proc_ctx:1;
-			u64 priority:10;
-			u64 rsvd1:5;
-			u64 retain_hdr:1;
-			u64 rule_id:10;
-			u64 rsvd2:6;
-		} hdr;
-	} u;
-};
-
-/**
- * struct ipa3_0_rt_rule_hw_hdr - HW header of IPA routing rule
- * @word: routing rule header properties
- * @en_rule: enable rule - Equation bit fields
- * @pipe_dest_idx: destination pipe index
- * @system: Is referenced header is lcl or sys memory
- * @hdr_offset: header offset
- * @proc_ctx: whether hdr_offset points to header table or to
- *	header processing context table
- * @priority: Rule priority. Added to distinguish rules order
- *  at the integrated table consisting from hashable and
- *  non-hashable parts
- * @stats_cnt_idx_msb: stats cnt index msb
- * @rsvd2: reserved bits
- * @retain_hdr: added to add back to the packet the header removed
- *  as part of header removal. This will be done as part of
- *  header insertion block.
- * @rule_id: rule ID that will be returned in the packet status
- * @stats_cnt_idx_lsb: stats cnt index lsb
- */
-struct ipa4_5_rt_rule_hw_hdr {
-	union {
-		u64 word;
-		struct {
-			u64 en_rule:16;
-			u64 pipe_dest_idx:5;
-			u64 system:1;
-			u64 hdr_offset:9;
-			u64 proc_ctx:1;
-			u64 priority:10;
-			u64 stats_cnt_idx_msb : 2;
-			u64 rsvd2 : 3;
-			u64 retain_hdr:1;
-			u64 rule_id:10;
-			u64 stats_cnt_idx_lsb : 6;
-		} hdr;
-	} u;
-};
-
-/**
- * struct ipa3_0_flt_rule_hw_hdr - HW header of IPA filter rule
- * @word: filtering rule properties
- * @en_rule: enable rule
- * @action: post filtering action
- * @rt_tbl_idx: index in routing table
- * @retain_hdr: added to add back to the packet the header removed
- *  as part of header removal. This will be done as part of
- *  header insertion block.
- * @rsvd1: reserved bits
- * @priority: Rule priority. Added to distinguish rules order
- *  at the integrated table consisting from hashable and
- *  non-hashable parts
- * @rsvd2: reserved bits
- * @rule_id: rule ID that will be returned in the packet status
- * @rsvd3: reserved bits
- */
-struct ipa3_0_flt_rule_hw_hdr {
-	union {
-		u64 word;
-		struct {
-			u64 en_rule:16;
-			u64 action:5;
-			u64 rt_tbl_idx:5;
-			u64 retain_hdr:1;
-			u64 rsvd1:5;
-			u64 priority:10;
-			u64 rsvd2:6;
-			u64 rule_id:10;
-			u64 rsvd3:6;
-		} hdr;
-	} u;
-};
-
-/**
- * struct ipa4_0_flt_rule_hw_hdr - HW header of IPA filter rule
- * @word: filtering rule properties
- * @en_rule: enable rule
- * @action: post filtering action
- * @rt_tbl_idx: index in routing table
- * @retain_hdr: added to add back to the packet the header removed
- *  as part of header removal. This will be done as part of
- *  header insertion block.
- * @pdn_idx: in case of go to src nat action possible to input the pdn index to
- *  the NAT block
- * @set_metadata: enable metadata replacement in the NAT block
- * @priority: Rule priority. Added to distinguish rules order
- *  at the integrated table consisting from hashable and
- *  non-hashable parts
- * @rsvd2: reserved bits
- * @rule_id: rule ID that will be returned in the packet status
- * @rsvd3: reserved bits
- */
-struct ipa4_0_flt_rule_hw_hdr {
-	union {
-		u64 word;
-		struct {
-			u64 en_rule : 16;
-			u64 action : 5;
-			u64 rt_tbl_idx : 5;
-			u64 retain_hdr : 1;
-			u64 pdn_idx : 4;
-			u64 set_metadata : 1;
-			u64 priority : 10;
-			u64 rsvd2 : 6;
-			u64 rule_id : 10;
-			u64 rsvd3 : 6;
-		} hdr;
-	} u;
-};
-
-/**
- * struct ipa4_5_flt_rule_hw_hdr - HW header of IPA filter rule
- * @word: filtering rule properties
- * @en_rule: enable rule
- * @action: post filtering action
- * @rt_tbl_idx: index in routing table
- * @retain_hdr: added to add back to the packet the header removed
- *  as part of header removal. This will be done as part of
- *  header insertion block.
- * @pdn_idx: in case of go to src nat action possible to input the pdn index to
- *  the NAT block
- * @set_metadata: enable metadata replacement in the NAT block
- * @priority: Rule priority. Added to distinguish rules order
- *  at the integrated table consisting from hashable and
- *  non-hashable parts
- * @stats_cnt_idx_msb: stats cnt index msb
- * @rsvd2: reserved bits
- * @rule_id: rule ID that will be returned in the packet status
- * @stats_cnt_idx_lsb: stats cnt index lsb
- */
-struct ipa4_5_flt_rule_hw_hdr {
-	union {
-		u64 word;
-		struct {
-			u64 en_rule : 16;
-			u64 action : 5;
-			u64 rt_tbl_idx : 5;
-			u64 retain_hdr : 1;
-			u64 pdn_idx : 4;
-			u64 set_metadata : 1;
-			u64 priority : 10;
-			u64 stats_cnt_idx_msb : 2;
-			u64 rsvd2 : 4;
-			u64 rule_id : 10;
-			u64 stats_cnt_idx_lsb : 6;
-		} hdr;
-	} u;
-};
-
-int ipahal_fltrt_init(enum ipa_hw_type ipa_hw_type);
-void ipahal_fltrt_destroy(void);
-
-#endif /* _IPAHAL_FLTRT_I_H_ */

ipa/ipa_v3/ipahal/ipahal_hw_stats.c

@@ -1,634 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2017-2019, The Linux Foundation. All rights reserved.
- */
-
-#include "ipahal.h"
-#include "ipahal_hw_stats.h"
-#include "ipahal_hw_stats_i.h"
-#include "ipahal_i.h"
-
-struct ipahal_hw_stats_obj {
-	struct ipahal_stats_init_pyld *(*generate_init_pyld)(void *params,
-		bool is_atomic_ctx);
-	int (*get_offset)(void *params, struct ipahal_stats_offset *out);
-	int (*parse_stats)(void *init_params, void *raw_stats,
-		void *parsed_stats);
-};
-
-static int _count_ones(u32 number)
-{
-	int count = 0;
-
-	while (number) {
-		count++;
-		number = number & (number - 1);
-	}
-
-	return count;
-}
-
-static struct ipahal_stats_init_pyld *ipahal_generate_init_pyld_quota(
-	void *params, bool is_atomic_ctx)
-{
-	struct ipahal_stats_init_pyld *pyld;
-	struct ipahal_stats_init_quota *in =
-		(struct ipahal_stats_init_quota *)params;
-	int entries = _count_ones(in->enabled_bitmask);
-
-	IPAHAL_DBG_LOW("entries = %d\n", entries);
-	pyld = IPAHAL_MEM_ALLOC(sizeof(*pyld) +
-		entries * sizeof(struct ipahal_stats_quota_hw), is_atomic_ctx);
-	if (!pyld) {
-		IPAHAL_ERR("no mem\n");
-		return NULL;
-	}
-
-	pyld->len = entries * sizeof(struct ipahal_stats_quota_hw);
-	return pyld;
-}
-
-static int ipahal_get_offset_quota(void *params,
-	struct ipahal_stats_offset *out)
-{
-	struct ipahal_stats_get_offset_quota *in =
-		(struct ipahal_stats_get_offset_quota *)params;
-	int entries = _count_ones(in->init.enabled_bitmask);
-
-	IPAHAL_DBG_LOW("\n");
-	out->offset = 0;
-	out->size = entries * sizeof(struct ipahal_stats_quota_hw);
-
-	return 0;
-}
-
-static int ipahal_parse_stats_quota(void *init_params, void *raw_stats,
-	void *parsed_stats)
-{
-	struct ipahal_stats_init_quota *init =
-		(struct ipahal_stats_init_quota *)init_params;
-	struct ipahal_stats_quota_hw *raw_hw =
-		(struct ipahal_stats_quota_hw *)raw_stats;
-	struct ipahal_stats_quota_all *out =
-		(struct ipahal_stats_quota_all *)parsed_stats;
-	int stat_idx = 0;
-	int i;
-
-	memset(out, 0, sizeof(*out));
-	IPAHAL_DBG_LOW("\n");
-	for (i = 0; i < IPAHAL_MAX_PIPES; i++) {
-		if (init->enabled_bitmask & (1 << i)) {
-			IPAHAL_DBG_LOW("pipe %d stat_idx %d\n", i, stat_idx);
-			out->stats[i].num_ipv4_bytes =
-				raw_hw[stat_idx].num_ipv4_bytes;
-			out->stats[i].num_ipv4_pkts =
-				raw_hw[stat_idx].num_ipv4_pkts;
-			out->stats[i].num_ipv6_pkts =
-				raw_hw[stat_idx].num_ipv6_pkts;
-			out->stats[i].num_ipv6_bytes =
-				raw_hw[stat_idx].num_ipv6_bytes;
-			stat_idx++;
-		}
-	}
-
-	return 0;
-}
-
-static struct ipahal_stats_init_pyld *ipahal_generate_init_pyld_tethering(
-	void *params, bool is_atomic_ctx)
-{
-	struct ipahal_stats_init_pyld *pyld;
-	struct ipahal_stats_init_tethering *in =
-		(struct ipahal_stats_init_tethering *)params;
-	int hdr_entries = _count_ones(in->prod_bitmask);
-	int entries = 0;
-	int i;
-	void *pyld_ptr;
-	u32 incremental_offset;
-
-	IPAHAL_DBG_LOW("prod entries = %d\n", hdr_entries);
-	for (i = 0; i < sizeof(in->prod_bitmask) * 8; i++) {
-		if (in->prod_bitmask & (1 << i)) {
-			if (in->cons_bitmask[i] == 0) {
-				IPAHAL_ERR("no cons bitmask for prod %d\n", i);
-				return NULL;
-			}
-			entries += _count_ones(in->cons_bitmask[i]);
-		}
-	}
-	IPAHAL_DBG_LOW("sum all entries = %d\n", entries);
-
-	pyld = IPAHAL_MEM_ALLOC(sizeof(*pyld) +
-		hdr_entries * sizeof(struct ipahal_stats_tethering_hdr_hw) +
-		entries * sizeof(struct ipahal_stats_tethering_hw),
-		is_atomic_ctx);
-	if (!pyld)
-		return NULL;
-
-	pyld->len = hdr_entries * sizeof(struct ipahal_stats_tethering_hdr_hw) +
-		entries * sizeof(struct ipahal_stats_tethering_hw);
-
-	pyld_ptr = pyld->data;
-	incremental_offset =
-		(hdr_entries * sizeof(struct ipahal_stats_tethering_hdr_hw))
-			/ 8;
-	for (i = 0; i < sizeof(in->prod_bitmask) * 8; i++) {
-		if (in->prod_bitmask & (1 << i)) {
-			struct ipahal_stats_tethering_hdr_hw *hdr = pyld_ptr;
-
-			hdr->dst_mask = in->cons_bitmask[i];
-			hdr->offset = incremental_offset;
-			IPAHAL_DBG_LOW("hdr->dst_mask=0x%x\n", hdr->dst_mask);
-			IPAHAL_DBG_LOW("hdr->offset=0x%x\n", hdr->offset);
-			/* add the stats entry */
-			incremental_offset += _count_ones(in->cons_bitmask[i]) *
-				sizeof(struct ipahal_stats_tethering_hw) / 8;
-			pyld_ptr += sizeof(*hdr);
-		}
-	}
-
-	return pyld;
-}
-
-static int ipahal_get_offset_tethering(void *params,
-	struct ipahal_stats_offset *out)
-{
-	struct ipahal_stats_get_offset_tethering *in =
-		(struct ipahal_stats_get_offset_tethering *)params;
-	int entries = 0;
-	int i;
-
-	for (i = 0; i < sizeof(in->init.prod_bitmask) * 8; i++) {
-		if (in->init.prod_bitmask & (1 << i)) {
-			if (in->init.cons_bitmask[i] == 0) {
-				IPAHAL_ERR("no cons bitmask for prod %d\n", i);
-				return -EPERM;
-			}
-			entries += _count_ones(in->init.cons_bitmask[i]);
-		}
-	}
-	IPAHAL_DBG_LOW("sum all entries = %d\n", entries);
-
-	/* skip the header */
-	out->offset = _count_ones(in->init.prod_bitmask) *
-		sizeof(struct ipahal_stats_tethering_hdr_hw);
-	out->size = entries * sizeof(struct ipahal_stats_tethering_hw);
-
-	return 0;
-}
-
-static int ipahal_parse_stats_tethering(void *init_params, void *raw_stats,
-	void *parsed_stats)
-{
-	struct ipahal_stats_init_tethering *init =
-		(struct ipahal_stats_init_tethering *)init_params;
-	struct ipahal_stats_tethering_hw *raw_hw =
-		(struct ipahal_stats_tethering_hw *)raw_stats;
-	struct ipahal_stats_tethering_all *out =
-		(struct ipahal_stats_tethering_all *)parsed_stats;
-	int i, j;
-	int stat_idx = 0;
-
-	memset(out, 0, sizeof(*out));
-	IPAHAL_DBG_LOW("\n");
-	for (i = 0; i < IPAHAL_MAX_PIPES; i++) {
-		for (j = 0; j < IPAHAL_MAX_PIPES; j++) {
-			if ((init->prod_bitmask & (1 << i)) &&
-			    init->cons_bitmask[i] & (1 << j)) {
-				IPAHAL_DBG_LOW("prod %d cons %d\n", i, j);
-				IPAHAL_DBG_LOW("stat_idx %d\n", stat_idx);
-				out->stats[i][j].num_ipv4_bytes =
-					raw_hw[stat_idx].num_ipv4_bytes;
-				IPAHAL_DBG_LOW("num_ipv4_bytes %lld\n",
-					out->stats[i][j].num_ipv4_bytes);
-				out->stats[i][j].num_ipv4_pkts =
-					raw_hw[stat_idx].num_ipv4_pkts;
-				IPAHAL_DBG_LOW("num_ipv4_pkts %lld\n",
-					out->stats[i][j].num_ipv4_pkts);
-				out->stats[i][j].num_ipv6_pkts =
-					raw_hw[stat_idx].num_ipv6_pkts;
-				IPAHAL_DBG_LOW("num_ipv6_pkts %lld\n",
-					out->stats[i][j].num_ipv6_pkts);
-				out->stats[i][j].num_ipv6_bytes =
-					raw_hw[stat_idx].num_ipv6_bytes;
-				IPAHAL_DBG_LOW("num_ipv6_bytes %lld\n",
-					out->stats[i][j].num_ipv6_bytes);
-				stat_idx++;
-			}
-		}
-	}
-
-	return 0;
-}
-
-static struct ipahal_stats_init_pyld *ipahal_generate_init_pyld_flt_rt_v4_5(
-	void *params, bool is_atomic_ctx)
-{
-	struct ipahal_stats_init_pyld *pyld;
-	int num = (int)(params);
-
-	if (num > IPA_MAX_FLT_RT_CNT_INDEX ||
-		num <= 0) {
-		IPAHAL_ERR("num %d not valid\n", num);
-		return NULL;
-	}
-	pyld = IPAHAL_MEM_ALLOC(sizeof(*pyld) +
-		num *
-		sizeof(struct ipahal_stats_flt_rt_v4_5_hw),
-		is_atomic_ctx);
-	if (!pyld)
-		return NULL;
-	pyld->len = num *
-		sizeof(struct ipahal_stats_flt_rt_v4_5_hw);
-	return pyld;
-}
-
-static int ipahal_get_offset_flt_rt_v4_5(void *params,
-	struct ipahal_stats_offset *out)
-{
-	struct ipahal_stats_get_offset_flt_rt_v4_5 *in =
-		(struct ipahal_stats_get_offset_flt_rt_v4_5 *)params;
-	int num;
-
-	out->offset = (in->start_id - 1) *
-		sizeof(struct ipahal_stats_flt_rt_v4_5);
-	num = in->end_id - in->start_id + 1;
-	out->size = num * sizeof(struct ipahal_stats_flt_rt_v4_5);
-
-	return 0;
-}
-
-static int ipahal_parse_stats_flt_rt_v4_5(void *init_params,
-	void *raw_stats, void *parsed_stats)
-{
-	struct ipahal_stats_flt_rt_v4_5_hw *raw_hw =
-		(struct ipahal_stats_flt_rt_v4_5_hw *)raw_stats;
-	struct ipa_ioc_flt_rt_query *query =
-		(struct ipa_ioc_flt_rt_query *)parsed_stats;
-	int num, i;
-
-	num = query->end_id - query->start_id + 1;
-	IPAHAL_DBG_LOW("\n");
-	for (i = 0; i < num; i++) {
-		((struct ipa_flt_rt_stats *)
-		query->stats)[i].num_bytes =
-			raw_hw[i].num_bytes;
-		((struct ipa_flt_rt_stats *)
-		query->stats)[i].num_pkts_hash =
-			raw_hw[i].num_packets_hash;
-		((struct ipa_flt_rt_stats *)
-		query->stats)[i].num_pkts =
-			raw_hw[i].num_packets;
-	}
-
-	return 0;
-}
-
-
-static struct ipahal_stats_init_pyld *ipahal_generate_init_pyld_flt_rt(
-	void *params, bool is_atomic_ctx)
-{
-	struct ipahal_stats_init_pyld *pyld;
-	struct ipahal_stats_init_flt_rt *in =
-		(struct ipahal_stats_init_flt_rt *)params;
-	int hdr_entries;
-	int num_rules = 0;
-	int i, start_entry;
-	void *pyld_ptr;
-	u32 incremental_offset;
-
-	for (i = 0; i < IPAHAL_MAX_RULE_ID_32; i++)
-		num_rules += _count_ones(in->rule_id_bitmask[i]);
-
-	if (num_rules == 0) {
-		IPAHAL_ERR("no rule ids provided\n");
-		return NULL;
-	}
-	IPAHAL_DBG_LOW("num_rules = %d\n", num_rules);
-
-	hdr_entries = IPAHAL_MAX_RULE_ID_32;
-	for (i = 0; i < IPAHAL_MAX_RULE_ID_32; i++) {
-		if (in->rule_id_bitmask[i] != 0)
-			break;
-		hdr_entries--;
-	}
-	start_entry = i;
-
-	for (i = IPAHAL_MAX_RULE_ID_32 - 1; i >= start_entry; i--) {
-		if (in->rule_id_bitmask[i] != 0)
-			break;
-		hdr_entries--;
-	}
-	IPAHAL_DBG_LOW("hdr_entries = %d\n", hdr_entries);
-
-	pyld = IPAHAL_MEM_ALLOC(sizeof(*pyld) +
-		hdr_entries * sizeof(struct ipahal_stats_flt_rt_hdr_hw) +
-		num_rules * sizeof(struct ipahal_stats_flt_rt_hw),
-		is_atomic_ctx);
-	if (!pyld) {
-		IPAHAL_ERR("no mem\n");
-		return NULL;
-	}
-
-	pyld->len = hdr_entries * sizeof(struct ipahal_stats_flt_rt_hdr_hw) +
-		num_rules * sizeof(struct ipahal_stats_flt_rt_hw);
-
-	pyld_ptr = pyld->data;
-	incremental_offset =
-		(hdr_entries * sizeof(struct ipahal_stats_flt_rt_hdr_hw))
-			/ 8;
-	for (i = start_entry; i < hdr_entries; i++) {
-		struct ipahal_stats_flt_rt_hdr_hw *hdr = pyld_ptr;
-
-		hdr->en_mask = in->rule_id_bitmask[i];
-		hdr->cnt_offset = incremental_offset;
-		/* add the stats entry */
-		incremental_offset += _count_ones(in->rule_id_bitmask[i]) *
-			sizeof(struct ipahal_stats_flt_rt_hw) / 8;
-		pyld_ptr += sizeof(*hdr);
-	}
-
-	return pyld;
-}
-
-static int ipahal_get_offset_flt_rt(void *params,
-	struct ipahal_stats_offset *out)
-{
-	struct ipahal_stats_get_offset_flt_rt *in =
-		(struct ipahal_stats_get_offset_flt_rt *)params;
-	int i;
-	int hdr_entries;
-	int skip_rules = 0;
-	int start_entry;
-	int rule_bit = in->rule_id % 32;
-	int rule_idx = in->rule_id / 32;
-
-	if (rule_idx >= IPAHAL_MAX_RULE_ID_32) {
-		IPAHAL_ERR("invalid rule_id %d\n", in->rule_id);
-		return -EPERM;
-	}
-
-	hdr_entries = IPAHAL_MAX_RULE_ID_32;
-	for (i = 0; i < IPAHAL_MAX_RULE_ID_32; i++) {
-		if (in->init.rule_id_bitmask[i] != 0)
-			break;
-		hdr_entries--;
-	}
-
-	if (hdr_entries == 0) {
-		IPAHAL_ERR("no rule ids provided\n");
-		return -EPERM;
-	}
-	start_entry = i;
-
-	for (i = IPAHAL_MAX_RULE_ID_32 - 1; i >= 0; i--) {
-		if (in->init.rule_id_bitmask[i] != 0)
-			break;
-		hdr_entries--;
-	}
-	IPAHAL_DBG_LOW("hdr_entries = %d\n", hdr_entries);
-
-	/* skip the header */
-	out->offset = hdr_entries * sizeof(struct ipahal_stats_flt_rt_hdr_hw);
-
-	/* skip the previous rules  */
-	for (i = start_entry; i < rule_idx; i++)
-		skip_rules += _count_ones(in->init.rule_id_bitmask[i]);
-
-	for (i = 0; i < rule_bit; i++)
-		if (in->init.rule_id_bitmask[rule_idx] & (1 << i))
-			skip_rules++;
-
-	out->offset += skip_rules * sizeof(struct ipahal_stats_flt_rt_hw);
-	out->size = sizeof(struct ipahal_stats_flt_rt_hw);
-
-	return 0;
-}
-
-static int ipahal_parse_stats_flt_rt(void *init_params, void *raw_stats,
-	void *parsed_stats)
-{
-	struct ipahal_stats_flt_rt_hw *raw_hw =
-		(struct ipahal_stats_flt_rt_hw *)raw_stats;
-	struct ipahal_stats_flt_rt *out =
-		(struct ipahal_stats_flt_rt *)parsed_stats;
-
-	memset(out, 0, sizeof(*out));
-	IPAHAL_DBG_LOW("\n");
-	out->num_packets = raw_hw->num_packets;
-	out->num_packets_hash = raw_hw->num_packets_hash;
-
-	return 0;
-}
-
-static struct ipahal_stats_init_pyld *ipahal_generate_init_pyld_drop(
-	void *params, bool is_atomic_ctx)
-{
-	struct ipahal_stats_init_pyld *pyld;
-	struct ipahal_stats_init_drop *in =
-		(struct ipahal_stats_init_drop *)params;
-	int entries = _count_ones(in->enabled_bitmask);
-
-	IPAHAL_DBG_LOW("entries = %d\n", entries);
-	pyld = IPAHAL_MEM_ALLOC(sizeof(*pyld) +
-		entries * sizeof(struct ipahal_stats_drop_hw), is_atomic_ctx);
-	if (!pyld)
-		return NULL;
-
-	pyld->len = entries * sizeof(struct ipahal_stats_drop_hw);
-
-	return pyld;
-}
-
-static int ipahal_get_offset_drop(void *params,
-	struct ipahal_stats_offset *out)
-{
-	struct ipahal_stats_get_offset_drop *in =
-		(struct ipahal_stats_get_offset_drop *)params;
-	int entries = _count_ones(in->init.enabled_bitmask);
-
-	IPAHAL_DBG_LOW("\n");
-	out->offset = 0;
-	out->size = entries * sizeof(struct ipahal_stats_drop_hw);
-
-	return 0;
-}
-
-static int ipahal_parse_stats_drop(void *init_params, void *raw_stats,
-	void *parsed_stats)
-{
-	struct ipahal_stats_init_drop *init =
-		(struct ipahal_stats_init_drop *)init_params;
-	struct ipahal_stats_drop_hw *raw_hw =
-		(struct ipahal_stats_drop_hw *)raw_stats;
-	struct ipahal_stats_drop_all *out =
-		(struct ipahal_stats_drop_all *)parsed_stats;
-	int stat_idx = 0;
-	int i;
-
-	memset(out, 0, sizeof(*out));
-	IPAHAL_DBG_LOW("\n");
-	for (i = 0; i < IPAHAL_MAX_PIPES; i++) {
-		if (init->enabled_bitmask & (1 << i)) {
-			out->stats[i].drop_byte_cnt =
-				raw_hw[stat_idx].drop_byte_cnt;
-			out->stats[i].drop_packet_cnt =
-				raw_hw[stat_idx].drop_packet_cnt;
-			stat_idx++;
-		}
-	}
-
-	return 0;
-}
-
-static struct ipahal_hw_stats_obj
-	ipahal_hw_stats_objs[IPA_HW_MAX][IPAHAL_HW_STATS_MAX] = {
-	/* IPAv4 */
-	[IPA_HW_v4_0][IPAHAL_HW_STATS_QUOTA] = {
-		ipahal_generate_init_pyld_quota,
-		ipahal_get_offset_quota,
-		ipahal_parse_stats_quota
-	},
-	[IPA_HW_v4_0][IPAHAL_HW_STATS_TETHERING] = {
-		ipahal_generate_init_pyld_tethering,
-		ipahal_get_offset_tethering,
-		ipahal_parse_stats_tethering
-	},
-	[IPA_HW_v4_0][IPAHAL_HW_STATS_FNR] = {
-		ipahal_generate_init_pyld_flt_rt,
-		ipahal_get_offset_flt_rt,
-		ipahal_parse_stats_flt_rt
-	},
-	[IPA_HW_v4_0][IPAHAL_HW_STATS_DROP] = {
-		ipahal_generate_init_pyld_drop,
-		ipahal_get_offset_drop,
-		ipahal_parse_stats_drop
-	},
-	[IPA_HW_v4_5][IPAHAL_HW_STATS_QUOTA] = {
-		ipahal_generate_init_pyld_quota,
-		ipahal_get_offset_quota,
-		ipahal_parse_stats_quota
-	},
-	[IPA_HW_v4_5][IPAHAL_HW_STATS_FNR] = {
-		ipahal_generate_init_pyld_flt_rt_v4_5,
-		ipahal_get_offset_flt_rt_v4_5,
-		ipahal_parse_stats_flt_rt_v4_5
-	},
-	[IPA_HW_v4_5][IPAHAL_HW_STATS_TETHERING] = {
-		ipahal_generate_init_pyld_tethering,
-		ipahal_get_offset_tethering,
-		ipahal_parse_stats_tethering
-	},
-	[IPA_HW_v4_5][IPAHAL_HW_STATS_DROP] = {
-		ipahal_generate_init_pyld_drop,
-		ipahal_get_offset_drop,
-		ipahal_parse_stats_drop
-	},
-};
-
-int ipahal_hw_stats_init(enum ipa_hw_type ipa_hw_type)
-{
-	int i;
-	int j;
-	struct ipahal_hw_stats_obj zero_obj;
-	struct ipahal_hw_stats_obj *hw_stat_ptr;
-
-	IPAHAL_DBG_LOW("Entry - HW_TYPE=%d\n", ipa_hw_type);
-
-	if ((ipa_hw_type < 0) || (ipa_hw_type >= IPA_HW_MAX)) {
-		IPAHAL_ERR("invalid IPA HW type (%d)\n", ipa_hw_type);
-		return -EINVAL;
-	}
-
-	memset(&zero_obj, 0, sizeof(zero_obj));
-	for (i = IPA_HW_v4_0 ; i < ipa_hw_type ; i++) {
-		for (j = 0; j < IPAHAL_HW_STATS_MAX; j++) {
-			if (!memcmp(&ipahal_hw_stats_objs[i + 1][j], &zero_obj,
-				sizeof(struct ipahal_hw_stats_obj))) {
-				memcpy(&ipahal_hw_stats_objs[i + 1][j],
-					&ipahal_hw_stats_objs[i][j],
-					sizeof(struct ipahal_hw_stats_obj));
-			} else {
-				/*
-				 * explicitly overridden stat.
-				 * Check validity
-				 */
-				hw_stat_ptr = &ipahal_hw_stats_objs[i + 1][j];
-				if (!hw_stat_ptr->get_offset) {
-					IPAHAL_ERR(
-					  "stat=%d get_offset null ver=%d\n",
-					  j, i+1);
-					WARN_ON(1);
-				}
-				if (!hw_stat_ptr->parse_stats) {
-					IPAHAL_ERR(
-					  "stat=%d parse_stats null ver=%d\n",
-						j, i + 1);
-					WARN_ON(1);
-				}
-			}
-		}
-	}
-
-	return 0;
-}
-
-int ipahal_stats_get_offset(enum ipahal_hw_stats_type type, void *params,
-	struct ipahal_stats_offset *out)
-{
-	if (type < 0 || type >= IPAHAL_HW_STATS_MAX) {
-		IPAHAL_ERR("Invalid type stat=%d\n", type);
-		WARN_ON(1);
-		return -EFAULT;
-	}
-
-	if (!params || !out) {
-		IPAHAL_ERR("Null arg\n");
-		WARN_ON(1);
-		return -EFAULT;
-	}
-
-	return ipahal_hw_stats_objs[ipahal_ctx->hw_type][type].get_offset(
-		params, out);
-}
-
-struct ipahal_stats_init_pyld *ipahal_stats_generate_init_pyld(
-	enum ipahal_hw_stats_type type, void *params, bool is_atomic_ctx)
-{
-	struct ipahal_hw_stats_obj *hw_obj_ptr;
-
-	if (type < 0 || type >= IPAHAL_HW_STATS_MAX) {
-		IPAHAL_ERR("Invalid type stat=%d\n", type);
-		WARN_ON(1);
-		return NULL;
-	}
-
-	hw_obj_ptr = &ipahal_hw_stats_objs[ipahal_ctx->hw_type][type];
-	return hw_obj_ptr->generate_init_pyld(params, is_atomic_ctx);
-}
-
-int ipahal_parse_stats(enum ipahal_hw_stats_type type, void *init_params,
-	void *raw_stats, void *parsed_stats)
-{
-	if (WARN((type < 0 || type >= IPAHAL_HW_STATS_MAX),
-		"Invalid type stat = %d\n", type))
-		return -EFAULT;
-
-	if (WARN((!raw_stats || !parsed_stats), "Null arg\n"))
-		return -EFAULT;
-
-	return ipahal_hw_stats_objs[ipahal_ctx->hw_type][type].parse_stats(
-		init_params, raw_stats, parsed_stats);
-}
-
-void ipahal_set_flt_rt_sw_stats(void *raw_stats,
-	struct ipa_flt_rt_stats sw_stats)
-{
-	struct ipahal_stats_flt_rt_v4_5_hw *raw_hw =
-		(struct ipahal_stats_flt_rt_v4_5_hw *)raw_stats;
-
-	IPAHAL_DBG_LOW("\n");
-	raw_hw->num_bytes = sw_stats.num_bytes;
-	raw_hw->num_packets_hash = sw_stats.num_pkts_hash;
-	raw_hw->num_packets = sw_stats.num_pkts;
-}

ipa/ipa_v3/ipahal/ipahal_hw_stats.h

@@ -1,273 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2017-2019, The Linux Foundation. All rights reserved.
- */
-
-#ifndef _IPAHAL_HW_STATS_H_
-#define _IPAHAL_HW_STATS_H_
-
-#include <linux/ipa.h>
-
-#define IPAHAL_MAX_PIPES 32
-#define IPAHAL_MAX_RULE_ID_32 (1024 / 32) /* 10 bits of rule id */
-
-enum ipahal_hw_stats_type {
-	IPAHAL_HW_STATS_QUOTA,
-	IPAHAL_HW_STATS_TETHERING,
-	IPAHAL_HW_STATS_FNR,
-	IPAHAL_HW_STATS_DROP,
-	IPAHAL_HW_STATS_MAX
-};
-
-/*
- * struct ipahal_stats_init_pyld - Statistics initialization payload
- * @len: length of payload
- * @data: actual payload data
- */
-struct ipahal_stats_init_pyld {
-	u16 len;
-	u16 reserved;
-	u8 data[0];
-};
-
-/*
- * struct ipahal_stats_offset - Statistics offset parameters
- * @offset: offset of the statistic from beginning of stats table
- * @size: size of the statistics
- */
-struct ipahal_stats_offset {
-	u32 offset;
-	u16 size;
-};
-
-/*
- * struct ipahal_stats_init_quota - Initializations parameters for quota
- * @enabled_bitmask: bit mask of pipes to be monitored
- */
-struct ipahal_stats_init_quota {
-	u32 enabled_bitmask;
-};
-
-/*
- * struct ipahal_stats_get_offset_quota - Get offset parameters for quota
- * @init: initialization parameters used in initialization of stats
- */
-struct ipahal_stats_get_offset_quota {
-	struct ipahal_stats_init_quota init;
-};
-
-/*
- * struct ipahal_stats_quota - Quota statistics
- * @num_ipv4_bytes: IPv4 bytes
- * @num_ipv6_bytes: IPv6 bytes
- * @num_ipv4_pkts: IPv4 packets
- * @num_ipv6_pkts: IPv6 packets
- */
-struct ipahal_stats_quota {
-	u64 num_ipv4_bytes;
-	u64 num_ipv6_bytes;
-	u64 num_ipv4_pkts;
-	u64 num_ipv6_pkts;
-};
-
-/*
- * struct ipahal_stats_quota_all - Quota statistics for all pipes
- * @stats: array of statistics per pipe
- */
-struct ipahal_stats_quota_all {
-	struct ipahal_stats_quota stats[IPAHAL_MAX_PIPES];
-};
-
-/*
- * struct ipahal_stats_init_tethering - Initializations parameters for tethering
- * @prod_bitmask: bit mask of producer pipes to be monitored
- * @cons_bitmask: bit mask of consumer pipes to be monitored per producer
- */
-struct ipahal_stats_init_tethering {
-	u32 prod_bitmask;
-	u32 cons_bitmask[IPAHAL_MAX_PIPES];
-};
-
-/*
- * struct ipahal_stats_get_offset_tethering - Get offset parameters for
- *	tethering
- * @init: initialization parameters used in initialization of stats
- */
-struct ipahal_stats_get_offset_tethering {
-	struct ipahal_stats_init_tethering init;
-};
-
-/*
- * struct ipahal_stats_tethering - Tethering statistics
- * @num_ipv4_bytes: IPv4 bytes
- * @num_ipv6_bytes: IPv6 bytes
- * @num_ipv4_pkts: IPv4 packets
- * @num_ipv6_pkts: IPv6 packets
- */
-struct ipahal_stats_tethering {
-	u64 num_ipv4_bytes;
-	u64 num_ipv6_bytes;
-	u64 num_ipv4_pkts;
-	u64 num_ipv6_pkts;
-};
-
-/*
- * struct ipahal_stats_tethering_all - Tethering statistics for all pipes
- * @stats: matrix of statistics per pair of pipes
- */
-struct ipahal_stats_tethering_all {
-	struct ipahal_stats_tethering
-		stats[IPAHAL_MAX_PIPES][IPAHAL_MAX_PIPES];
-};
-
-/*
- * struct ipahal_stats_init_flt_rt - Initializations parameters for flt_rt
- * @rule_id_bitmask: array describes which rule ids to monitor.
- *	rule_id bit is determined by:
- *		index to the array => rule_id / 32
- *		bit to enable => rule_id % 32
- */
-struct ipahal_stats_init_flt_rt {
-	u32 rule_id_bitmask[IPAHAL_MAX_RULE_ID_32];
-};
-
-/*
- * struct ipahal_stats_get_offset_flt_rt - Get offset parameters for flt_rt
- * @init: initialization parameters used in initialization of stats
- * @rule_id: rule_id to get the offset for
- */
-struct ipahal_stats_get_offset_flt_rt {
-	struct ipahal_stats_init_flt_rt init;
-	u32 rule_id;
-};
-
-/*
- * struct ipahal_stats_flt_rt - flt_rt statistics
- * @num_packets: Total number of packets hit this rule
- * @num_packets_hash: Total number of packets hit this rule in hash table
- */
-struct ipahal_stats_flt_rt {
-	u32 num_packets;
-	u32 num_packets_hash;
-};
-
-/*
- * struct ipahal_stats_flt_rt_v4_5 - flt_rt statistics
- * @num_packets: Total number of packets hit this rule
- * @num_packets_hash: Total number of packets hit this rule in hash table
- * @num_bytes: Total number of bytes hit this rule
- */
-struct ipahal_stats_flt_rt_v4_5 {
-	u32 num_packets;
-	u32 num_packets_hash;
-	u64 num_bytes;
-};
-
-/*
- * struct ipahal_stats_get_offset_flt_rt_v4_5 - Get offset parameters for flt_rt
- * @start_id: start_id to get the offset
- * @end_id: end_id to get the offset
- */
-struct ipahal_stats_get_offset_flt_rt_v4_5 {
-	u8 start_id;
-	u8 end_id;
-};
-
-/*
- * struct ipahal_stats_init_drop - Initializations parameters for Drop
- * @enabled_bitmask: bit mask of pipes to be monitored
- */
-struct ipahal_stats_init_drop {
-	u32 enabled_bitmask;
-};
-
-/*
- * struct ipahal_stats_get_offset_drop - Get offset parameters for Drop
- * @init: initialization parameters used in initialization of stats
- */
-struct ipahal_stats_get_offset_drop {
-	struct ipahal_stats_init_drop init;
-};
-
-/*
- * struct ipahal_stats_drop - Packet Drop statistics
- * @drop_packet_cnt: number of packets dropped
- * @drop_byte_cnt: number of bytes dropped
- */
-struct ipahal_stats_drop {
-	u32 drop_packet_cnt;
-	u32 drop_byte_cnt;
-};
-
-/*
- * struct ipahal_stats_drop_all - Drop statistics for all pipes
- * @stats: array of statistics per pipes
- */
-struct ipahal_stats_drop_all {
-	struct ipahal_stats_drop stats[IPAHAL_MAX_PIPES];
-};
-
-/*
- * ipahal_stats_generate_init_pyld - Generate the init payload for stats
- * @type: type of stats
- * @params: init_pyld parameters based of stats type
- * @is_atomic_ctx: is calling context atomic ?
- *
- * This function will generate the initialization payload for a particular
- * statistic in hardware. IPA driver is expected to use this payload to
- * initialize the SRAM.
- *
- * Return: pointer to ipahal_stats_init_pyld on success or NULL on failure.
- */
-struct ipahal_stats_init_pyld *ipahal_stats_generate_init_pyld(
-	enum ipahal_hw_stats_type type, void *params, bool is_atomic_ctx);
-
-/*
- * ipahal_destroy_stats_init_pyld() - Destroy/Release bulk that was built
- *  by the ipahal_stats_generate_init_pyld function.
- */
-static inline void ipahal_destroy_stats_init_pyld(
-	struct ipahal_stats_init_pyld *pyld)
-{
-	kfree(pyld);
-}
-
-/*
- * ipahal_stats_get_offset - Get the offset / size of payload for stats
- * @type: type of stats
- * @params: get_offset parameters based of stats type
- * @out: out parameter for the offset and size.
- *
- * This function will return the offset of the counter from beginning of
- * the table.IPA driver is expected to read this portion in SRAM and pass
- * it to ipahal_parse_stats() to interprete the stats.
- *
- * Return: 0 on success and negative on failure
- */
-int ipahal_stats_get_offset(enum ipahal_hw_stats_type type, void *params,
-	struct ipahal_stats_offset *out);
-
-/*
- * ipahal_parse_stats - parse statistics
- * @type: type of stats
- * @init_params: init_pyld parameters used on init
- * @raw_stats: stats read from IPA SRAM
- * @parsed_stats: pointer to parsed stats based on type
- *
- * Return: 0 on success and negative on failure
- */
-int ipahal_parse_stats(enum ipahal_hw_stats_type type, void *init_params,
-	void *raw_stats, void *parsed_stats);
-
-
-/*
- * ipahal_set_flt_rt_sw_stats - set sw counter stats for FnR
- * @raw_stats: stats write to IPA SRAM
- * @sw_stats: FnR sw stats to be written
- *
- * Return: None
- */
-void ipahal_set_flt_rt_sw_stats(void *raw_stats,
-	struct ipa_flt_rt_stats sw_stats);
-
-#endif /* _IPAHAL_HW_STATS_H_ */

ipa/ipa_v3/ipahal/ipahal_hw_stats_i.h

@@ -1,54 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2017-2019, The Linux Foundation. All rights reserved.
- */
-
-#ifndef _IPAHAL_HW_STATS_I_H_
-#define _IPAHAL_HW_STATS_I_H_
-
-#include "ipahal_hw_stats.h"
-
-int ipahal_hw_stats_init(enum ipa_hw_type ipa_hw_type);
-
-struct ipahal_stats_quota_hw {
-	u64 num_ipv4_bytes;
-	u64 num_ipv4_pkts:32;
-	u64 num_ipv6_pkts:32;
-	u64 num_ipv6_bytes;
-};
-
-struct ipahal_stats_tethering_hdr_hw {
-	u64 dst_mask:32;
-	u64 offset:32;
-};
-
-struct ipahal_stats_tethering_hw {
-	u64 num_ipv4_bytes;
-	u64 num_ipv4_pkts:32;
-	u64 num_ipv6_pkts:32;
-	u64 num_ipv6_bytes;
-};
-
-struct ipahal_stats_flt_rt_hdr_hw {
-	u64 en_mask:32;
-	u64 reserved:16;
-	u64 cnt_offset:16;
-};
-
-struct ipahal_stats_flt_rt_hw {
-	u64 num_packets_hash:32;
-	u64 num_packets:32;
-};
-
-struct ipahal_stats_flt_rt_v4_5_hw {
-	u64 num_packets_hash:32;
-	u64 num_packets:32;
-	u64 num_bytes;
-};
-
-struct ipahal_stats_drop_hw {
-	u64 drop_byte_cnt:40;
-	u64 drop_packet_cnt:24;
-};
-
-#endif /* _IPAHAL_HW_STATS_I_H_ */

ipa/ipa_v3/ipahal/ipahal_i.h

@@ -1,727 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2016-2019, The Linux Foundation. All rights reserved.
- */
-
-#ifndef _IPAHAL_I_H_
-#define _IPAHAL_I_H_
-
-#include <linux/ipa.h>
-#include "../../ipa_common_i.h"
-
-#define IPAHAL_DRV_NAME "ipahal"
-
-#define IPAHAL_DBG(fmt, args...) \
-	do { \
-		pr_debug(IPAHAL_DRV_NAME " %s:%d " fmt, __func__, __LINE__, \
-			## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-			IPAHAL_DRV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			IPAHAL_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-#define IPAHAL_DBG_LOW(fmt, args...) \
-	do { \
-		pr_debug(IPAHAL_DRV_NAME " %s:%d " fmt, __func__, __LINE__, \
-			## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			IPAHAL_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-#define IPAHAL_ERR(fmt, args...) \
-	do { \
-		pr_err(IPAHAL_DRV_NAME " %s:%d " fmt, __func__, __LINE__, \
-			## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-			IPAHAL_DRV_NAME " %s:%d " fmt, ## args); \
-		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-			IPAHAL_DRV_NAME " %s:%d " fmt, ## args); \
-	} while (0)
-
-#define IPAHAL_ERR_RL(fmt, args...) \
-		do { \
-			pr_err_ratelimited_ipa(IPAHAL_DRV_NAME " %s:%d " fmt, \
-			__func__, __LINE__, ## args); \
-			IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
-				IPAHAL_DRV_NAME " %s:%d " fmt, ## args); \
-			IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
-				IPAHAL_DRV_NAME " %s:%d " fmt, ## args); \
-		} while (0)
-
-#define IPAHAL_DBG_REG(fmt, args...) \
-	do { \
-		pr_err(fmt, ## args); \
-		IPA_IPC_LOGGING(ipahal_ctx->regdumpbuf, \
-			" %s:%d " fmt, ## args); \
-	} while (0)
-
-#define IPAHAL_DBG_REG_IPC_ONLY(fmt, args...) \
-		IPA_IPC_LOGGING(ipahal_ctx->regdumpbuf, " %s:%d " fmt, ## args)
-
-#define IPAHAL_MEM_ALLOC(__size, __is_atomic_ctx) \
-	(kzalloc((__size), ((__is_atomic_ctx) ? GFP_ATOMIC : GFP_KERNEL)))
-
-#define IPAHAL_IPC_LOG_PAGES 50
-
-#define IPAHAL_PKT_STATUS_FLTRT_RULE_MISS_ID 0x3ff
-
-/*
- * struct ipahal_context - HAL global context data
- * @hw_type: IPA H/W type/version.
- * @base: Base address to be used for accessing IPA memory. This is
- *  I/O memory mapped address.
- *  Controlled by debugfs. default is off
- * @dent: Debugfs folder dir entry
- * @ipa_pdev: IPA Platform Device. Will be used for DMA memory
- * @empty_fltrt_tbl: Empty table to be used at tables init.
- */
-struct ipahal_context {
-	enum ipa_hw_type hw_type;
-	void __iomem *base;
-	struct dentry *dent;
-	struct device *ipa_pdev;
-	struct ipa_mem_buffer empty_fltrt_tbl;
-	void *regdumpbuf;
-};
-
-extern struct ipahal_context *ipahal_ctx;
-
-
-
-/* Immediate commands H/W structures */
-
-/*
- * struct ipa_imm_cmd_hw_ip_v4_filter_init - IP_V4_FILTER_INIT command payload
- *  in H/W format.
- * Inits IPv4 filter block.
- * @hash_rules_addr: Addr in system mem where ipv4 hashable flt rules starts
- * @hash_rules_size: Size in bytes of the hashable tbl to cpy to local mem
- * @hash_local_addr: Addr in shared mem where ipv4 hashable flt tbl should
- *  be copied to
- * @nhash_rules_size: Size in bytes of the non-hashable tbl to cpy to local mem
- * @nhash_local_addr: Addr in shared mem where ipv4 non-hashable flt tbl should
- *  be copied to
- * @rsvd: reserved
- * @nhash_rules_addr: Addr in sys mem where ipv4 non-hashable flt tbl starts
- */
-struct ipa_imm_cmd_hw_ip_v4_filter_init {
-	u64 hash_rules_addr:64;
-	u64 hash_rules_size:12;
-	u64 hash_local_addr:16;
-	u64 nhash_rules_size:12;
-	u64 nhash_local_addr:16;
-	u64 rsvd:8;
-	u64 nhash_rules_addr:64;
-};
-
-/*
- * struct ipa_imm_cmd_hw_ip_v6_filter_init - IP_V6_FILTER_INIT command payload
- *  in H/W format.
- * Inits IPv6 filter block.
- * @hash_rules_addr: Addr in system mem where ipv6 hashable flt rules starts
- * @hash_rules_size: Size in bytes of the hashable tbl to cpy to local mem
- * @hash_local_addr: Addr in shared mem where ipv6 hashable flt tbl should
- *  be copied to
- * @nhash_rules_size: Size in bytes of the non-hashable tbl to cpy to local mem
- * @nhash_local_addr: Addr in shared mem where ipv6 non-hashable flt tbl should
- *  be copied to
- * @rsvd: reserved
- * @nhash_rules_addr: Addr in sys mem where ipv6 non-hashable flt tbl starts
- */
-struct ipa_imm_cmd_hw_ip_v6_filter_init {
-	u64 hash_rules_addr:64;
-	u64 hash_rules_size:12;
-	u64 hash_local_addr:16;
-	u64 nhash_rules_size:12;
-	u64 nhash_local_addr:16;
-	u64 rsvd:8;
-	u64 nhash_rules_addr:64;
-};
-
-/*
- * struct ipa_imm_cmd_hw_ip_v4_nat_init - IP_V4_NAT_INIT command payload
- *  in H/W format.
- * Inits IPv4 NAT block. Initiate NAT table with it dimensions, location
- *  cache address and other related parameters.
- * @ipv4_rules_addr: Addr in sys/shared mem where ipv4 NAT rules start
- * @ipv4_expansion_rules_addr: Addr in sys/shared mem where expansion NAT
- *  table starts. IPv4 NAT rules that result in hash collision are located
- *  in this table.
- * @index_table_addr: Addr in sys/shared mem where index table, which points
- *  to NAT table starts
- * @index_table_expansion_addr: Addr in sys/shared mem where expansion index
- *  table starts
- * @table_index: For future support of multiple NAT tables
- * @rsvd1: reserved
- * @ipv4_rules_addr_type: ipv4_rules_addr in sys or shared mem
- * @ipv4_expansion_rules_addr_type: ipv4_expansion_rules_addr in
- *  sys or shared mem
- * @index_table_addr_type: index_table_addr in sys or shared mem
- * @index_table_expansion_addr_type: index_table_expansion_addr in
- *  sys or shared mem
- * @size_base_tables: Num of entries in NAT tbl and idx tbl (each)
- * @size_expansion_tables: Num of entries in NAT expansion tbl and expansion
- *  idx tbl (each)
- * @rsvd2: reserved
- * @public_addr_info: Public IP addresses info suitable to the IPA H/W version
- *                    IPA H/W >= 4.0 - PDN config table offset in SMEM
- *                    IPA H/W < 4.0  - The public IP address
- */
-struct ipa_imm_cmd_hw_ip_v4_nat_init {
-	u64 ipv4_rules_addr:64;
-	u64 ipv4_expansion_rules_addr:64;
-	u64 index_table_addr:64;
-	u64 index_table_expansion_addr:64;
-	u64 table_index:3;
-	u64 rsvd1:1;
-	u64 ipv4_rules_addr_type:1;
-	u64 ipv4_expansion_rules_addr_type:1;
-	u64 index_table_addr_type:1;
-	u64 index_table_expansion_addr_type:1;
-	u64 size_base_tables:12;
-	u64 size_expansion_tables:10;
-	u64 rsvd2:2;
-	u64 public_addr_info:32;
-};
-
-/*
- * struct ipa_imm_cmd_hw_ip_v6_ct_init - IP_V6_CONN_TRACK_INIT command payload
- *  in H/W format.
- * Inits IPv6CT block. Initiate IPv6CT table with it dimensions, location
- *  cache address and other related parameters.
- * @table_addr: Address in sys/shared mem where IPv6CT rules start
- * @expansion_table_addr: Address in sys/shared mem where IPv6CT expansion
- *  table starts. IPv6CT rules that result in hash collision are located
- *  in this table.
- * @table_index: For future support of multiple IPv6CT tables
- * @rsvd1: reserved
- * @table_addr_type: table_addr in sys or shared mem
- * @expansion_table_addr_type: expansion_table_addr in sys or shared mem
- * @rsvd2: reserved
- * @size_base_tables: Number of entries in IPv6CT table
- * @size_expansion_tables: Number of entries in IPv6CT expansion table
- * @rsvd3: reserved
- */
-struct ipa_imm_cmd_hw_ip_v6_ct_init {
-	u64 table_addr:64;
-	u64 expansion_table_addr:64;
-	u64 table_index:3;
-	u64 rsvd1:1;
-	u64 table_addr_type:1;
-	u64 expansion_table_addr_type:1;
-	u64 rsvd2:2;
-	u64 size_base_table:12;
-	u64 size_expansion_table:10;
-	u64 rsvd3:34;
-};
-
-/*
- * struct ipa_imm_cmd_hw_ip_v4_routing_init - IP_V4_ROUTING_INIT command payload
- *  in H/W format.
- * Inits IPv4 routing table/structure - with the rules and other related params
- * @hash_rules_addr: Addr in system mem where ipv4 hashable rt rules starts
- * @hash_rules_size: Size in bytes of the hashable tbl to cpy to local mem
- * @hash_local_addr: Addr in shared mem where ipv4 hashable rt tbl should
- *  be copied to
- * @nhash_rules_size: Size in bytes of the non-hashable tbl to cpy to local mem
- * @nhash_local_addr: Addr in shared mem where ipv4 non-hashable rt tbl should
- *  be copied to
- * @rsvd: reserved
- * @nhash_rules_addr: Addr in sys mem where ipv4 non-hashable rt tbl starts
- */
-struct ipa_imm_cmd_hw_ip_v4_routing_init {
-	u64 hash_rules_addr:64;
-	u64 hash_rules_size:12;
-	u64 hash_local_addr:16;
-	u64 nhash_rules_size:12;
-	u64 nhash_local_addr:16;
-	u64 rsvd:8;
-	u64 nhash_rules_addr:64;
-};
-
-/*
- * struct ipa_imm_cmd_hw_ip_v6_routing_init - IP_V6_ROUTING_INIT command payload
- *  in H/W format.
- * Inits IPv6 routing table/structure - with the rules and other related params
- * @hash_rules_addr: Addr in system mem where ipv6 hashable rt rules starts
- * @hash_rules_size: Size in bytes of the hashable tbl to cpy to local mem
- * @hash_local_addr: Addr in shared mem where ipv6 hashable rt tbl should
- *  be copied to
- * @nhash_rules_size: Size in bytes of the non-hashable tbl to cpy to local mem
- * @nhash_local_addr: Addr in shared mem where ipv6 non-hashable rt tbl should
- *  be copied to
- * @rsvd: reserved
- * @nhash_rules_addr: Addr in sys mem where ipv6 non-hashable rt tbl starts
- */
-struct ipa_imm_cmd_hw_ip_v6_routing_init {
-	u64 hash_rules_addr:64;
-	u64 hash_rules_size:12;
-	u64 hash_local_addr:16;
-	u64 nhash_rules_size:12;
-	u64 nhash_local_addr:16;
-	u64 rsvd:8;
-	u64 nhash_rules_addr:64;
-};
-
-/*
- * struct ipa_imm_cmd_hw_hdr_init_local - HDR_INIT_LOCAL command payload
- *  in H/W format.
- * Inits hdr table within local mem with the hdrs and their length.
- * @hdr_table_addr: Word address in sys mem where the table starts (SRC)
- * @size_hdr_table: Size of the above (in bytes)
- * @hdr_addr: header address in IPA sram (used as DST for memory copy)
- * @rsvd: reserved
- */
-struct ipa_imm_cmd_hw_hdr_init_local {
-	u64 hdr_table_addr:64;
-	u64 size_hdr_table:12;
-	u64 hdr_addr:16;
-	u64 rsvd:4;
-};
-
-/*
- * struct ipa_imm_cmd_hw_nat_dma - NAT_DMA command payload
- *  in H/W format
- * Perform DMA operation on NAT related mem addressess. Copy data into
- *  different locations within NAT associated tbls. (For add/remove NAT rules)
- * @table_index: NAT tbl index. Defines the NAT tbl on which to perform DMA op.
- * @rsvd1: reserved
- * @base_addr: Base addr to which the DMA operation should be performed.
- * @rsvd2: reserved
- * @offset: offset in bytes from base addr to write 'data' to
- * @data: data to be written
- * @rsvd3: reserved
- */
-struct ipa_imm_cmd_hw_nat_dma {
-	u64 table_index:3;
-	u64 rsvd1:1;
-	u64 base_addr:2;
-	u64 rsvd2:2;
-	u64 offset:32;
-	u64 data:16;
-	u64 rsvd3:8;
-};
-
-/*
- * struct ipa_imm_cmd_hw_table_dma_ipav4 - TABLE_DMA command payload
- *  in H/W format
- * Perform DMA operation on NAT and ipv6 connection tracking related mem
- * addresses. Copy data into different locations within NAT associated tbls
- * (For add/remove NAT rules)
- * @table_index: NAT tbl index. Defines the NAT tbl on which to perform DMA op.
- * @rsvd1: reserved
- * @base_addr: Base addr to which the DMA operation should be performed.
- * @rsvd2: reserved
- * @offset: offset in bytes from base addr to write 'data' to
- * @data: data to be written
- * @rsvd3: reserved
- */
-struct ipa_imm_cmd_hw_table_dma_ipav4 {
-	u64 table_index : 3;
-	u64 rsvd1 : 1;
-	u64 base_addr : 3;
-	u64 rsvd2 : 1;
-	u64 offset : 32;
-	u64 data : 16;
-	u64 rsvd3 : 8;
-};
-
-/*
- * struct ipa_imm_cmd_hw_hdr_init_system - HDR_INIT_SYSTEM command payload
- *  in H/W format.
- * Inits hdr table within sys mem with the hdrs and their length.
- * @hdr_table_addr: Word address in system memory where the hdrs tbl starts.
- */
-struct ipa_imm_cmd_hw_hdr_init_system {
-	u64 hdr_table_addr:64;
-};
-
-/*
- * struct ipa_imm_cmd_hw_ip_packet_init - IP_PACKET_INIT command payload
- *  in H/W format.
- * Configuration for specific IP pkt. Shall be called prior to an IP pkt
- *  data. Pkt will not go through IP pkt processing.
- * @destination_pipe_index: Destination pipe index  (in case routing
- *  is enabled, this field will overwrite the rt  rule)
- * @rsvd: reserved
- */
-struct ipa_imm_cmd_hw_ip_packet_init {
-	u64 destination_pipe_index:5;
-	u64 rsv1:59;
-};
-
-/*
- * struct ipa_imm_cmd_hw_register_write - REGISTER_WRITE command payload
- *  in H/W format.
- * Write value to register. Allows reg changes to be synced with data packet
- *  and other immediate command. Can be used to access the sram
- * @sw_rsvd: Ignored by H/W. May be used by S/W
- * @skip_pipeline_clear: 0 to wait until IPA pipeline is clear. 1 don't wait
- * @offset: offset from IPA base address - Lower 16bit of the IPA reg addr
- * @value: value to write to register
- * @value_mask: mask specifying which value bits to write to the register
- * @pipeline_clear_options: options for pipeline to clear
- *	0: HPS - no pkt inside HPS (not grp specific)
- *	1: source group - The immediate cmd src grp does not use any pkt ctxs
- *	2: Wait until no pkt reside inside IPA pipeline
- *	3: reserved
- * @rsvd: reserved - should be set to zero
- */
-struct ipa_imm_cmd_hw_register_write {
-	u64 sw_rsvd:15;
-	u64 skip_pipeline_clear:1;
-	u64 offset:16;
-	u64 value:32;
-	u64 value_mask:32;
-	u64 pipeline_clear_options:2;
-	u64 rsvd:30;
-};
-
-/*
- * struct ipa_imm_cmd_hw_register_write - REGISTER_WRITE command payload
- *  in H/W format.
- * Write value to register. Allows reg changes to be synced with data packet
- *  and other immediate command. Can be used to access the sram
- * @sw_rsvd: Ignored by H/W. May be used by S/W
- * @offset_high: high bits of the Offset field - bits 17-20
- * @rsvd: reserved - should be set to zero
- * @offset: offset from IPA base address - Lower 16bit of the IPA reg addr
- * @value: value to write to register
- * @value_mask: mask specifying which value bits to write to the register
- * @rsvd2: reserved - should be set to zero
- */
-struct ipa_imm_cmd_hw_register_write_v_4_0 {
-	u64 sw_rsvd:11;
-	u64 offset_high:4;
-	u64 rsvd:1;
-	u64 offset:16;
-	u64 value:32;
-	u64 value_mask:32;
-	u64 rsvd2:32;
-};
-
-/*
- * struct ipa_imm_cmd_hw_dma_shared_mem - DMA_SHARED_MEM command payload
- *  in H/W format.
- * Perform mem copy into or out of the SW area of IPA local mem
- * @sw_rsvd: Ignored by H/W. My be used by S/W
- * @size: Size in bytes of data to copy. Expected size is up to 2K bytes
- * @local_addr: Address in IPA local memory
- * @direction: Read or write?
- *	0: IPA write, Write to local address from system address
- *	1: IPA read, Read from local address to system address
- * @skip_pipeline_clear: 0 to wait until IPA pipeline is clear. 1 don't wait
- * @pipeline_clear_options: options for pipeline to clear
- *	0: HPS - no pkt inside HPS (not grp specific)
- *	1: source group - The immediate cmd src grp does npt use any pkt ctxs
- *	2: Wait until no pkt reside inside IPA pipeline
- *	3: reserved
- * @rsvd: reserved - should be set to zero
- * @system_addr: Address in system memory
- */
-struct ipa_imm_cmd_hw_dma_shared_mem {
-	u64 sw_rsvd:16;
-	u64 size:16;
-	u64 local_addr:16;
-	u64 direction:1;
-	u64 skip_pipeline_clear:1;
-	u64 pipeline_clear_options:2;
-	u64 rsvd:12;
-	u64 system_addr:64;
-};
-
-/*
- * struct ipa_imm_cmd_hw_dma_shared_mem - DMA_SHARED_MEM command payload
- *  in H/W format.
- * Perform mem copy into or out of the SW area of IPA local mem
- * @sw_rsvd: Ignored by H/W. My be used by S/W
- * @size: Size in bytes of data to copy. Expected size is up to 2K bytes
- * @clear_after_read: Clear local memory at the end of a read operation allows
- *  atomic read and clear if HPS is clear. Ignore for writes.
- * @local_addr: Address in IPA local memory
- * @direction: Read or write?
- *	0: IPA write, Write to local address from system address
- *	1: IPA read, Read from local address to system address
- * @rsvd: reserved - should be set to zero
- * @system_addr: Address in system memory
- */
-struct ipa_imm_cmd_hw_dma_shared_mem_v_4_0 {
-	u64 sw_rsvd:15;
-	u64 clear_after_read:1;
-	u64 size:16;
-	u64 local_addr:16;
-	u64 direction:1;
-	u64 rsvd:15;
-	u64 system_addr:64;
-};
-
-/*
- * struct ipa_imm_cmd_hw_ip_packet_tag_status -
- *  IP_PACKET_TAG_STATUS command payload in H/W format.
- * This cmd is used for to allow SW to track HW processing by setting a TAG
- *  value that is passed back to SW inside Packet Status information.
- *  TAG info will be provided as part of Packet Status info generated for
- *  the next pkt transferred over the pipe.
- *  This immediate command must be followed by a packet in the same transfer.
- * @sw_rsvd: Ignored by H/W. My be used by S/W
- * @tag: Tag that is provided back to SW
- */
-struct ipa_imm_cmd_hw_ip_packet_tag_status {
-	u64 sw_rsvd:16;
-	u64 tag:48;
-};
-
-/*
- * struct ipa_imm_cmd_hw_dma_task_32b_addr -
- *	IPA_DMA_TASK_32B_ADDR command payload in H/W format.
- * Used by clients using 32bit addresses. Used to perform DMA operation on
- *  multiple descriptors.
- *  The Opcode is dynamic, where it holds the number of buffer to process
- * @sw_rsvd: Ignored by H/W. My be used by S/W
- * @cmplt: Complete flag: When asserted IPA will interrupt SW when the entire
- *  DMA related data was completely xfered to its destination.
- * @eof: Enf Of Frame flag: When asserted IPA will assert the EOT to the
- *  dest client. This is used used for aggr sequence
- * @flsh: Flush flag: When asserted, pkt will go through the IPA blocks but
- *  will not be xfered to dest client but rather will be discarded
- * @lock: Lock pipe flag: When asserted, IPA will stop processing descriptors
- *  from other EPs in the same src grp (RX queue)
- * @unlock: Unlock pipe flag: When asserted, IPA will stop exclusively
- *  servicing current EP out of the src EPs of the grp (RX queue)
- * @size1: Size of buffer1 data
- * @addr1: Pointer to buffer1 data
- * @packet_size: Total packet size. If a pkt send using multiple DMA_TASKs,
- *  only the first one needs to have this field set. It will be ignored
- *  in subsequent DMA_TASKs until the packet ends (EOT). First DMA_TASK
- *  must contain this field (2 or more buffers) or EOT.
- */
-struct ipa_imm_cmd_hw_dma_task_32b_addr {
-	u64 sw_rsvd:11;
-	u64 cmplt:1;
-	u64 eof:1;
-	u64 flsh:1;
-	u64 lock:1;
-	u64 unlock:1;
-	u64 size1:16;
-	u64 addr1:32;
-	u64 packet_size:16;
-};
-
-
-
-/* IPA Status packet H/W structures and info */
-
-/*
- * struct ipa_status_pkt_hw - IPA status packet payload in H/W format.
- *  This structure describes the status packet H/W structure for the
- *   following statuses: IPA_STATUS_PACKET, IPA_STATUS_DROPPED_PACKET,
- *   IPA_STATUS_SUSPENDED_PACKET.
- *  Other statuses types has different status packet structure.
- * @status_opcode: The Type of the status (Opcode).
- * @exception: (not bitmask) - the first exception that took place.
- *  In case of exception, src endp and pkt len are always valid.
- * @status_mask: Bit mask specifying on which H/W blocks the pkt was processed.
- * @pkt_len: Pkt pyld len including hdr, include retained hdr if used. Does
- *  not include padding or checksum trailer len.
- * @endp_src_idx: Source end point index.
- * @rsvd1: reserved
- * @endp_dest_idx: Destination end point index.
- *  Not valid in case of exception
- * @rsvd2: reserved
- * @metadata: meta data value used by packet
- * @flt_local: Filter table location flag: Does matching flt rule belongs to
- *  flt tbl that resides in lcl memory? (if not, then system mem)
- * @flt_hash: Filter hash hit flag: Does matching flt rule was in hash tbl?
- * @flt_global: Global filter rule flag: Does matching flt rule belongs to
- *  the global flt tbl? (if not, then the per endp tables)
- * @flt_ret_hdr: Retain header in filter rule flag: Does matching flt rule
- *  specifies to retain header?
- *  Starting IPA4.5, this will be true only if packet has L2 header.
- * @flt_rule_id: The ID of the matching filter rule. This info can be combined
- *  with endp_src_idx to locate the exact rule. ID=0x3FF reserved to specify
- *  flt miss. In case of miss, all flt info to be ignored
- * @rt_local: Route table location flag: Does matching rt rule belongs to
- *  rt tbl that resides in lcl memory? (if not, then system mem)
- * @rt_hash: Route hash hit flag: Does matching rt rule was in hash tbl?
- * @ucp: UC Processing flag.
- * @rt_tbl_idx: Index of rt tbl that contains the rule on which was a match
- * @rt_rule_id: The ID of the matching rt rule. This info can be combined
- *  with rt_tbl_idx to locate the exact rule. ID=0x3FF reserved to specify
- *  rt miss. In case of miss, all rt info to be ignored
- * @nat_hit: NAT hit flag: Was their NAT hit?
- * @nat_entry_idx: Index of the NAT entry used of NAT processing
- * @nat_type: Defines the type of the NAT operation:
- *	00: No NAT
- *	01: Source NAT
- *	10: Destination NAT
- *	11: Reserved
- * @tag_info: S/W defined value provided via immediate command
- * @seq_num: Per source endp unique packet sequence number
- * @time_of_day_ctr: running counter from IPA clock
- * @hdr_local: Header table location flag: In header insertion, was the header
- *  taken from the table resides in local memory? (If no, then system mem)
- * @hdr_offset: Offset of used header in the header table
- * @frag_hit: Frag hit flag: Was their frag rule hit in H/W frag table?
- * @frag_rule: Frag rule index in H/W frag table in case of frag hit
- * @hw_specific: H/W specific reserved value
- */
-struct ipa_pkt_status_hw {
-	u64 status_opcode:8;
-	u64 exception:8;
-	u64 status_mask:16;
-	u64 pkt_len:16;
-	u64 endp_src_idx:5;
-	u64 rsvd1:3;
-	u64 endp_dest_idx:5;
-	u64 rsvd2:3;
-	u64 metadata:32;
-	u64 flt_local:1;
-	u64 flt_hash:1;
-	u64 flt_global:1;
-	u64 flt_ret_hdr:1;
-	u64 flt_rule_id:10;
-	u64 rt_local:1;
-	u64 rt_hash:1;
-	u64 ucp:1;
-	u64 rt_tbl_idx:5;
-	u64 rt_rule_id:10;
-	u64 nat_hit:1;
-	u64 nat_entry_idx:13;
-	u64 nat_type:2;
-	u64 tag_info:48;
-	u64 seq_num:8;
-	u64 time_of_day_ctr:24;
-	u64 hdr_local:1;
-	u64 hdr_offset:10;
-	u64 frag_hit:1;
-	u64 frag_rule:4;
-	u64 hw_specific:16;
-};
-
-/* Size of H/W Packet Status */
-#define IPA3_0_PKT_STATUS_SIZE 32
-
-/* Headers and processing context H/W structures and definitions */
-
-/* uCP command numbers */
-#define IPA_HDR_UCP_802_3_TO_802_3 6
-#define IPA_HDR_UCP_802_3_TO_ETHII 7
-#define IPA_HDR_UCP_ETHII_TO_802_3 8
-#define IPA_HDR_UCP_ETHII_TO_ETHII 9
-#define IPA_HDR_UCP_L2TP_HEADER_ADD 10
-#define IPA_HDR_UCP_L2TP_HEADER_REMOVE 11
-
-/* Processing context TLV type */
-#define IPA_PROC_CTX_TLV_TYPE_END 0
-#define IPA_PROC_CTX_TLV_TYPE_HDR_ADD 1
-#define IPA_PROC_CTX_TLV_TYPE_PROC_CMD 3
-
-/**
- * struct ipa_hw_hdr_proc_ctx_tlv -
- * HW structure of IPA processing context header - TLV part
- * @type: 0 - end type
- *        1 - header addition type
- *        3 - processing command type
- * @length: number of bytes after tlv
- *        for type:
- *        0 - needs to be 0
- *        1 - header addition length
- *        3 - number of 32B including type and length.
- * @value: specific value for type
- *        for type:
- *        0 - needs to be 0
- *        1 - header length
- *        3 - command ID (see IPA_HDR_UCP_* definitions)
- */
-struct ipa_hw_hdr_proc_ctx_tlv {
-	u32 type:8;
-	u32 length:8;
-	u32 value:16;
-};
-
-/**
- * struct ipa_hw_hdr_proc_ctx_hdr_add -
- * HW structure of IPA processing context - add header tlv
- * @tlv: IPA processing context TLV
- * @hdr_addr: processing context header address
- */
-struct ipa_hw_hdr_proc_ctx_hdr_add {
-	struct ipa_hw_hdr_proc_ctx_tlv tlv;
-	u32 hdr_addr;
-	u32 hdr_addr_hi;
-};
-
-/**
- * struct ipa_hw_hdr_proc_ctx_l2tp_add_hdr -
- * HW structure of IPA processing context - add l2tp header tlv
- * @tlv: IPA processing context TLV
- * @l2tp_params: l2tp parameters
- */
-struct ipa_hw_hdr_proc_ctx_l2tp_add_hdr {
-	struct ipa_hw_hdr_proc_ctx_tlv tlv;
-	struct ipa_l2tp_header_add_procparams l2tp_params;
-};
-
-/**
- * struct ipa_hw_hdr_proc_ctx_l2tp_remove_hdr -
- * HW structure of IPA processing context - remove l2tp header tlv
- * @tlv: IPA processing context TLV
- * @l2tp_params: l2tp parameters
- */
-struct ipa_hw_hdr_proc_ctx_l2tp_remove_hdr {
-	struct ipa_hw_hdr_proc_ctx_tlv tlv;
-	struct ipa_l2tp_header_remove_procparams l2tp_params;
-};
-
-/**
- * struct ipa_hw_hdr_proc_ctx_add_hdr_seq -
- * IPA processing context header - add header sequence
- * @hdr_add: add header command
- * @end: tlv end command (cmd.type must be 0)
- */
-struct ipa_hw_hdr_proc_ctx_add_hdr_seq {
-	struct ipa_hw_hdr_proc_ctx_hdr_add hdr_add;
-	struct ipa_hw_hdr_proc_ctx_tlv end;
-};
-
-/**
- * struct ipa_hw_hdr_proc_ctx_add_hdr_cmd_seq -
- * IPA processing context header - process command sequence
- * @hdr_add: add header command
- * @cmd: tlv processing command (cmd.type must be 3)
- * @end: tlv end command (cmd.type must be 0)
- */
-struct ipa_hw_hdr_proc_ctx_add_hdr_cmd_seq {
-	struct ipa_hw_hdr_proc_ctx_hdr_add hdr_add;
-	struct ipa_hw_hdr_proc_ctx_tlv cmd;
-	struct ipa_hw_hdr_proc_ctx_tlv end;
-};
-
-/**
- * struct ipa_hw_hdr_proc_ctx_add_l2tp_hdr_cmd_seq -
- * IPA processing context header - process command sequence
- * @hdr_add: add header command
- * @l2tp_params: l2tp params for header addition
- * @end: tlv end command (cmd.type must be 0)
- */
-struct ipa_hw_hdr_proc_ctx_add_l2tp_hdr_cmd_seq {
-	struct ipa_hw_hdr_proc_ctx_hdr_add hdr_add;
-	struct ipa_hw_hdr_proc_ctx_l2tp_add_hdr l2tp_params;
-	struct ipa_hw_hdr_proc_ctx_tlv end;
-};
-
-/**
- * struct ipa_hw_hdr_proc_ctx_remove_l2tp_hdr_cmd_seq -
- * IPA processing context header - process command sequence
- * @hdr_add: add header command
- * @l2tp_params: l2tp params for header removal
- * @end: tlv end command (cmd.type must be 0)
- */
-struct ipa_hw_hdr_proc_ctx_remove_l2tp_hdr_cmd_seq {
-	struct ipa_hw_hdr_proc_ctx_hdr_add hdr_add;
-	struct ipa_hw_hdr_proc_ctx_l2tp_remove_hdr l2tp_params;
-	struct ipa_hw_hdr_proc_ctx_tlv end;
-};
-
-#endif /* _IPAHAL_I_H_ */

ipa/ipa_v3/ipahal/ipahal_nat.c

@@ -1,510 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/debugfs.h>
-#include "ipahal_nat.h"
-#include "ipahal_nat_i.h"
-#include "ipahal_i.h"
-
-#define IPA_64_LOW_32_MASK (0xFFFFFFFF)
-#define IPA_64_HIGH_32_MASK (0xFFFFFFFF00000000ULL)
-
-static const char *ipahal_nat_type_to_str[IPA_NAT_MAX] = {
-	__stringify(IPAHAL_NAT_IPV4),
-	__stringify(IPAHAL_NAT_IPV4_INDEX),
-	__stringify(IPAHAL_NAT_IPV4_PDN),
-	__stringify(IPAHAL_NAT_IPV6CT)
-};
-
-static size_t ipa_nat_ipv4_entry_size_v_3_0(void)
-{
-	return sizeof(struct ipa_nat_hw_ipv4_entry);
-}
-
-static size_t ipa_nat_ipv4_index_entry_size_v_3_0(void)
-{
-	return sizeof(struct ipa_nat_hw_indx_entry);
-}
-
-static size_t ipa_nat_ipv4_pdn_entry_size_v_4_0(void)
-{
-	return sizeof(struct ipa_nat_hw_pdn_entry);
-}
-
-static size_t ipa_nat_ipv6ct_entry_size_v_4_0(void)
-{
-	return sizeof(struct ipa_nat_hw_ipv6ct_entry);
-}
-
-static bool ipa_nat_ipv4_is_entry_zeroed_v_3_0(const void *entry)
-{
-	struct ipa_nat_hw_ipv4_entry zero_entry = { 0 };
-
-	return (memcmp(&zero_entry, entry, sizeof(zero_entry))) ? false : true;
-}
-
-static bool ipa_nat_ipv4_is_index_entry_zeroed_v_3_0(const void *entry)
-{
-	struct ipa_nat_hw_indx_entry zero_entry = { 0 };
-
-	return (memcmp(&zero_entry, entry, sizeof(zero_entry))) ? false : true;
-}
-
-static bool ipa_nat_ipv4_is_pdn_entry_zeroed_v_4_0(const void *entry)
-{
-	struct ipa_nat_hw_pdn_entry zero_entry = { 0 };
-
-	return (memcmp(&zero_entry, entry, sizeof(zero_entry))) ? false : true;
-}
-
-static bool ipa_nat_ipv6ct_is_entry_zeroed_v_4_0(const void *entry)
-{
-	struct ipa_nat_hw_ipv6ct_entry zero_entry = { 0 };
-
-	return (memcmp(&zero_entry, entry, sizeof(zero_entry))) ? false : true;
-}
-
-static bool ipa_nat_ipv4_is_entry_valid_v_3_0(const void *entry)
-{
-	struct ipa_nat_hw_ipv4_entry *hw_entry =
-		(struct ipa_nat_hw_ipv4_entry *)entry;
-
-	return hw_entry->enable &&
-		hw_entry->protocol != IPAHAL_NAT_INVALID_PROTOCOL;
-}
-
-static bool ipa_nat_ipv4_is_index_entry_valid_v_3_0(const void *entry)
-{
-	struct ipa_nat_hw_indx_entry *hw_entry =
-		(struct ipa_nat_hw_indx_entry *)entry;
-
-	return hw_entry->tbl_entry != 0;
-}
-
-static bool ipa_nat_ipv4_is_pdn_entry_valid_v_4_0(const void *entry)
-{
-	struct ipa_nat_hw_pdn_entry *hw_entry =
-		(struct ipa_nat_hw_pdn_entry *)entry;
-
-	return hw_entry->public_ip != 0;
-}
-
-static bool ipa_nat_ipv6ct_is_entry_valid_v_4_0(const void *entry)
-{
-	struct ipa_nat_hw_ipv6ct_entry *hw_entry =
-		(struct ipa_nat_hw_ipv6ct_entry *)entry;
-
-	return hw_entry->enable &&
-		hw_entry->protocol != IPAHAL_NAT_INVALID_PROTOCOL;
-}
-
-static int ipa_nat_ipv4_stringify_entry_v_3_0(const void *entry,
-	char *buff, size_t buff_size)
-{
-	const struct ipa_nat_hw_ipv4_entry *nat_entry =
-		(const struct ipa_nat_hw_ipv4_entry *)entry;
-
-	return scnprintf(buff, buff_size,
-		"\t\tPrivate_IP=%pI4h  Target_IP=%pI4h\n"
-		"\t\tNext_Index=%d  Public_Port=%d\n"
-		"\t\tPrivate_Port=%d  Target_Port=%d\n"
-		"\t\tIP_CKSM_delta=0x%x  Enable=%s  Redirect=%s\n"
-		"\t\tTime_stamp=0x%x Proto=%d\n"
-		"\t\tPrev_Index=%d  Indx_tbl_entry=%d\n"
-		"\t\tTCP_UDP_cksum_delta=0x%x\n",
-		&nat_entry->private_ip, &nat_entry->target_ip,
-		nat_entry->next_index, nat_entry->public_port,
-		nat_entry->private_port, nat_entry->target_port,
-		nat_entry->ip_chksum,
-		(nat_entry->enable) ? "true" : "false",
-		(nat_entry->redirect) ? "Direct_To_APPS" : "Fwd_to_route",
-		nat_entry->time_stamp, nat_entry->protocol,
-		nat_entry->prev_index, nat_entry->indx_tbl_entry,
-		nat_entry->tcp_udp_chksum);
-}
-
-static int ipa_nat_ipv4_stringify_entry_v_4_0(const void *entry,
-	char *buff, size_t buff_size)
-{
-	int length;
-	const struct ipa_nat_hw_ipv4_entry *nat_entry =
-		(const struct ipa_nat_hw_ipv4_entry *)entry;
-
-	length = ipa_nat_ipv4_stringify_entry_v_3_0(entry, buff, buff_size);
-
-	length += scnprintf(buff + length, buff_size - length,
-		"\t\tPDN_Index=%d\n", nat_entry->pdn_index);
-
-	return length;
-}
-
-static int ipa_nat_ipv4_index_stringify_entry_v_3_0(const void *entry,
-	char *buff, size_t buff_size)
-{
-	const struct ipa_nat_hw_indx_entry *index_entry =
-		(const struct ipa_nat_hw_indx_entry *)entry;
-
-	return scnprintf(buff, buff_size,
-		"\t\tTable_Entry=%d  Next_Index=%d\n",
-		index_entry->tbl_entry, index_entry->next_index);
-}
-
-static int ipa_nat_ipv4_pdn_stringify_entry_v_4_0(const void *entry,
-	char *buff, size_t buff_size)
-{
-	const struct ipa_nat_hw_pdn_entry *pdn_entry =
-		(const struct ipa_nat_hw_pdn_entry *)entry;
-
-	return scnprintf(buff, buff_size,
-		"ip=%pI4h src_metadata=0x%X, dst_metadata=0x%X\n",
-		&pdn_entry->public_ip,
-		pdn_entry->src_metadata, pdn_entry->dst_metadata);
-}
-
-static inline int ipa_nat_ipv6_stringify_addr(char *buff, size_t buff_size,
-	const char *msg, u64 lsb, u64 msb)
-{
-	struct in6_addr addr;
-
-	addr.s6_addr32[0] = cpu_to_be32((msb & IPA_64_HIGH_32_MASK) >> 32);
-	addr.s6_addr32[1] = cpu_to_be32(msb & IPA_64_LOW_32_MASK);
-	addr.s6_addr32[2] = cpu_to_be32((lsb & IPA_64_HIGH_32_MASK) >> 32);
-	addr.s6_addr32[3] = cpu_to_be32(lsb & IPA_64_LOW_32_MASK);
-
-	return scnprintf(buff, buff_size,
-		"\t\t%s_IPv6_Addr=%pI6c\n", msg, &addr);
-}
-
-static int ipa_nat_ipv6ct_stringify_entry_v_4_0(const void *entry,
-	char *buff, size_t buff_size)
-{
-	int length = 0;
-	const struct ipa_nat_hw_ipv6ct_entry *ipv6ct_entry =
-		(const struct ipa_nat_hw_ipv6ct_entry *)entry;
-
-	length += ipa_nat_ipv6_stringify_addr(
-		buff + length,
-		buff_size - length,
-		"Src",
-		ipv6ct_entry->src_ipv6_lsb,
-		ipv6ct_entry->src_ipv6_msb);
-
-	length += ipa_nat_ipv6_stringify_addr(
-		buff + length,
-		buff_size - length,
-		"Dest",
-		ipv6ct_entry->dest_ipv6_lsb,
-		ipv6ct_entry->dest_ipv6_msb);
-
-	length += scnprintf(buff + length, buff_size - length,
-		"\t\tEnable=%s Redirect=%s Time_Stamp=0x%x Proto=%d\n"
-		"\t\tNext_Index=%d Dest_Port=%d Src_Port=%d\n"
-		"\t\tDirection Settings: Out=%s In=%s\n"
-		"\t\tPrev_Index=%d\n",
-		(ipv6ct_entry->enable) ? "true" : "false",
-		(ipv6ct_entry->redirect) ? "Direct_To_APPS" : "Fwd_to_route",
-		ipv6ct_entry->time_stamp,
-		ipv6ct_entry->protocol,
-		ipv6ct_entry->next_index,
-		ipv6ct_entry->dest_port,
-		ipv6ct_entry->src_port,
-		(ipv6ct_entry->out_allowed) ? "Allow" : "Deny",
-		(ipv6ct_entry->in_allowed) ? "Allow" : "Deny",
-		ipv6ct_entry->prev_index);
-
-	return length;
-}
-
-static void ipa_nat_ipv4_pdn_construct_entry_v_4_0(const void *fields,
-	u32 *address)
-{
-	const struct ipahal_nat_pdn_entry *pdn_entry =
-		(const struct ipahal_nat_pdn_entry *)fields;
-
-	struct ipa_nat_hw_pdn_entry *pdn_entry_address =
-		(struct ipa_nat_hw_pdn_entry *)address;
-
-	memset(pdn_entry_address, 0, sizeof(struct ipa_nat_hw_pdn_entry));
-
-	pdn_entry_address->public_ip = pdn_entry->public_ip;
-	pdn_entry_address->src_metadata = pdn_entry->src_metadata;
-	pdn_entry_address->dst_metadata = pdn_entry->dst_metadata;
-}
-
-static void ipa_nat_ipv4_pdn_parse_entry_v_4_0(void *fields,
-	const u32 *address)
-{
-	struct ipahal_nat_pdn_entry *pdn_entry =
-		(struct ipahal_nat_pdn_entry *)fields;
-
-	const struct ipa_nat_hw_pdn_entry *pdn_entry_address =
-		(const struct ipa_nat_hw_pdn_entry *)address;
-
-	pdn_entry->public_ip = pdn_entry_address->public_ip;
-	pdn_entry->src_metadata = pdn_entry_address->src_metadata;
-	pdn_entry->dst_metadata = pdn_entry_address->dst_metadata;
-}
-
-/*
- * struct ipahal_nat_obj - H/W information for specific IPA version
- * @entry_size - CB to get the size of the entry
- * @is_entry_zeroed - CB to determine whether an entry is definitely zero
- * @is_entry_valid - CB to determine whether an entry is valid
- *  Validity criterium depends on entry type. E.g. for NAT base table
- *   Entry need to be with valid protocol and enabled.
- * @stringify_entry - CB to create string that represents an entry
- * @construct_entry - CB to create NAT entry using the given fields
- * @parse_entry - CB to parse NAT entry to the given fields structure
- */
-struct ipahal_nat_obj {
-	size_t (*entry_size)(void);
-	bool (*is_entry_zeroed)(const void *entry);
-	bool (*is_entry_valid)(const void *entry);
-	int (*stringify_entry)(const void *entry, char *buff, size_t buff_size);
-	void (*construct_entry)(const void *fields, u32 *address);
-	void (*parse_entry)(void *fields, const u32 *address);
-};
-
-/*
- * This table contains the info regard each NAT type for IPAv3 and later.
- * Information like: get entry size and stringify entry functions.
- * All the information on all the NAT types on IPAv3 are statically
- * defined below. If information is missing regard some NAT type on some
- * IPA version, the init function will fill it with the information from the
- * previous IPA version.
- * Information is considered missing if all of the fields are 0
- */
-static struct ipahal_nat_obj ipahal_nat_objs[IPA_HW_MAX][IPA_NAT_MAX] = {
-	/* IPAv3 */
-	[IPA_HW_v3_0][IPAHAL_NAT_IPV4] = {
-			ipa_nat_ipv4_entry_size_v_3_0,
-			ipa_nat_ipv4_is_entry_zeroed_v_3_0,
-			ipa_nat_ipv4_is_entry_valid_v_3_0,
-			ipa_nat_ipv4_stringify_entry_v_3_0
-		},
-	[IPA_HW_v3_0][IPAHAL_NAT_IPV4_INDEX] = {
-			ipa_nat_ipv4_index_entry_size_v_3_0,
-			ipa_nat_ipv4_is_index_entry_zeroed_v_3_0,
-			ipa_nat_ipv4_is_index_entry_valid_v_3_0,
-			ipa_nat_ipv4_index_stringify_entry_v_3_0
-		},
-
-	/* IPAv4 */
-	[IPA_HW_v4_0][IPAHAL_NAT_IPV4] = {
-			ipa_nat_ipv4_entry_size_v_3_0,
-			ipa_nat_ipv4_is_entry_zeroed_v_3_0,
-			ipa_nat_ipv4_is_entry_valid_v_3_0,
-			ipa_nat_ipv4_stringify_entry_v_4_0
-		},
-	[IPA_HW_v4_0][IPAHAL_NAT_IPV4_PDN] = {
-			ipa_nat_ipv4_pdn_entry_size_v_4_0,
-			ipa_nat_ipv4_is_pdn_entry_zeroed_v_4_0,
-			ipa_nat_ipv4_is_pdn_entry_valid_v_4_0,
-			ipa_nat_ipv4_pdn_stringify_entry_v_4_0,
-			ipa_nat_ipv4_pdn_construct_entry_v_4_0,
-			ipa_nat_ipv4_pdn_parse_entry_v_4_0
-		},
-	[IPA_HW_v4_0][IPAHAL_NAT_IPV6CT] = {
-			ipa_nat_ipv6ct_entry_size_v_4_0,
-			ipa_nat_ipv6ct_is_entry_zeroed_v_4_0,
-			ipa_nat_ipv6ct_is_entry_valid_v_4_0,
-			ipa_nat_ipv6ct_stringify_entry_v_4_0
-		}
-};
-
-static void ipahal_nat_check_obj(struct ipahal_nat_obj *obj,
-	int nat_type, int ver)
-{
-	WARN(obj->entry_size == NULL, "%s missing entry_size for version %d\n",
-		ipahal_nat_type_str(nat_type), ver);
-	WARN(obj->is_entry_zeroed == NULL,
-		"%s missing is_entry_zeroed for version %d\n",
-		ipahal_nat_type_str(nat_type), ver);
-	WARN(obj->stringify_entry == NULL,
-		"%s missing stringify_entry for version %d\n",
-		ipahal_nat_type_str(nat_type), ver);
-}
-
-/*
- * ipahal_nat_init() - Build the NAT information table
- *  See ipahal_nat_objs[][] comments
- */
-int ipahal_nat_init(enum ipa_hw_type ipa_hw_type)
-{
-	int i;
-	int j;
-	struct ipahal_nat_obj zero_obj, *next_obj;
-
-	IPAHAL_DBG("Entry - HW_TYPE=%d\n", ipa_hw_type);
-
-	memset(&zero_obj, 0, sizeof(zero_obj));
-
-	if ((ipa_hw_type < 0) || (ipa_hw_type >= IPA_HW_MAX)) {
-		IPAHAL_ERR("invalid IPA HW type (%d)\n", ipa_hw_type);
-		return -EINVAL;
-	}
-
-	for (i = IPA_HW_v3_0 ; i < ipa_hw_type ; ++i) {
-		for (j = 0; j < IPA_NAT_MAX; ++j) {
-			next_obj = &ipahal_nat_objs[i + 1][j];
-			if (!memcmp(next_obj, &zero_obj, sizeof(*next_obj))) {
-				memcpy(next_obj, &ipahal_nat_objs[i][j],
-					sizeof(*next_obj));
-			} else {
-				ipahal_nat_check_obj(next_obj, j, i + 1);
-			}
-		}
-	}
-
-	return 0;
-}
-
-const char *ipahal_nat_type_str(enum ipahal_nat_type nat_type)
-{
-	if (nat_type < 0 || nat_type >= IPA_NAT_MAX) {
-		IPAHAL_ERR("requested NAT type %d is invalid\n", nat_type);
-		return "Invalid NAT type";
-	}
-
-	return ipahal_nat_type_to_str[nat_type];
-}
-
-int ipahal_nat_entry_size(enum ipahal_nat_type nat_type, size_t *entry_size)
-{
-	if (WARN(entry_size == NULL, "entry_size is NULL\n"))
-		return -EINVAL;
-	if (WARN(nat_type < 0 || nat_type >= IPA_NAT_MAX,
-		"requested NAT type %d is invalid\n", nat_type))
-		return -EINVAL;
-
-	IPAHAL_DBG("Get the entry size for NAT type=%s\n",
-		ipahal_nat_type_str(nat_type));
-
-	*entry_size =
-		ipahal_nat_objs[ipahal_ctx->hw_type][nat_type].entry_size();
-
-	IPAHAL_DBG("The entry size is %zu\n", *entry_size);
-
-	return 0;
-}
-
-int ipahal_nat_is_entry_zeroed(enum ipahal_nat_type nat_type, void *entry,
-	bool *entry_zeroed)
-{
-	struct ipahal_nat_obj *nat_ptr;
-
-	if (WARN(entry == NULL || entry_zeroed == NULL,
-		"NULL pointer received\n"))
-		return -EINVAL;
-	if (WARN(nat_type < 0 || nat_type >= IPA_NAT_MAX,
-		"requested NAT type %d is invalid\n", nat_type))
-		return -EINVAL;
-
-	IPAHAL_DBG("Determine whether the entry is zeroed for NAT type=%s\n",
-		ipahal_nat_type_str(nat_type));
-
-	nat_ptr =
-		&ipahal_nat_objs[ipahal_ctx->hw_type][nat_type];
-
-	*entry_zeroed = nat_ptr->is_entry_zeroed(entry);
-
-	IPAHAL_DBG("The entry is %szeroed\n", (*entry_zeroed) ? "" : "not ");
-
-	return 0;
-}
-
-int ipahal_nat_is_entry_valid(enum ipahal_nat_type nat_type, void *entry,
-	bool *entry_valid)
-{
-	struct ipahal_nat_obj *nat_obj;
-
-	if (WARN(entry == NULL || entry_valid == NULL,
-		"NULL pointer received\n"))
-		return -EINVAL;
-	if (WARN(nat_type < 0 || nat_type >= IPA_NAT_MAX,
-		"requested NAT type %d is invalid\n", nat_type))
-		return -EINVAL;
-
-	IPAHAL_DBG("Determine whether the entry is valid for NAT type=%s\n",
-		ipahal_nat_type_str(nat_type));
-	nat_obj = &ipahal_nat_objs[ipahal_ctx->hw_type][nat_type];
-	*entry_valid = nat_obj->is_entry_valid(entry);
-	IPAHAL_DBG("The entry is %svalid\n", (*entry_valid) ? "" : "not ");
-
-	return 0;
-}
-
-int ipahal_nat_stringify_entry(enum ipahal_nat_type nat_type, void *entry,
-	char *buff, size_t buff_size)
-{
-	int result;
-	struct ipahal_nat_obj *nat_obj_ptr;
-
-	if (WARN(entry == NULL || buff == NULL, "NULL pointer received\n"))
-		return -EINVAL;
-	if (WARN(!buff_size, "The output buff size is zero\n"))
-		return -EINVAL;
-	if (WARN(nat_type < 0 || nat_type >= IPA_NAT_MAX,
-		"requested NAT type %d is invalid\n", nat_type))
-		return -EINVAL;
-
-	nat_obj_ptr =
-		&ipahal_nat_objs[ipahal_ctx->hw_type][nat_type];
-
-	IPAHAL_DBG("Create the string for the entry of NAT type=%s\n",
-		ipahal_nat_type_str(nat_type));
-
-	result = nat_obj_ptr->stringify_entry(entry, buff, buff_size);
-
-	IPAHAL_DBG("The string successfully created with length %d\n",
-		result);
-
-	return result;
-}
-
-int ipahal_nat_construct_entry(enum ipahal_nat_type nat_type,
-	const void *fields,
-	void *address)
-{
-	struct ipahal_nat_obj *nat_obj_ptr;
-
-	if (WARN(address == NULL || fields == NULL, "NULL pointer received\n"))
-		return -EINVAL;
-	if (WARN(nat_type < 0 || nat_type >= IPA_NAT_MAX,
-		"requested NAT type %d is invalid\n", nat_type))
-		return -EINVAL;
-
-	IPAHAL_DBG("Create %s entry using given fields\n",
-		ipahal_nat_type_str(nat_type));
-
-	nat_obj_ptr =
-		&ipahal_nat_objs[ipahal_ctx->hw_type][nat_type];
-
-	nat_obj_ptr->construct_entry(fields, address);
-
-	return 0;
-}
-
-int ipahal_nat_parse_entry(enum ipahal_nat_type nat_type, void *fields,
-	const void *address)
-{
-	struct ipahal_nat_obj *nat_obj_ptr;
-
-	if (WARN(address == NULL || fields == NULL, "NULL pointer received\n"))
-		return -EINVAL;
-	if (WARN(nat_type < 0 || nat_type >= IPA_NAT_MAX,
-		"requested NAT type %d is invalid\n", nat_type))
-		return -EINVAL;
-
-	IPAHAL_DBG("Get the parsed values for NAT type=%s\n",
-		ipahal_nat_type_str(nat_type));
-
-	nat_obj_ptr =
-		&ipahal_nat_objs[ipahal_ctx->hw_type][nat_type];
-
-	nat_obj_ptr->parse_entry(fields, address);
-
-	return 0;
-}