android_kernel_samsung_sm8650-modules/ipa/ipa_v3/ipa_mpm.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2019, The Linux Foundation. All rights reserved.
 */

#include <linux/dma-mapping.h>
#include <linux/list.h>
#include <linux/slab.h>
#include <linux/device.h>
#include <linux/module.h>
#include <linux/mhi.h>
#include <linux/msm_gsi.h>
#include <linux/delay.h>
#include <linux/log2.h>
#include "../ipa_common_i.h"
#include "ipa_i.h"

#define IPA_MPM_DRV_NAME "ipa_mpm"

#define IPA_MPM_DBG(fmt, args...) \
	do { \
		pr_debug(IPA_MPM_DRV_NAME " %s:%d " fmt, \
			__func__, __LINE__, ## args); \
		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
			IPA_MPM_DRV_NAME " %s:%d " fmt, ## args); \
		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
			IPA_MPM_DRV_NAME " %s:%d " fmt, ## args); \
	} while (0)

#define IPA_MPM_DBG_LOW(fmt, args...) \
	do { \
		pr_debug(IPA_MPM_DRV_NAME " %s:%d " fmt, \
			__func__, __LINE__, ## args); \
		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
			IPA_MPM_DRV_NAME " %s:%d " fmt, ## args); \
	} while (0)


#define IPA_MPM_ERR(fmt, args...) \
	do { \
		pr_err(IPA_MPM_DRV_NAME " %s:%d " fmt, \
			__func__, __LINE__, ## args); \
		IPA_IPC_LOGGING(ipa_get_ipc_logbuf(), \
				IPA_MPM_DRV_NAME " %s:%d " fmt, ## args); \
		IPA_IPC_LOGGING(ipa_get_ipc_logbuf_low(), \
				IPA_MPM_DRV_NAME " %s:%d " fmt, ## args); \
	} while (0)


#define IPA_MPM_FUNC_ENTRY() \
	IPA_MPM_DBG("ENTRY\n")
#define IPA_MPM_FUNC_EXIT() \
	IPA_MPM_DBG("EXIT\n")

#define IPA_MPM_MAX_MHIP_CHAN 3

#define IPA_MPM_NUM_RING_DESC 0x400
#define IPA_MPM_RING_LEN (IPA_MPM_NUM_RING_DESC - 10)

#define IPA_MPM_MHI_HOST_UL_CHANNEL 4
#define IPA_MPM_MHI_HOST_DL_CHANNEL  5
#define TETH_AGGR_TIME_LIMIT 10000 /* 10ms */
#define TETH_AGGR_BYTE_LIMIT 24
#define TETH_AGGR_DL_BYTE_LIMIT 16
#define TRE_BUFF_SIZE 32768
#define IPA_HOLB_TMR_EN 0x1
#define IPA_HOLB_TMR_DIS 0x0
#define RNDIS_IPA_DFLT_RT_HDL 0
#define IPA_POLL_FOR_EMPTINESS_NUM 50
#define IPA_POLL_FOR_EMPTINESS_SLEEP_USEC 20
#define IPA_CHANNEL_STOP_IN_PROC_TO_MSEC 5
#define IPA_CHANNEL_STOP_IN_PROC_SLEEP_USEC 200

enum mhip_re_type {
	MHIP_RE_XFER = 0x2,
	MHIP_RE_NOP = 0x4,
};

enum ipa_mpm_mhi_ch_id_type {
	IPA_MPM_MHIP_CH_ID_0,
	IPA_MPM_MHIP_CH_ID_1,
	IPA_MPM_MHIP_CH_ID_2,
	IPA_MPM_MHIP_CH_ID_MAX,
};

enum ipa_mpm_dma_data_direction {
	DMA_HIPA_BIDIRECTIONAL = 0,
	DMA_TO_HIPA = 1,
	DMA_FROM_HIPA = 2,
	DMA_HIPA_NONE = 3,
};

enum ipa_mpm_ipa_teth_client_type {
	IPA_MPM_MHIP_USB,
	IPA_MPM_MHIP_WIFI,
};

enum ipa_mpm_mhip_client_type {
	IPA_MPM_MHIP_INIT,
	/* USB RMNET CLIENT */
	IPA_MPM_MHIP_USB_RMNET,
	/* USB RNDIS / WIFI CLIENT */
	IPA_MPM_MHIP_TETH,
	/* USB DPL CLIENT */
	IPA_MPM_MHIP_USB_DPL,
	IPA_MPM_MHIP_NONE,
};

enum ipa_mpm_clk_vote_type {
	CLK_ON,
	CLK_OFF,
};

enum mhip_status_type {
	MHIP_STATUS_SUCCESS,
	MHIP_STATUS_NO_OP,
	MHIP_STATUS_FAIL,
	MHIP_STATUS_BAD_STATE,
	MHIP_STATUS_EP_NOT_FOUND,
	MHIP_STATUS_EP_NOT_READY,
};

enum mhip_smmu_domain_type {
	MHIP_SMMU_DOMAIN_IPA,
	MHIP_SMMU_DOMAIN_PCIE,
	MHIP_SMMU_DOMAIN_NONE,
};

enum ipa_mpm_start_stop_type {
	MPM_MHIP_STOP,
	MPM_MHIP_START,
};
/* each pair of UL/DL channels are defined below */
static const struct mhi_device_id mhi_driver_match_table[] = {
	{ .chan = "IP_HW_MHIP_0" }, /* for rndis/Wifi teth pipes */
	{ .chan = "IP_HW_MHIP_1" }, /* for MHIP rmnet */
	{ .chan = "IP_HW_ADPL" }, /* ADPL/ODL DL pipe */
};

static const char *ipa_mpm_mhip_chan_str[IPA_MPM_MHIP_CH_ID_MAX] = {
	__stringify(IPA_MPM_MHIP_TETH),
	__stringify(IPA_MPM_MHIP_USB_RMNET),
	__stringify(IPA_MPM_MHIP_USB_DPL),
};
/*
 * MHI PRIME GSI Descriptor format that Host IPA uses.
 */
struct __packed mhi_p_desc {
	uint64_t buffer_ptr;
	uint16_t buff_len;
	uint16_t resvd1;
	uint16_t chain : 1;
	uint16_t resvd4 : 7;
	uint16_t ieob : 1;
	uint16_t ieot : 1;
	uint16_t bei : 1;
	uint16_t sct : 1;
	uint16_t resvd3 : 4;
	uint8_t re_type;
	uint8_t resvd2;
};

/*
 * MHI PRIME Channel Context and Event Context Array
 * Information that is sent to Device IPA.
 */
struct ipa_mpm_channel_context_type {
	u32 chstate : 8;
	u32 reserved1 : 24;
	u32 chtype;
	u32 erindex;
	u64 rbase;
	u64 rlen;
	u64 reserved2;
	u64 reserved3;
} __packed;

struct ipa_mpm_event_context_type {
	u32 reserved1 : 8;
	u32 update_rp_modc : 8;
	u32 update_rp_intmodt : 16;
	u32 ertype;
	u32 update_rp_addr;
	u64 rbase;
	u64 rlen;
	u32 buff_size : 16;
	u32 reserved2 : 16;
	u32 reserved3;
	u64 reserved4;
} __packed;

struct ipa_mpm_pipes_info_type {
	enum ipa_client_type ipa_client;
	struct ipa_ep_cfg ep_cfg;
};

struct ipa_mpm_channel_type {
	struct ipa_mpm_pipes_info_type dl_cons;
	struct ipa_mpm_pipes_info_type ul_prod;
	enum ipa_mpm_mhip_client_type mhip_client;
};

static struct ipa_mpm_channel_type ipa_mpm_pipes[IPA_MPM_MHIP_CH_ID_MAX];

/* For configuring IPA_CLIENT_MHI_PRIME_TETH_CONS */
static struct ipa_ep_cfg mhip_dl_teth_ep_cfg = {
	.mode = {
		.mode = IPA_BASIC,
		.dst = IPA_CLIENT_MHI_PRIME_TETH_CONS,
	},
	.hdr = {
		.hdr_len = 4,
		.hdr_ofst_metadata_valid = 1,
		.hdr_ofst_metadata = 1,
		.hdr_ofst_pkt_size_valid = 1,
		.hdr_ofst_pkt_size = 2,
	},
	.hdr_ext = {
		.hdr_total_len_or_pad_valid = true,
		.hdr_payload_len_inc_padding = true,
	},
	.aggr = {
		.aggr_en = IPA_ENABLE_DEAGGR,
		.aggr = IPA_QCMAP,
		.aggr_byte_limit = TETH_AGGR_DL_BYTE_LIMIT,
		.aggr_time_limit = TETH_AGGR_TIME_LIMIT,
	},
};

static struct ipa_ep_cfg mhip_ul_teth_ep_cfg = {
	.mode = {
		.mode = IPA_BASIC,
		.dst = IPA_CLIENT_MHI_PRIME_TETH_PROD,
	},
	.hdr = {
		.hdr_len = 4,
		.hdr_ofst_metadata_valid = 1,
		.hdr_ofst_metadata = 0,
		.hdr_ofst_pkt_size_valid = 1,
		.hdr_ofst_pkt_size = 2,
	},
	.hdr_ext = {
		.hdr_total_len_or_pad_valid = true,
		.hdr_payload_len_inc_padding = true,
	},
	.aggr = {
		.aggr_en = IPA_ENABLE_AGGR,
		.aggr = IPA_QCMAP,
		.aggr_byte_limit = TETH_AGGR_BYTE_LIMIT,
		.aggr_time_limit = TETH_AGGR_TIME_LIMIT,
	},

};

/* WARNING!! Temporary for rndis intgration only */


/* For configuring IPA_CLIENT_MHIP_RMNET_PROD */
static struct ipa_ep_cfg mhip_dl_rmnet_ep_cfg = {
	.mode = {
		.mode = IPA_DMA,
		.dst = IPA_CLIENT_USB_CONS,
	},
};

/* For configuring IPA_CLIENT_MHIP_RMNET_CONS */
static struct ipa_ep_cfg mhip_ul_rmnet_ep_cfg = {
	.mode = {
		.mode = IPA_DMA,
		.dst = IPA_CLIENT_USB_CONS,
	},
};

/* For configuring IPA_CLIENT_MHIP_DPL_PROD */
static struct ipa_ep_cfg mhip_dl_dpl_ep_cfg = {
	.mode = {
		.mode = IPA_DMA,
		.dst = IPA_CLIENT_USB_DPL_CONS,
	},
};


struct ipa_mpm_iova_addr {
	dma_addr_t base;
	unsigned int size;
};

struct ipa_mpm_dev_info {
	struct platform_device *pdev;
	struct device *dev;
	bool ipa_smmu_enabled;
	bool pcie_smmu_enabled;
	struct ipa_mpm_iova_addr ctrl;
	struct ipa_mpm_iova_addr data;
	u32 chdb_base;
	u32 erdb_base;
	bool is_cache_coherent;
};

struct ipa_mpm_event_props {
	u16 id;
	phys_addr_t device_db;
	struct ipa_mpm_event_context_type ev_ctx;
};

struct ipa_mpm_channel_props {
	u16 id;
	phys_addr_t device_db;
	struct ipa_mpm_channel_context_type ch_ctx;
};

enum ipa_mpm_gsi_state {
	GSI_ERR,
	GSI_INIT,
	GSI_ALLOCATED,
	GSI_STARTED,
	GSI_STOPPED,
};

struct ipa_mpm_channel {
	struct ipa_mpm_channel_props chan_props;
	struct ipa_mpm_event_props evt_props;
	enum ipa_mpm_gsi_state gsi_state;
	dma_addr_t db_host_iova;
	dma_addr_t db_device_iova;
};

enum ipa_mpm_teth_state {
	IPA_MPM_TETH_INIT = 0,
	IPA_MPM_TETH_INPROGRESS,
	IPA_MPM_TETH_CONNECTED,
};

enum ipa_mpm_mhip_chan {
	IPA_MPM_MHIP_CHAN_UL,
	IPA_MPM_MHIP_CHAN_DL,
	IPA_MPM_MHIP_CHAN_BOTH,
};

struct ipa_mpm_clk_cnt_type {
	atomic_t pcie_clk_cnt;
	atomic_t ipa_clk_cnt;
};

struct producer_rings {
	struct mhi_p_desc *tr_va;
	struct mhi_p_desc *er_va;
	void *tre_buff_va[IPA_MPM_RING_LEN];
	dma_addr_t tr_pa;
	dma_addr_t er_pa;
	dma_addr_t tre_buff_iova[IPA_MPM_RING_LEN];
	/*
	 * The iova generated for AP CB,
	 * used only for dma_map_single to flush the cache.
	 */
	dma_addr_t ap_iova_er;
	dma_addr_t ap_iova_tr;
	dma_addr_t ap_iova_buff[IPA_MPM_RING_LEN];
};

struct ipa_mpm_mhi_driver {
	struct mhi_device *mhi_dev;
	struct producer_rings ul_prod_ring;
	struct producer_rings dl_prod_ring;
	struct ipa_mpm_channel ul_prod;
	struct ipa_mpm_channel dl_cons;
	enum ipa_mpm_mhip_client_type mhip_client;
	enum ipa_mpm_teth_state teth_state;
	struct mutex mutex;
	bool init_complete;
	struct mutex lpm_mutex;
	bool in_lpm;
	struct ipa_mpm_clk_cnt_type clk_cnt;
};

struct ipa_mpm_context {
	struct ipa_mpm_dev_info dev_info;
	struct ipa_mpm_mhi_driver md[IPA_MPM_MAX_MHIP_CHAN];
	struct mutex mutex;
	atomic_t probe_cnt;
	atomic_t pcie_clk_total_cnt;
	atomic_t ipa_clk_total_cnt;
	struct device *parent_pdev;
	struct ipa_smmu_cb_ctx carved_smmu_cb;
	struct device *mhi_parent_dev;
};

#define IPA_MPM_DESC_SIZE (sizeof(struct mhi_p_desc))
#define IPA_MPM_RING_TOTAL_SIZE (IPA_MPM_RING_LEN * IPA_MPM_DESC_SIZE)
/* WA: Make the IPA_MPM_PAGE_SIZE from 16k (next power of ring size) to
 * 32k. This is to make sure IOMMU map happens for the same size
 * for all TR/ER and doorbells.
 */
#define IPA_MPM_PAGE_SIZE TRE_BUFF_SIZE


static struct ipa_mpm_context *ipa_mpm_ctx;
static struct platform_device *m_pdev;
static int ipa_mpm_mhi_probe_cb(struct mhi_device *,
	const struct mhi_device_id *);
static void ipa_mpm_mhi_remove_cb(struct mhi_device *);
static void ipa_mpm_mhi_status_cb(struct mhi_device *, enum MHI_CB);
static void ipa_mpm_change_teth_state(int probe_id,
	enum ipa_mpm_teth_state ip_state);
static void ipa_mpm_change_gsi_state(int probe_id,
	enum ipa_mpm_mhip_chan mhip_chan,
	enum ipa_mpm_gsi_state next_state);
static int ipa_mpm_start_stop_ul_mhip_data_path(int probe_id,
	enum ipa_mpm_start_stop_type start);
static int ipa_mpm_probe(struct platform_device *pdev);
static int ipa_mpm_vote_unvote_pcie_clk(enum ipa_mpm_clk_vote_type vote,
	int probe_id);
static void ipa_mpm_vote_unvote_ipa_clk(enum ipa_mpm_clk_vote_type vote,
	int probe_id);
static enum mhip_status_type ipa_mpm_start_stop_mhip_chan(
	enum ipa_mpm_mhip_chan mhip_chan,
	int probe_id,
	enum ipa_mpm_start_stop_type start_stop);

static struct mhi_driver mhi_driver = {
	.id_table = mhi_driver_match_table,
	.probe = ipa_mpm_mhi_probe_cb,
	.remove = ipa_mpm_mhi_remove_cb,
	.status_cb = ipa_mpm_mhi_status_cb,
	.driver = {
		.name = IPA_MPM_DRV_NAME,
		.owner = THIS_MODULE,
	},
};

static void ipa_mpm_ipa3_delayed_probe(struct work_struct *work)
{
	(void)ipa_mpm_probe(m_pdev);
}

static DECLARE_WORK(ipa_mpm_ipa3_scheduled_probe, ipa_mpm_ipa3_delayed_probe);

static void ipa_mpm_ipa3_ready_cb(void *user_data)
{
	struct platform_device *pdev = (struct platform_device *)(user_data);

	m_pdev = pdev;

	IPA_MPM_DBG("IPA ready callback has been triggered\n");

	schedule_work(&ipa_mpm_ipa3_scheduled_probe);
}

static void ipa_mpm_gsi_evt_ring_err_cb(struct gsi_evt_err_notify *err_data)
{
	IPA_MPM_ERR("GSI EVT RING ERROR, not expected..\n");
	ipa_assert();
}

static void ipa_mpm_gsi_chan_err_cb(struct gsi_chan_err_notify *err_data)
{
	IPA_MPM_ERR("GSI CHAN ERROR, not expected..\n");
	ipa_assert();
}

/**
 * ipa_mpm_smmu_map() - SMMU maps ring and the buffer pointer.
 * @va_addr: virtual address that needs to be mapped
 * @sz: size of the address to be mapped
 * @dir: ipa_mpm_dma_data_direction
 * @ap_cb_iova: iova for AP context bank
 *
 * This function SMMU maps both ring and the buffer pointer.
 * The ring pointers will be aligned to ring size and
 * the buffer pointers should be aligned to buffer size.
 *
 * Returns: iova of the mapped address
 */
static dma_addr_t ipa_mpm_smmu_map(void *va_addr,
	int sz,
	int dir,
	dma_addr_t *ap_cb_iova)
{
	struct iommu_domain *ipa_smmu_domain, *pcie_smmu_domain;
	phys_addr_t phys_addr;
	dma_addr_t iova;
	int smmu_enabled;
	unsigned long iova_p;
	phys_addr_t pa_p;
	u32 size_p;
	int prot = IOMMU_READ | IOMMU_WRITE;
	struct ipa_smmu_cb_ctx *cb = &ipa_mpm_ctx->carved_smmu_cb;
	unsigned long carved_iova = roundup(cb->next_addr, IPA_MPM_PAGE_SIZE);
	int ret = 0;

	/* check cache coherent */
	if (ipa_mpm_ctx->dev_info.is_cache_coherent)  {
		IPA_MPM_DBG(" enable cache coherent\n");
		prot |= IOMMU_CACHE;
	}

	if (carved_iova >= cb->va_end) {
		IPA_MPM_ERR("running out of carved_iova %lx\n", carved_iova);
		ipa_assert();
	}
	/*
	 * Both Host IPA and PCIE SMMU should be enabled or disabled
	 * for proceed.
	 * If SMMU Enabled => iova == pa
	 * If SMMU Disabled => iova == iommu mapped iova
	 * dma_map_single ensures cache is flushed and the memory is not
	 * touched again until dma_unmap_single() is called
	 */
	smmu_enabled = (ipa_mpm_ctx->dev_info.ipa_smmu_enabled &&
		ipa_mpm_ctx->dev_info.pcie_smmu_enabled) ? 1 : 0;

	if (smmu_enabled) {
		/* Map the phys addr to both PCIE and IPA AP CB
		 * from the carved out common iova range.
		 */
		ipa_smmu_domain = ipa3_get_smmu_domain();

		if (!ipa_smmu_domain) {
			IPA_MPM_ERR("invalid IPA smmu domain\n");
			ipa_assert();
		}

		if (!ipa_mpm_ctx->mhi_parent_dev) {
			IPA_MPM_ERR("invalid PCIE SMMU domain\n");
			ipa_assert();
		}

		phys_addr = virt_to_phys((void *) va_addr);
		IPA_SMMU_ROUND_TO_PAGE(carved_iova, phys_addr, sz,
					iova_p, pa_p, size_p);

		/* Flush the cache with dma_map_single for IPA AP CB */
		*ap_cb_iova = dma_map_single(ipa3_ctx->pdev, va_addr,
						IPA_MPM_RING_TOTAL_SIZE, dir);
		ret = ipa3_iommu_map(ipa_smmu_domain, iova_p,
					pa_p, size_p, prot);
		if (ret) {
			IPA_MPM_ERR("IPA IOMMU returned failure, ret = %d\n",
					ret);
			ipa_assert();
		}

		pcie_smmu_domain = iommu_get_domain_for_dev(
			ipa_mpm_ctx->mhi_parent_dev);
		if (!pcie_smmu_domain) {
			IPA_MPM_ERR("invalid pcie smmu domain\n");
			ipa_assert();
		}
		ret = iommu_map(pcie_smmu_domain, iova_p, pa_p, size_p, prot);

		if (ret) {
			IPA_MPM_ERR("PCIe IOMMU returned failure, ret = %d\n",
				ret);
			ipa_assert();
		}

		iova = iova_p;
		cb->next_addr = iova_p + size_p;
	} else {
		iova = dma_map_single(ipa3_ctx->pdev, va_addr,
					IPA_MPM_RING_TOTAL_SIZE, dir);
		*ap_cb_iova = iova;
	}
	return iova;
}

/**
 * ipa_mpm_smmu_unmap() - SMMU unmaps ring and the buffer pointer.
 * @va_addr: virtual address that needs to be mapped
 * @sz: size of the address to be mapped
 * @dir: ipa_mpm_dma_data_direction
 * @ap_cb_iova: iova for AP context bank
 *
 * This function SMMU unmaps both ring and the buffer pointer.
 * The ring pointers will be aligned to ring size and
 * the buffer pointers should be aligned to buffer size.
 *
 * Return: none
 */
static void ipa_mpm_smmu_unmap(dma_addr_t carved_iova, int sz, int dir,
	dma_addr_t ap_cb_iova)
{
	unsigned long iova_p;
	unsigned long pa_p;
	u32 size_p = 0;
	struct iommu_domain *ipa_smmu_domain, *pcie_smmu_domain;
	struct ipa_smmu_cb_ctx *cb = &ipa_mpm_ctx->carved_smmu_cb;
	int smmu_enabled = (ipa_mpm_ctx->dev_info.ipa_smmu_enabled &&
		ipa_mpm_ctx->dev_info.pcie_smmu_enabled) ? 1 : 0;

	if (carved_iova <= 0) {
		IPA_MPM_ERR("carved_iova is zero/negative\n");
		WARN_ON(1);
		return;
	}

	if (smmu_enabled) {
		ipa_smmu_domain = ipa3_get_smmu_domain();
		if (!ipa_smmu_domain) {
			IPA_MPM_ERR("invalid IPA smmu domain\n");
			ipa_assert();
		}

		if (!ipa_mpm_ctx->mhi_parent_dev) {
			IPA_MPM_ERR("invalid PCIE SMMU domain\n");
			ipa_assert();
		}

		IPA_SMMU_ROUND_TO_PAGE(carved_iova, carved_iova, sz,
			iova_p, pa_p, size_p);
		pcie_smmu_domain = iommu_get_domain_for_dev(
			ipa_mpm_ctx->mhi_parent_dev);
		if (pcie_smmu_domain) {
			iommu_unmap(pcie_smmu_domain, iova_p, size_p);
		} else {
			IPA_MPM_ERR("invalid PCIE SMMU domain\n");
			ipa_assert();
		}
		iommu_unmap(ipa_smmu_domain, iova_p, size_p);

		cb->next_addr -= size_p;
		dma_unmap_single(ipa3_ctx->pdev, ap_cb_iova,
			IPA_MPM_RING_TOTAL_SIZE, dir);
	} else {
		dma_unmap_single(ipa3_ctx->pdev, ap_cb_iova,
			IPA_MPM_RING_TOTAL_SIZE, dir);
	}
}

static u32 ipa_mpm_smmu_map_doorbell(enum mhip_smmu_domain_type smmu_domain,
	u32 pa_addr)
{
	/*
	 * Doorbells are already in PA, map these to
	 * PCIE/IPA doman if SMMUs are enabled.
	 */
	struct iommu_domain *ipa_smmu_domain, *pcie_smmu_domain;
	int smmu_enabled;
	unsigned long iova_p;
	phys_addr_t pa_p;
	u32 size_p;
	int ret = 0;
	int prot = IOMMU_READ | IOMMU_WRITE;
	struct ipa_smmu_cb_ctx *cb = &ipa_mpm_ctx->carved_smmu_cb;
	unsigned long carved_iova = roundup(cb->next_addr, IPA_MPM_PAGE_SIZE);
	u32 iova = 0;
	u64 offset = 0;

	/* check cache coherent */
	if (ipa_mpm_ctx->dev_info.is_cache_coherent)  {
		IPA_MPM_DBG(" enable cache coherent\n");
		prot |= IOMMU_CACHE;
	}

	if (carved_iova >= cb->va_end) {
		IPA_MPM_ERR("running out of carved_iova %lx\n", carved_iova);
		ipa_assert();
	}

	smmu_enabled = (ipa_mpm_ctx->dev_info.ipa_smmu_enabled &&
		ipa_mpm_ctx->dev_info.pcie_smmu_enabled) ? 1 : 0;

	if (smmu_enabled) {
		IPA_SMMU_ROUND_TO_PAGE(carved_iova, pa_addr, IPA_MPM_PAGE_SIZE,
					iova_p, pa_p, size_p);
		if (smmu_domain == MHIP_SMMU_DOMAIN_IPA) {
			ipa_smmu_domain = ipa3_get_smmu_domain();
			if (!ipa_smmu_domain) {
				IPA_MPM_ERR("invalid IPA smmu domain\n");
				ipa_assert();
			}
			ret = ipa3_iommu_map(ipa_smmu_domain,
				iova_p, pa_p, size_p, prot);
			if (ret) {
				IPA_MPM_ERR("IPA doorbell mapping failed\n");
				ipa_assert();
			}
			offset = pa_addr - pa_p;
		} else if (smmu_domain == MHIP_SMMU_DOMAIN_PCIE) {
			pcie_smmu_domain = iommu_get_domain_for_dev(
				ipa_mpm_ctx->mhi_parent_dev);
			if (!pcie_smmu_domain) {
				IPA_MPM_ERR("invalid IPA smmu domain\n");
				ipa_assert();
			}
			ret = iommu_map(pcie_smmu_domain,
					iova_p, pa_p, size_p, prot);
			if (ret) {
				IPA_MPM_ERR("PCIe doorbell mapping failed\n");
				ipa_assert();
			}
			offset = pa_addr - pa_p;
		}
		iova = iova_p + offset;
		cb->next_addr = iova_p + IPA_MPM_PAGE_SIZE;
	} else {
		iova = pa_addr;
	}
	return iova;
}

static void ipa_mpm_smmu_unmap_doorbell(enum mhip_smmu_domain_type smmu_domain,
	dma_addr_t iova)
{
	/*
	 * Doorbells are already in PA, map these to
	 * PCIE/IPA doman if SMMUs are enabled.
	 */
	struct iommu_domain *ipa_smmu_domain, *pcie_smmu_domain;
	int smmu_enabled;
	unsigned long iova_p;
	phys_addr_t pa_p;
	u32 size_p;
	struct ipa_smmu_cb_ctx *cb = &ipa_mpm_ctx->carved_smmu_cb;

	smmu_enabled = (ipa_mpm_ctx->dev_info.ipa_smmu_enabled &&
		ipa_mpm_ctx->dev_info.pcie_smmu_enabled) ? 1 : 0;

	if (smmu_enabled) {
		IPA_SMMU_ROUND_TO_PAGE(iova, iova, IPA_MPM_PAGE_SIZE,
					iova_p, pa_p, size_p);
		if (smmu_domain == MHIP_SMMU_DOMAIN_IPA) {
			ipa_smmu_domain = ipa3_get_smmu_domain();
			if (ipa_smmu_domain) {
				iommu_unmap(ipa_smmu_domain, iova_p, size_p);
			} else {
				IPA_MPM_ERR("invalid IPA smmu domain\n");
				ipa_assert();
			}
		} else if (smmu_domain == MHIP_SMMU_DOMAIN_PCIE) {
			pcie_smmu_domain = iommu_get_domain_for_dev(
				ipa_mpm_ctx->mhi_parent_dev);
			if (pcie_smmu_domain) {
				iommu_unmap(pcie_smmu_domain, iova_p, size_p);
			} else {
				IPA_MPM_ERR("invalid PCIE smmu domain\n");
				ipa_assert();
			}
			cb->next_addr -=  IPA_MPM_PAGE_SIZE;
		}
	}
}
static int get_idx_from_id(const struct mhi_device_id *id)
{
	return (id - mhi_driver_match_table);
}

static void get_ipa3_client(int id,
	enum ipa_client_type *ul_prod,
	enum ipa_client_type *dl_cons)
{
	IPA_MPM_FUNC_ENTRY();

	if (id >= IPA_MPM_MHIP_CH_ID_MAX) {
		*ul_prod = IPA_CLIENT_MAX;
		*dl_cons = IPA_CLIENT_MAX;
	} else {
		*ul_prod = ipa_mpm_pipes[id].ul_prod.ipa_client;
		*dl_cons = ipa_mpm_pipes[id].dl_cons.ipa_client;
	}
	IPA_MPM_FUNC_EXIT();
}

static int ipa_mpm_connect_mhip_gsi_pipe(enum ipa_client_type mhip_client,
	int mhi_idx, struct ipa_req_chan_out_params *out_params)
{
	int ipa_ep_idx;
	int res;
	struct mhi_p_desc *ev_ring;
	struct mhi_p_desc *tr_ring;
	int tr_ring_sz, ev_ring_sz;
	dma_addr_t ev_ring_iova, tr_ring_iova;
	dma_addr_t ap_cb_iova;
	dma_addr_t ap_cb_er_iova;
	struct ipa_request_gsi_channel_params gsi_params;
	int dir;
	int i;
	void *buff;
	int result;
	int k;
	struct ipa3_ep_context *ep;

	if (mhip_client == IPA_CLIENT_MAX)
		goto fail_gen;

	if ((mhi_idx < IPA_MPM_MHIP_CH_ID_0) ||
		(mhi_idx >= IPA_MPM_MHIP_CH_ID_MAX))
		goto fail_gen;

	ipa_ep_idx = ipa3_get_ep_mapping(mhip_client);
	if (ipa_ep_idx == IPA_EP_NOT_ALLOCATED) {
		IPA_MPM_ERR("fail to find channel EP.\n");
		goto fail_gen;
	}
	ep = &ipa3_ctx->ep[ipa_ep_idx];
	if (ep->valid == 1) {
		IPAERR("EP %d already allocated.\n", ipa_ep_idx);
		return 0;
	}

	IPA_MPM_DBG("connecting client %d (ep: %d)\n", mhip_client, ipa_ep_idx);

	IPA_MPM_FUNC_ENTRY();

	ev_ring_sz = IPA_MPM_RING_TOTAL_SIZE;
	ev_ring = kzalloc(ev_ring_sz, GFP_KERNEL);
	if (!ev_ring)
		goto fail_evt_alloc;

	tr_ring_sz = IPA_MPM_RING_TOTAL_SIZE;
	tr_ring = kzalloc(tr_ring_sz, GFP_KERNEL);
	if (!tr_ring)
		goto fail_tr_alloc;

	tr_ring[0].re_type = MHIP_RE_NOP;

	dir = IPA_CLIENT_IS_PROD(mhip_client) ?
		DMA_TO_HIPA : DMA_FROM_HIPA;

	/* allocate transfer ring elements */
	for (i = 1, k = 1; i < IPA_MPM_RING_LEN; i++, k++) {
		buff = kzalloc(TRE_BUFF_SIZE, GFP_KERNEL);

		if (!buff)
			goto fail_buff_alloc;

		tr_ring[i].buffer_ptr =
			ipa_mpm_smmu_map(buff, TRE_BUFF_SIZE, dir,
				&ap_cb_iova);
		if (!tr_ring[i].buffer_ptr)
			goto fail_smmu_map_ring;

		if (IPA_CLIENT_IS_PROD(mhip_client)) {
			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.tre_buff_va[k] =
							buff;
			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.tre_buff_iova[k] =
							tr_ring[i].buffer_ptr;
		} else {
			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.tre_buff_va[k] =
							buff;
			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.tre_buff_iova[k] =
							tr_ring[i].buffer_ptr;
		}


		tr_ring[i].buff_len = TRE_BUFF_SIZE;
		tr_ring[i].chain = 0;
		tr_ring[i].ieob = 0;
		tr_ring[i].ieot = 0;
		tr_ring[i].bei = 0;
		tr_ring[i].sct = 0;
		tr_ring[i].re_type = MHIP_RE_XFER;

		if (IPA_CLIENT_IS_PROD(mhip_client))
			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.ap_iova_buff[k] =
				ap_cb_iova;
		else
			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.ap_iova_buff[k] =
				ap_cb_iova;
	}

	tr_ring_iova = ipa_mpm_smmu_map(tr_ring, IPA_MPM_PAGE_SIZE, dir,
		&ap_cb_iova);
	if (!tr_ring_iova)
		goto fail_smmu_map_ring;

	ev_ring_iova = ipa_mpm_smmu_map(ev_ring, IPA_MPM_PAGE_SIZE, dir,
		&ap_cb_er_iova);
	if (!ev_ring_iova)
		goto fail_smmu_map_ring;

	/* Store Producer channel rings */
	if (IPA_CLIENT_IS_PROD(mhip_client)) {
		/* Device UL */
		ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.er_va = ev_ring;
		ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.tr_va = tr_ring;
		ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.er_pa = ev_ring_iova;
		ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.tr_pa = tr_ring_iova;
		ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.ap_iova_tr =
			ap_cb_iova;
		ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.ap_iova_er =
			ap_cb_er_iova;
	} else {
		/* Host UL */
		ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.er_va = ev_ring;
		ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.tr_va = tr_ring;
		ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.er_pa = ev_ring_iova;
		ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.tr_pa = tr_ring_iova;
		ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.ap_iova_tr =
			ap_cb_iova;
		ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.ap_iova_er =
			ap_cb_er_iova;
	}

	memset(&gsi_params, 0, sizeof(struct ipa_request_gsi_channel_params));

	if (IPA_CLIENT_IS_PROD(mhip_client))
		gsi_params.ipa_ep_cfg =
		ipa_mpm_pipes[mhi_idx].dl_cons.ep_cfg;
	else
		gsi_params.ipa_ep_cfg =
		ipa_mpm_pipes[mhi_idx].ul_prod.ep_cfg;

	gsi_params.client = mhip_client;
	gsi_params.skip_ep_cfg = false;

	/*
	 * RP update address = Device channel DB address
	 * CLIENT_PROD -> Host DL
	 * CLIENT_CONS -> Host UL
	 */
	if (IPA_CLIENT_IS_PROD(mhip_client)) {
		gsi_params.evt_ring_params.rp_update_addr =
			ipa_mpm_smmu_map_doorbell(
			MHIP_SMMU_DOMAIN_IPA,
			ipa_mpm_ctx->md[mhi_idx].dl_cons.chan_props.device_db);
		if (gsi_params.evt_ring_params.rp_update_addr == 0)
			goto fail_smmu_map_db;

		ipa_mpm_ctx->md[mhi_idx].dl_cons.db_host_iova =
			gsi_params.evt_ring_params.rp_update_addr;

		gsi_params.evt_ring_params.ring_base_addr =
			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.tr_pa;
		gsi_params.chan_params.ring_base_addr =
			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.er_pa;
	} else {
		gsi_params.evt_ring_params.rp_update_addr =
			ipa_mpm_smmu_map_doorbell(
			MHIP_SMMU_DOMAIN_IPA,
			ipa_mpm_ctx->md[mhi_idx].ul_prod.chan_props.device_db);
		if (gsi_params.evt_ring_params.rp_update_addr == 0)
			goto fail_smmu_map_db;
		ipa_mpm_ctx->md[mhi_idx].ul_prod.db_host_iova =
			gsi_params.evt_ring_params.rp_update_addr;
		gsi_params.evt_ring_params.ring_base_addr =
			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.er_pa;
		gsi_params.chan_params.ring_base_addr =
			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.tr_pa;
	}

	/* Fill Event ring params */
	gsi_params.evt_ring_params.intf = GSI_EVT_CHTYPE_MHIP_EV;
	gsi_params.evt_ring_params.intr = GSI_INTR_MSI;
	gsi_params.evt_ring_params.re_size = GSI_EVT_RING_RE_SIZE_16B;
	gsi_params.evt_ring_params.ring_len =
		(IPA_MPM_RING_LEN) * GSI_EVT_RING_RE_SIZE_16B;
	gsi_params.evt_ring_params.ring_base_vaddr = NULL;
	gsi_params.evt_ring_params.int_modt = 0;
	gsi_params.evt_ring_params.int_modc = 0;
	gsi_params.evt_ring_params.intvec = 0;
	gsi_params.evt_ring_params.msi_addr = 0;
	gsi_params.evt_ring_params.exclusive = true;
	gsi_params.evt_ring_params.err_cb = ipa_mpm_gsi_evt_ring_err_cb;
	gsi_params.evt_ring_params.user_data = NULL;

	/* Evt Scratch Params */
	/* Disable the Moderation for ringing doorbells */
	gsi_params.evt_scratch.mhip.rp_mod_threshold = 1;
	gsi_params.evt_scratch.mhip.rp_mod_timer = 0;
	gsi_params.evt_scratch.mhip.rp_mod_counter = 0;
	gsi_params.evt_scratch.mhip.rp_mod_timer_id = 0;
	gsi_params.evt_scratch.mhip.rp_mod_timer_running = 0;
	gsi_params.evt_scratch.mhip.fixed_buffer_sz = TRE_BUFF_SIZE;

	if (IPA_CLIENT_IS_PROD(mhip_client))
		gsi_params.evt_scratch.mhip.rp_mod_threshold = 4;

	/* Channel Params */
	gsi_params.chan_params.prot = GSI_CHAN_PROT_MHIP;
	gsi_params.chan_params.dir = IPA_CLIENT_IS_PROD(mhip_client) ?
		GSI_CHAN_DIR_TO_GSI : GSI_CHAN_DIR_FROM_GSI;
	/* chan_id is set in ipa3_request_gsi_channel() */
	gsi_params.chan_params.re_size = GSI_CHAN_RE_SIZE_16B;
	gsi_params.chan_params.ring_len =
		(IPA_MPM_RING_LEN) * GSI_EVT_RING_RE_SIZE_16B;
	gsi_params.chan_params.ring_base_vaddr = NULL;
	gsi_params.chan_params.use_db_eng = GSI_CHAN_DIRECT_MODE;
	gsi_params.chan_params.max_prefetch = GSI_ONE_PREFETCH_SEG;
	gsi_params.chan_params.low_weight = 1;
	gsi_params.chan_params.xfer_cb = NULL;
	gsi_params.chan_params.err_cb = ipa_mpm_gsi_chan_err_cb;
	gsi_params.chan_params.chan_user_data = NULL;

	/* Channel scratch */
	gsi_params.chan_scratch.mhip.assert_bit_40 = 0;
	gsi_params.chan_scratch.mhip.host_channel = 1;

	res = ipa3_request_gsi_channel(&gsi_params, out_params);
	if (res) {
		IPA_MPM_ERR("failed to allocate GSI channel res=%d\n", res);
		goto fail_alloc_channel;
	}

	if (IPA_CLIENT_IS_PROD(mhip_client))
		ipa_mpm_change_gsi_state(mhi_idx,
			IPA_MPM_MHIP_CHAN_DL,
			GSI_ALLOCATED);
	else
		ipa_mpm_change_gsi_state(mhi_idx,
			IPA_MPM_MHIP_CHAN_UL,
			GSI_ALLOCATED);
	result = ipa3_start_gsi_channel(ipa_ep_idx);
	if (result) {
		IPA_MPM_ERR("start MHIP channel %d failed\n", mhip_client);
		if (IPA_CLIENT_IS_PROD(mhip_client))
			ipa_mpm_change_gsi_state(mhi_idx,
				IPA_MPM_MHIP_CHAN_DL, GSI_ERR);
		else
			ipa_mpm_change_gsi_state(mhi_idx,
				IPA_MPM_MHIP_CHAN_UL, GSI_ERR);
		goto fail_start_channel;
	}
	if (IPA_CLIENT_IS_PROD(mhip_client))
		ipa_mpm_change_gsi_state(mhi_idx,
			IPA_MPM_MHIP_CHAN_DL, GSI_STARTED);
	else
		ipa_mpm_change_gsi_state(mhi_idx,
			IPA_MPM_MHIP_CHAN_UL, GSI_STARTED);

	/* Fill in the Device Context params */
	if (IPA_CLIENT_IS_PROD(mhip_client)) {
		/* This is the DL channel :: Device -> Host */
		ipa_mpm_ctx->md[mhi_idx].dl_cons.evt_props.ev_ctx.rbase =
			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.er_pa;
		ipa_mpm_ctx->md[mhi_idx].dl_cons.chan_props.ch_ctx.rbase =
			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.tr_pa;
	} else {
		ipa_mpm_ctx->md[mhi_idx].ul_prod.evt_props.ev_ctx.rbase =
			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.tr_pa;
		ipa_mpm_ctx->md[mhi_idx].ul_prod.chan_props.ch_ctx.rbase =
			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.er_pa;
	}

	IPA_MPM_FUNC_EXIT();

	return 0;

fail_start_channel:
	ipa3_disable_data_path(ipa_ep_idx);
	ipa3_stop_gsi_channel(ipa_ep_idx);
fail_alloc_channel:
	ipa3_release_gsi_channel(ipa_ep_idx);
fail_smmu_map_db:
fail_smmu_map_ring:
fail_tr_alloc:
fail_evt_alloc:
fail_buff_alloc:
	ipa_assert();
fail_gen:
	return -EFAULT;
}

static void ipa_mpm_clean_mhip_chan(int mhi_idx,
	enum ipa_client_type mhip_client)
{
	int dir;
	int i;
	int ipa_ep_idx;
	int result;

	IPA_MPM_FUNC_ENTRY();

	if (mhip_client == IPA_CLIENT_MAX)
		return;

	if ((mhi_idx < IPA_MPM_MHIP_CH_ID_0) ||
		(mhi_idx >= IPA_MPM_MHIP_CH_ID_MAX))
		return;

	dir = IPA_CLIENT_IS_PROD(mhip_client) ?
		DMA_TO_HIPA : DMA_FROM_HIPA;

	ipa_ep_idx = ipa3_get_ep_mapping(mhip_client);
	if (ipa_ep_idx == IPA_EP_NOT_ALLOCATED) {
		IPA_MPM_ERR("fail to find channel EP.\n");
		return;
	}

	/* For the uplink channels, enable HOLB. */
	if (IPA_CLIENT_IS_CONS(mhip_client))
		ipa3_disable_data_path(ipa_ep_idx);

	/* Release channel */
	result = ipa3_stop_gsi_channel(ipa_ep_idx);
	if (result) {
		IPA_MPM_ERR("Stop channel for MHIP_Client =  %d failed\n",
					mhip_client);
		goto fail_chan;
	}
	result = ipa3_reset_gsi_channel(ipa_ep_idx);
	if (result) {
		IPA_MPM_ERR("Reset channel for MHIP_Client =  %d failed\n",
					mhip_client);
		goto fail_chan;
	}
	result = ipa3_reset_gsi_event_ring(ipa_ep_idx);
	if (result) {
		IPA_MPM_ERR("Reset ev ring for MHIP_Client =  %d failed\n",
					mhip_client);
		goto fail_chan;
	}
	result = ipa3_release_gsi_channel(ipa_ep_idx);
	if (result) {
		IPA_MPM_ERR("Release tr ring for MHIP_Client =  %d failed\n",
					mhip_client);
		if (IPA_CLIENT_IS_PROD(mhip_client))
			ipa_mpm_change_gsi_state(mhi_idx,
				IPA_MPM_MHIP_CHAN_DL, GSI_ERR);
		else
			ipa_mpm_change_gsi_state(mhi_idx,
				IPA_MPM_MHIP_CHAN_UL, GSI_ERR);
		goto fail_chan;
	}

	if (IPA_CLIENT_IS_PROD(mhip_client))
		ipa_mpm_change_gsi_state(mhi_idx,
					IPA_MPM_MHIP_CHAN_DL, GSI_INIT);
	else
		ipa_mpm_change_gsi_state(mhi_idx,
					IPA_MPM_MHIP_CHAN_UL, GSI_INIT);

	memset(&ipa3_ctx->ep[ipa_ep_idx], 0, sizeof(struct ipa3_ep_context));

	/* Unmap Doorbells */
	if (IPA_CLIENT_IS_PROD(mhip_client)) {
		ipa_mpm_smmu_unmap_doorbell(MHIP_SMMU_DOMAIN_PCIE,
			ipa_mpm_ctx->md[mhi_idx].dl_cons.db_device_iova);

		ipa_mpm_smmu_unmap_doorbell(MHIP_SMMU_DOMAIN_IPA,
			ipa_mpm_ctx->md[mhi_idx].dl_cons.db_host_iova);

		ipa_mpm_ctx->md[mhi_idx].dl_cons.db_host_iova = 0;
		ipa_mpm_ctx->md[mhi_idx].dl_cons.db_device_iova = 0;

	} else {
		ipa_mpm_smmu_unmap_doorbell(MHIP_SMMU_DOMAIN_PCIE,
			ipa_mpm_ctx->md[mhi_idx].ul_prod.db_device_iova);

		ipa_mpm_smmu_unmap_doorbell(MHIP_SMMU_DOMAIN_IPA,
			ipa_mpm_ctx->md[mhi_idx].ul_prod.db_host_iova);

		ipa_mpm_ctx->md[mhi_idx].ul_prod.db_host_iova = 0;
		ipa_mpm_ctx->md[mhi_idx].ul_prod.db_device_iova = 0;
	}

	/* deallocate/Unmap transfer ring buffers */
	for (i = 1; i < IPA_MPM_RING_LEN; i++) {
		if (IPA_CLIENT_IS_PROD(mhip_client)) {
			ipa_mpm_smmu_unmap(
			(dma_addr_t)
			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.tre_buff_iova[i],
			TRE_BUFF_SIZE, dir,
			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.ap_iova_buff[i]);
			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.tre_buff_iova[i]
								= 0;
			kfree(
			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.tre_buff_va[i]);
			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.tre_buff_va[i]
								= NULL;
			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.ap_iova_buff[i]
								= 0;
			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.tre_buff_iova[i]
								= 0;
		} else {
			ipa_mpm_smmu_unmap(
			(dma_addr_t)
			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.tre_buff_iova[i],
			TRE_BUFF_SIZE, dir,
			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.ap_iova_buff[i]
			);
			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.tre_buff_iova[i]
								= 0;
			kfree(
			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.tre_buff_va[i]);
			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.tre_buff_va[i]
								= NULL;
			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.ap_iova_buff[i]
								= 0;
			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.tre_buff_iova[i]
								= 0;
		}
	}

	/* deallocate/Unmap rings */
	if (IPA_CLIENT_IS_PROD(mhip_client)) {
		ipa_mpm_smmu_unmap(
			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.er_pa,
			IPA_MPM_PAGE_SIZE, dir,
			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.ap_iova_er);

		ipa_mpm_smmu_unmap(
			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.tr_pa,
			IPA_MPM_PAGE_SIZE, dir,
			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.ap_iova_tr);

		kfree(ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.er_va);
		kfree(ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.tr_va);

		ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.er_va = NULL;
		ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.tr_va = NULL;
		ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.ap_iova_tr = 0;
		ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.ap_iova_er = 0;


	} else {
		ipa_mpm_smmu_unmap(
			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.tr_pa,
			IPA_MPM_PAGE_SIZE, dir,
			ipa_mpm_ctx->md[mhi_idx].dl_prod_ring.ap_iova_tr);
		ipa_mpm_smmu_unmap(
			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.er_pa,
			IPA_MPM_PAGE_SIZE, dir,
			ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.ap_iova_er);

		ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.tr_pa = 0;
		ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.er_pa = 0;

		kfree(ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.er_va);
		kfree(ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.tr_va);

		ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.er_va = NULL;
		ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.tr_va = NULL;
		ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.ap_iova_er = 0;
		ipa_mpm_ctx->md[mhi_idx].ul_prod_ring.ap_iova_tr = 0;
	}

	IPA_MPM_FUNC_EXIT();
	return;
fail_chan:
	ipa_assert();
}

/* round addresses for closest page per SMMU requirements */
static inline void ipa_mpm_smmu_round_to_page(uint64_t iova, uint64_t pa,
	uint64_t size, unsigned long *iova_p, phys_addr_t *pa_p, u32 *size_p)
{
	*iova_p = rounddown(iova, PAGE_SIZE);
	*pa_p = rounddown(pa, PAGE_SIZE);
	*size_p = roundup(size + pa - *pa_p, PAGE_SIZE);
}


static int __ipa_mpm_configure_mhi_device(struct ipa_mpm_channel *ch,
	int mhi_idx, int dir)
{
	struct mhi_buf ch_config[2];
	int ret;

	IPA_MPM_FUNC_ENTRY();

	if (ch == NULL) {
		IPA_MPM_ERR("ch config is NULL\n");
		return -EINVAL;
	}

	/* Populate CCA */
	ch_config[0].buf = &ch->chan_props.ch_ctx;
	ch_config[0].len = sizeof(ch->chan_props.ch_ctx);
	ch_config[0].name = "CCA";

	/* populate ECA */
	ch_config[1].buf = &ch->evt_props.ev_ctx;
	ch_config[1].len = sizeof(ch->evt_props.ev_ctx);
	ch_config[1].name = "ECA";

	IPA_MPM_DBG("Configuring MHI PRIME device for mhi_idx %d\n", mhi_idx);

	ret = mhi_device_configure(ipa_mpm_ctx->md[mhi_idx].mhi_dev, dir,
			ch_config, 2);
	if (ret) {
		IPA_MPM_ERR("mhi_device_configure failed\n");
		return -EINVAL;
	}
	IPA_MPM_FUNC_EXIT();
	return 0;
}

static void ipa_mpm_mhip_shutdown(int mhip_idx)
{
	enum ipa_client_type ul_prod_chan, dl_cons_chan;

	IPA_MPM_FUNC_ENTRY();

	get_ipa3_client(mhip_idx, &ul_prod_chan, &dl_cons_chan);

	if (mhip_idx != IPA_MPM_MHIP_CH_ID_2) {
		/* For DPL, stop only DL channel */
		ipa_mpm_start_stop_ul_mhip_data_path(mhip_idx, MPM_MHIP_STOP);
		ipa_mpm_clean_mhip_chan(mhip_idx, ul_prod_chan);
	}

	ipa_mpm_clean_mhip_chan(mhip_idx, dl_cons_chan);

	mutex_lock(&ipa_mpm_ctx->md[mhip_idx].lpm_mutex);
	if (!ipa_mpm_ctx->md[mhip_idx].in_lpm) {
		ipa_mpm_vote_unvote_ipa_clk(CLK_OFF, mhip_idx);
		/* while in modem shutdown scenarios such as SSR, no explicit
		 * PCIe vote is needed.
		 */
		ipa_mpm_ctx->md[mhip_idx].in_lpm = true;
	}
	mutex_unlock(&ipa_mpm_ctx->md[mhip_idx].lpm_mutex);
	ipa_mpm_ctx->md[mhip_idx].mhi_dev = NULL;
	ipa_mpm_ctx->md[mhip_idx].init_complete = false;
	IPA_MPM_FUNC_EXIT();
}

/*
 * Turning on/OFF PCIE Clock is done once for all clients.
 * Always vote for Probe_ID 0 as a standard.
 */
static int ipa_mpm_vote_unvote_pcie_clk(enum ipa_mpm_clk_vote_type vote,
	int probe_id)
{
	int result = 0;

	if (probe_id >= IPA_MPM_MHIP_CH_ID_MAX) {
		IPA_MPM_ERR("probe_id not found\n");
		return -EINVAL;
	}

	if (vote > CLK_OFF) {
		IPA_MPM_ERR("Invalid vote\n");
		return -EINVAL;
	}

	if (ipa_mpm_ctx->md[probe_id].mhi_dev == NULL) {
		IPA_MPM_ERR("MHI not initialized yet\n");
		return 0;
	}

	IPA_MPM_ERR("PCIe clock vote/unvote = %d probe_id = %d clk_cnt = %d\n",
		vote, probe_id,
		atomic_read(&ipa_mpm_ctx->md[probe_id].clk_cnt.pcie_clk_cnt));

	if (vote == CLK_ON) {
		result = mhi_device_get_sync(
			ipa_mpm_ctx->md[probe_id].mhi_dev, MHI_VOTE_BUS);
		if (result) {
			IPA_MPM_ERR("mhi_sync_get failed for probe_id %d\n",
				result, probe_id);
			return result;
		}

		IPA_MPM_DBG("probe_id %d PCIE clock now ON\n", probe_id);
		atomic_inc(&ipa_mpm_ctx->md[probe_id].clk_cnt.pcie_clk_cnt);
		atomic_inc(&ipa_mpm_ctx->pcie_clk_total_cnt);
	} else {
		if ((atomic_read(
			&ipa_mpm_ctx->md[probe_id].clk_cnt.pcie_clk_cnt)
								== 0)) {
			IPA_MPM_DBG("probe_id %d PCIE clock already devoted\n",
				probe_id);
			WARN_ON(1);
			return 0;
		}
		mhi_device_put(ipa_mpm_ctx->md[probe_id].mhi_dev, MHI_VOTE_BUS);
		IPA_MPM_DBG("probe_id %d PCIE clock off\n", probe_id);
		atomic_dec(&ipa_mpm_ctx->md[probe_id].clk_cnt.pcie_clk_cnt);
		atomic_dec(&ipa_mpm_ctx->pcie_clk_total_cnt);
	}

	return result;
}

/*
 * Turning on/OFF IPA Clock is done only once- for all clients
 */
static void ipa_mpm_vote_unvote_ipa_clk(enum ipa_mpm_clk_vote_type vote,
	int probe_id)
{
	if (vote > CLK_OFF)
		return;

	IPA_MPM_ERR("IPA clock vote/unvote = %d probe_id = %d clk_cnt = %d\n",
		vote, probe_id,
		atomic_read(&ipa_mpm_ctx->md[probe_id].clk_cnt.ipa_clk_cnt));

	if (vote == CLK_ON) {
		IPA_ACTIVE_CLIENTS_INC_SPECIAL(ipa_mpm_mhip_chan_str[probe_id]);
		IPA_MPM_DBG("IPA clock now ON for probe_id %d\n", probe_id);
		atomic_inc(&ipa_mpm_ctx->md[probe_id].clk_cnt.ipa_clk_cnt);
		atomic_inc(&ipa_mpm_ctx->ipa_clk_total_cnt);
	} else {
		if ((atomic_read
			(&ipa_mpm_ctx->md[probe_id].clk_cnt.ipa_clk_cnt)
								== 0)) {
			IPA_MPM_DBG("probe_id %d IPA clock count < 0\n",
				probe_id);
			WARN_ON(1);
			return;
		}
		IPA_ACTIVE_CLIENTS_DEC_SPECIAL(ipa_mpm_mhip_chan_str[probe_id]);
		IPA_MPM_DBG("probe_id %d IPA clock off\n", probe_id);
		atomic_dec(&ipa_mpm_ctx->md[probe_id].clk_cnt.ipa_clk_cnt);
		atomic_dec(&ipa_mpm_ctx->ipa_clk_total_cnt);
	}
}

static enum mhip_status_type ipa_mpm_start_stop_mhip_chan(
	enum ipa_mpm_mhip_chan mhip_chan,
	int probe_id,
	enum ipa_mpm_start_stop_type start_stop)
{
	int ipa_ep_idx;
	struct ipa3_ep_context *ep;
	bool is_start;
	enum ipa_client_type ul_chan, dl_chan;
	u32 source_pipe_bitmask = 0;
	enum gsi_status gsi_res = GSI_STATUS_SUCCESS;
	int result;

	IPA_MPM_FUNC_ENTRY();

	if (mhip_chan > IPA_MPM_MHIP_CHAN_BOTH) {
		IPA_MPM_ERR("MHI not initialized yet\n");
		return MHIP_STATUS_FAIL;
	}

	if (probe_id >= IPA_MPM_MHIP_CH_ID_MAX) {
		IPA_MPM_ERR("MHI not initialized yet\n");
		return MHIP_STATUS_FAIL;
	}

	get_ipa3_client(probe_id, &ul_chan, &dl_chan);

	if (mhip_chan == IPA_MPM_MHIP_CHAN_UL) {
		ipa_ep_idx = ipa3_get_ep_mapping(ul_chan);
	} else if (mhip_chan == IPA_MPM_MHIP_CHAN_DL) {
		ipa_ep_idx = ipa3_get_ep_mapping(dl_chan);
	} else if (mhip_chan == IPA_MPM_MHIP_CHAN_BOTH) {
		ipa_ep_idx = ipa3_get_ep_mapping(ul_chan);
		ipa_ep_idx = ipa3_get_ep_mapping(dl_chan);
	}

	if (ipa_ep_idx == IPA_EP_NOT_ALLOCATED) {
		IPA_MPM_ERR("fail to get EP# for idx %d\n", ipa_ep_idx);
		return MHIP_STATUS_EP_NOT_FOUND;
	}
	ep = &ipa3_ctx->ep[ipa_ep_idx];

	if (mhip_chan == IPA_MPM_MHIP_CHAN_UL) {
		IPA_MPM_DBG("current GSI state = %d, action = %d\n",
			ipa_mpm_ctx->md[probe_id].ul_prod.gsi_state,
			start_stop);
		if (ipa_mpm_ctx->md[probe_id].ul_prod.gsi_state <
			GSI_ALLOCATED) {
			IPA_MPM_ERR("GSI chan is not allocated yet\n");
			return MHIP_STATUS_EP_NOT_READY;
		}
	} else if (mhip_chan == IPA_MPM_MHIP_CHAN_DL) {
		IPA_MPM_DBG("current GSI state = %d, action = %d\n",
			ipa_mpm_ctx->md[probe_id].dl_cons.gsi_state,
			start_stop);
		if (ipa_mpm_ctx->md[probe_id].dl_cons.gsi_state <
			GSI_ALLOCATED) {
			IPA_MPM_ERR("GSI chan is not allocated yet\n");
			return MHIP_STATUS_EP_NOT_READY;
		}
	}

	is_start = (start_stop == MPM_MHIP_START) ? true : false;

	if (is_start) {
		if (mhip_chan == IPA_MPM_MHIP_CHAN_UL) {
			if (ipa_mpm_ctx->md[probe_id].ul_prod.gsi_state ==
				GSI_STARTED) {
				IPA_MPM_ERR("GSI chan is already started\n");
				return MHIP_STATUS_NO_OP;
			}
		}

		if (mhip_chan == IPA_MPM_MHIP_CHAN_DL) {
			if (ipa_mpm_ctx->md[probe_id].dl_cons.gsi_state ==
				GSI_STARTED) {
				IPA_MPM_ERR("GSI chan is already started\n");
				return MHIP_STATUS_NO_OP;
			}
		}
		/* Start GSI channel */
		gsi_res = ipa3_start_gsi_channel(ipa_ep_idx);
		if (gsi_res != GSI_STATUS_SUCCESS) {
			IPA_MPM_ERR("Error starting channel: err = %d\n",
					gsi_res);
			goto gsi_chan_fail;
		} else {
			ipa_mpm_change_gsi_state(probe_id, mhip_chan,
					GSI_STARTED);
		}
	} else {
		if (mhip_chan == IPA_MPM_MHIP_CHAN_UL) {
			if (ipa_mpm_ctx->md[probe_id].ul_prod.gsi_state ==
				GSI_STOPPED) {
				IPA_MPM_ERR("GSI chan is already stopped\n");
				return MHIP_STATUS_NO_OP;
			} else if (ipa_mpm_ctx->md[probe_id].ul_prod.gsi_state
				!= GSI_STARTED) {
				IPA_MPM_ERR("GSI chan isn't already started\n");
				return MHIP_STATUS_NO_OP;
			}
		}

		if (mhip_chan == IPA_MPM_MHIP_CHAN_DL) {
			if (ipa_mpm_ctx->md[probe_id].dl_cons.gsi_state ==
				GSI_STOPPED) {
				IPA_MPM_ERR("GSI chan is already stopped\n");
				return MHIP_STATUS_NO_OP;
			} else if (ipa_mpm_ctx->md[probe_id].dl_cons.gsi_state
				!= GSI_STARTED) {
				IPA_MPM_ERR("GSI chan isn't already started\n");
				return MHIP_STATUS_NO_OP;
			}
		}

		if (mhip_chan == IPA_MPM_MHIP_CHAN_UL) {
			source_pipe_bitmask = 1 <<
				ipa3_get_ep_mapping(ep->client);
			/* First Stop UL GSI channel before unvote PCIe clock */
			result = ipa3_stop_gsi_channel(ipa_ep_idx);

			if (result) {
				IPA_MPM_ERR("UL chan stop failed\n");
				goto gsi_chan_fail;
			} else {
				ipa_mpm_change_gsi_state(probe_id, mhip_chan,
							GSI_STOPPED);
			}
		}

		if (mhip_chan == IPA_MPM_MHIP_CHAN_DL) {
			result = ipa3_stop_gsi_channel(ipa_ep_idx);
			if (result) {
				IPA_MPM_ERR("Fail to stop DL channel\n");
				goto gsi_chan_fail;
			} else {
				ipa_mpm_change_gsi_state(probe_id, mhip_chan,
							GSI_STOPPED);
			}
		}
	}
	IPA_MPM_FUNC_EXIT();

	return MHIP_STATUS_SUCCESS;
gsi_chan_fail:
	ipa3_disable_data_path(ipa_ep_idx);
	ipa_mpm_change_gsi_state(probe_id, mhip_chan, GSI_ERR);
	ipa_assert();
	return MHIP_STATUS_FAIL;
}

int ipa_mpm_notify_wan_state(struct wan_ioctl_notify_wan_state *state)
{
	int probe_id = IPA_MPM_MHIP_CH_ID_MAX;
	int i;
	static enum mhip_status_type status;
	int ret = 0;
	enum ipa_mpm_mhip_client_type mhip_client = IPA_MPM_MHIP_TETH;

	if (!state)
		return -EPERM;

	if (!ipa3_is_mhip_offload_enabled())
		return -EPERM;

	for (i = 0; i < IPA_MPM_MHIP_CH_ID_MAX; i++) {
		if (ipa_mpm_pipes[i].mhip_client == mhip_client) {
			probe_id = i;
			break;
		}
	}

	if (probe_id == IPA_MPM_MHIP_CH_ID_MAX) {
		IPA_MPM_ERR("Unknown probe_id\n");
		return -EPERM;
	}

	IPA_MPM_DBG("WAN backhaul available for probe_id = %d\n", probe_id);

	if (state->up) {
		/* Start UL MHIP channel for offloading tethering connection */
		ret = ipa_mpm_vote_unvote_pcie_clk(CLK_ON, probe_id);
		if (ret) {
			IPA_MPM_ERR("Error cloking on PCIe clk, err = %d\n",
				ret);
			return ret;
		}
		status = ipa_mpm_start_stop_mhip_chan(
				IPA_MPM_MHIP_CHAN_UL, probe_id, MPM_MHIP_START);
		switch (status) {
		case MHIP_STATUS_SUCCESS:
			ipa_mpm_ctx->md[probe_id].teth_state =
						IPA_MPM_TETH_CONNECTED;
			ret = ipa_mpm_start_stop_ul_mhip_data_path(
						probe_id, MPM_MHIP_START);
			if (ret) {
				IPA_MPM_ERR("err UL chan start\n");
				ipa_mpm_vote_unvote_pcie_clk(CLK_OFF, probe_id);
				return ret;
			}
			break;
		case MHIP_STATUS_EP_NOT_READY:
		case MHIP_STATUS_NO_OP:
			IPA_MPM_DBG("UL chan already start, status = %d\n",
					status);
			ret = ipa_mpm_vote_unvote_pcie_clk(CLK_OFF, probe_id);
			return ret;
		case MHIP_STATUS_FAIL:
		case MHIP_STATUS_BAD_STATE:
		case MHIP_STATUS_EP_NOT_FOUND:
			IPA_MPM_ERR("UL chan start err =%d\n", status);
			ret = ipa_mpm_vote_unvote_pcie_clk(CLK_OFF, probe_id);
			ipa_assert();
			return -EFAULT;
		default:
			IPA_MPM_ERR("Err not found\n");
			ipa_mpm_vote_unvote_pcie_clk(CLK_OFF, probe_id);
			ret = -EFAULT;
			break;
		}
		ipa_mpm_ctx->md[probe_id].mhip_client = mhip_client;
	} else {
		status = ipa_mpm_start_stop_mhip_chan(
					IPA_MPM_MHIP_CHAN_UL, probe_id,
					MPM_MHIP_STOP);
		switch (status) {
		case MHIP_STATUS_SUCCESS:
			ipa_mpm_change_teth_state(probe_id, IPA_MPM_TETH_INIT);
			ipa_mpm_start_stop_ul_mhip_data_path(probe_id,
							MPM_MHIP_STOP);
			break;
		case MHIP_STATUS_NO_OP:
		case MHIP_STATUS_EP_NOT_READY:
			IPA_MPM_DBG("UL chan already stop, status = %d\n",
					status);
			break;
		case MHIP_STATUS_FAIL:
		case MHIP_STATUS_BAD_STATE:
		case MHIP_STATUS_EP_NOT_FOUND:
			IPA_MPM_ERR("UL chan cant be stopped err =%d\n",
				status);
			ipa_assert();
			return -EFAULT;
		default:
			IPA_MPM_ERR("Err not found\n");
			return -EFAULT;
		}
		/* Stop UL MHIP channel for offloading tethering connection */
		ret = ipa_mpm_vote_unvote_pcie_clk(CLK_OFF, probe_id);

		if (ret) {
			IPA_MPM_ERR("Error cloking on PCIe clk, err = %d\n",
				ret);
			return ret;
		}
		ipa_mpm_ctx->md[probe_id].mhip_client = IPA_MPM_MHIP_NONE;
	}
	return ret;
}

static void ipa_mpm_change_gsi_state(int probe_id,
	enum ipa_mpm_mhip_chan mhip_chan,
	enum ipa_mpm_gsi_state next_state)
{

	if (probe_id >= IPA_MPM_MHIP_CH_ID_MAX)
		return;

	if (mhip_chan == IPA_MPM_MHIP_CHAN_UL) {
		mutex_lock(&ipa_mpm_ctx->md[probe_id].mutex);
		ipa_mpm_ctx->md[probe_id].ul_prod.gsi_state = next_state;
		IPA_MPM_DBG("GSI next_state = %d\n",
			ipa_mpm_ctx->md[probe_id].ul_prod.gsi_state);
		 mutex_unlock(&ipa_mpm_ctx->md[probe_id].mutex);
	}

	if (mhip_chan == IPA_MPM_MHIP_CHAN_DL) {
		mutex_lock(&ipa_mpm_ctx->md[probe_id].mutex);
		ipa_mpm_ctx->md[probe_id].dl_cons.gsi_state = next_state;
		IPA_MPM_DBG("GSI next_state = %d\n",
			ipa_mpm_ctx->md[probe_id].dl_cons.gsi_state);
		 mutex_unlock(&ipa_mpm_ctx->md[probe_id].mutex);
	}
}

static void ipa_mpm_change_teth_state(int probe_id,
	enum ipa_mpm_teth_state next_state)
{
	enum ipa_mpm_teth_state curr_state;

	if (probe_id >= IPA_MPM_MHIP_CH_ID_MAX) {
		IPA_MPM_ERR("Unknown probe_id\n");
		return;
	}

	curr_state = ipa_mpm_ctx->md[probe_id].teth_state;

	IPA_MPM_DBG("curr_state = %d, ip_state = %d mhip_s\n",
		curr_state, next_state);

	switch (curr_state) {
	case IPA_MPM_TETH_INIT:
		if (next_state == IPA_MPM_TETH_CONNECTED)
			next_state = IPA_MPM_TETH_INPROGRESS;
		break;
	case IPA_MPM_TETH_INPROGRESS:
		break;
	case IPA_MPM_TETH_CONNECTED:
		break;
	default:
		IPA_MPM_ERR("No change in state\n");
		break;
	}

	ipa_mpm_ctx->md[probe_id].teth_state = next_state;
	IPA_MPM_DBG("next_state = %d\n", next_state);
}

static void ipa_mpm_read_channel(enum ipa_client_type chan)
{
	struct gsi_chan_info chan_info;
	int ipa_ep_idx;
	struct ipa3_ep_context *ep;
	int res;

	ipa_ep_idx = ipa3_get_ep_mapping(chan);

	if (ipa_ep_idx == IPA_EP_NOT_ALLOCATED) {
		IPAERR("failed to get idx");
		return;
	}

	ep = &ipa3_ctx->ep[ipa_ep_idx];

	IPA_MPM_ERR("Reading channel for chan %d, ep = %d, gsi_chan_hdl = %d\n",
		chan, ep, ep->gsi_chan_hdl);

	res = ipa3_get_gsi_chan_info(&chan_info, ep->gsi_chan_hdl);
	if (res)
		IPA_MPM_ERR("Reading of channel failed for ep %d\n", ep);
}

static int ipa_mpm_start_stop_ul_mhip_data_path(int probe_id,
	enum ipa_mpm_start_stop_type start)
{
	int ipa_ep_idx;
	int res = 0;
	enum ipa_client_type ul_chan, dl_chan;

	if (probe_id >= IPA_MPM_MHIP_CH_ID_MAX) {
		IPA_MPM_ERR("Unknown probe_id\n");
		return 0;
	}
	get_ipa3_client(probe_id, &ul_chan, &dl_chan);
	IPA_MPM_DBG("Start/Stop Data Path ? = %d\n", start);


	/* MHIP Start Data path:
	 * IPA MHIP Producer: remove HOLB
	 * IPA MHIP Consumer : no op as there is no delay on these pipes.
	 */
	if (start) {
		IPA_MPM_DBG("Enabling data path\n");
		if (ul_chan != IPA_CLIENT_MAX) {
			/* Remove HOLB on the producer pipe */
			IPA_MPM_DBG("Removing HOLB on ep = %s\n",
				__stringify(ul_chan));
			ipa_ep_idx = ipa3_get_ep_mapping(ul_chan);

			if (ipa_ep_idx == IPA_EP_NOT_ALLOCATED) {
				IPAERR("failed to get idx");
				return ipa_ep_idx;
			}

			res = ipa3_enable_data_path(ipa_ep_idx);
			if (res)
				IPA_MPM_ERR("Enable data path failed res=%d\n",
					res);
		}
	} else {
		IPA_MPM_DBG("Disabling data path\n");
		if (ul_chan != IPA_CLIENT_MAX) {
			/* Set HOLB on the producer pipe */
			ipa_ep_idx = ipa3_get_ep_mapping(ul_chan);

			if (ipa_ep_idx == IPA_EP_NOT_ALLOCATED) {
				IPAERR("failed to get idx");
				return ipa_ep_idx;
			}

			res = ipa3_disable_data_path(ipa_ep_idx);
			if (res)
				IPA_MPM_ERR("disable data path failed res=%d\n",
					res);
		}
	}

	return res;
}

/* ipa_mpm_mhi_probe_cb is received for each MHI'/MHI channel
 * Currently we have 4 MHI channels.
 */
static int ipa_mpm_mhi_probe_cb(struct mhi_device *mhi_dev,
	const struct mhi_device_id *mhi_id)
{
	struct ipa_mpm_channel *ch;
	int ret;
	enum ipa_client_type ul_prod, dl_cons;
	int probe_id;
	struct ipa_req_chan_out_params ul_out_params, dl_out_params;
	void __iomem  *db_addr;
	int ipa_ep_idx;
	struct ipa3_ep_context *ep;
	u32 evt_ring_db_addr_low, evt_ring_db_addr_high;
	u32 wp_addr;
	int pipe_idx;

	IPA_MPM_FUNC_ENTRY();

	if (ipa_mpm_ctx == NULL) {
		IPA_MPM_ERR("ipa_mpm_ctx is NULL not expected, returning..\n");
		return -ENOMEM;
	}

	probe_id = get_idx_from_id(mhi_id);

	if (probe_id >= IPA_MPM_MHIP_CH_ID_MAX) {
		IPA_MPM_ERR("chan=%pK is not supported for now\n", mhi_id);
		return -EPERM;
	}

	if (ipa_mpm_ctx->md[probe_id].init_complete) {
		IPA_MPM_ERR("Probe initialization already done, returning\n");
		return 0;
	}

	IPA_MPM_DBG("Received probe for id=%d\n", probe_id);

	get_ipa3_client(probe_id, &ul_prod, &dl_cons);

	/* Vote for IPA clock for first time in initialization seq.
	 * IPA clock will be devoted when MHI enters LPM
	 * PCIe clock will be voted / devoted with every channel probe
	 * we receive.
	 * ul_prod = Host -> Device
	 * dl_cons = Device -> Host
	 */
	ipa_mpm_ctx->md[probe_id].mhi_dev = mhi_dev;
	ipa_mpm_ctx->mhi_parent_dev =
		ipa_mpm_ctx->md[probe_id].mhi_dev->dev.parent;

	ipa_mpm_vote_unvote_pcie_clk(CLK_ON, probe_id);
	mutex_lock(&ipa_mpm_ctx->md[probe_id].lpm_mutex);
	ipa_mpm_vote_unvote_ipa_clk(CLK_ON, probe_id);
	ipa_mpm_ctx->md[probe_id].in_lpm = false;
	mutex_unlock(&ipa_mpm_ctx->md[probe_id].lpm_mutex);
	IPA_MPM_DBG("ul chan = %d, dl_chan = %d\n", ul_prod, dl_cons);

	/*
	 * Set up MHI' pipes for Device IPA filling in
	 * Channel Context and Event Context.
	 * These params will be sent to Device side.
	 * UL CHAN = HOST -> Device
	 * DL CHAN = Device -> HOST
	 * per channel a TRE and EV is allocated.
	 * for a UL channel -
	 * IPA HOST PROD TRE -> IPA DEVICE CONS EV
	 * IPA HOST PROD EV ->  IPA DEVICE CONS TRE
	 * for a DL channel -
	 * IPA Device PROD TRE -> IPA HOST CONS EV
	 * IPA Device PROD EV ->  IPA HOST CONS TRE
	 */
	if (ul_prod != IPA_CLIENT_MAX) {
		/* store UL properties */
		ch = &ipa_mpm_ctx->md[probe_id].ul_prod;
		/* Store Channel properties */
		ch->chan_props.id = mhi_dev->ul_chan_id;
		ch->chan_props.device_db =
			ipa_mpm_ctx->dev_info.chdb_base +
			ch->chan_props.id * 8;
		/* Fill Channel Conext to be sent to Device side */
		ch->chan_props.ch_ctx.chtype =
			IPA_MPM_MHI_HOST_UL_CHANNEL;
		ch->chan_props.ch_ctx.erindex =
			mhi_dev->ul_event_id;
		ch->chan_props.ch_ctx.rlen = (IPA_MPM_RING_LEN) *
			GSI_EVT_RING_RE_SIZE_16B;
		/* Store Event properties */
		ch->evt_props.ev_ctx.update_rp_modc = 0;
		ch->evt_props.ev_ctx.update_rp_intmodt = 0;
		ch->evt_props.ev_ctx.ertype = 1;
		ch->evt_props.ev_ctx.rlen = (IPA_MPM_RING_LEN) *
			GSI_EVT_RING_RE_SIZE_16B;
		ch->evt_props.ev_ctx.buff_size = TRE_BUFF_SIZE;
		ch->evt_props.device_db =
			ipa_mpm_ctx->dev_info.erdb_base +
			ch->chan_props.ch_ctx.erindex * 8;
	}

	if (dl_cons != IPA_CLIENT_MAX) {
		/* store DL channel properties */
		ch = &ipa_mpm_ctx->md[probe_id].dl_cons;
		/* Store Channel properties */
		ch->chan_props.id = mhi_dev->dl_chan_id;
		ch->chan_props.device_db =
			ipa_mpm_ctx->dev_info.chdb_base +
			ch->chan_props.id * 8;
		/* Fill Channel Conext to be be sent to Dev side */
		ch->chan_props.ch_ctx.chstate = 1;
		ch->chan_props.ch_ctx.chtype =
			IPA_MPM_MHI_HOST_DL_CHANNEL;
		ch->chan_props.ch_ctx.erindex = mhi_dev->dl_event_id;
		ch->chan_props.ch_ctx.rlen = (IPA_MPM_RING_LEN) *
			GSI_EVT_RING_RE_SIZE_16B;
		/* Store Event properties */
		ch->evt_props.ev_ctx.update_rp_modc = 0;
		ch->evt_props.ev_ctx.update_rp_intmodt = 0;
		ch->evt_props.ev_ctx.ertype = 1;
		ch->evt_props.ev_ctx.rlen = (IPA_MPM_RING_LEN) *
			GSI_EVT_RING_RE_SIZE_16B;
		ch->evt_props.ev_ctx.buff_size = TRE_BUFF_SIZE;
		ch->evt_props.device_db =
			ipa_mpm_ctx->dev_info.erdb_base +
			ch->chan_props.ch_ctx.erindex * 8;
	}
	/* connect Host GSI pipes with MHI' protocol */
	if (ul_prod != IPA_CLIENT_MAX)  {
		ret = ipa_mpm_connect_mhip_gsi_pipe(ul_prod,
			probe_id, &ul_out_params);
		if (ret) {
			IPA_MPM_ERR("failed connecting MPM client %d\n",
					ul_prod);
			goto fail_gsi_setup;
		}
	}

	if (dl_cons != IPA_CLIENT_MAX) {
		ret = ipa_mpm_connect_mhip_gsi_pipe(dl_cons,
			probe_id, &dl_out_params);
		if (ret) {
			IPA_MPM_ERR("connecting MPM client = %d failed\n",
				dl_cons);
			goto fail_gsi_setup;
		}
	}

	if (ul_prod != IPA_CLIENT_MAX) {
		ch = &ipa_mpm_ctx->md[probe_id].ul_prod;
		ch->evt_props.ev_ctx.update_rp_addr =
			ipa_mpm_smmu_map_doorbell(
				MHIP_SMMU_DOMAIN_PCIE,
				ul_out_params.db_reg_phs_addr_lsb);
		if (ch->evt_props.ev_ctx.update_rp_addr == 0)
			ipa_assert();
		ipa_mpm_ctx->md[probe_id].ul_prod.db_device_iova =
			ch->evt_props.ev_ctx.update_rp_addr;

		ret = __ipa_mpm_configure_mhi_device(
				ch, probe_id, DMA_TO_HIPA);
		if (ret) {
			IPA_MPM_ERR("configure_mhi_dev fail %d\n",
					ret);
			goto fail_smmu;
		}
	}

	if (dl_cons != IPA_CLIENT_MAX) {
		ch = &ipa_mpm_ctx->md[probe_id].dl_cons;
		ch->evt_props.ev_ctx.update_rp_addr =
			ipa_mpm_smmu_map_doorbell(
					MHIP_SMMU_DOMAIN_PCIE,
					dl_out_params.db_reg_phs_addr_lsb);

		if (ch->evt_props.ev_ctx.update_rp_addr == 0)
			ipa_assert();

		ipa_mpm_ctx->md[probe_id].dl_cons.db_device_iova =
			ch->evt_props.ev_ctx.update_rp_addr;

		ret = __ipa_mpm_configure_mhi_device(ch, probe_id,
					DMA_FROM_HIPA);
		if (ret) {
			IPA_MPM_ERR("mpm_config_mhi_dev failed %d\n", ret);
			goto fail_smmu;
		}
	}

	ret = mhi_prepare_for_transfer(ipa_mpm_ctx->md[probe_id].mhi_dev);
	if (ret) {
		IPA_MPM_ERR("mhi_prepare_for_transfer failed %d\n", ret);
		WARN_ON(1);
		/*
		 * WA to handle prepare_for_tx failures.
		 * Though prepare for transfer fails, indicate success
		 * to MHI driver. remove_cb will be called eventually when
		 * Device side comes from where pending cleanup happens.
		 */
		atomic_inc(&ipa_mpm_ctx->probe_cnt);
		ipa_mpm_ctx->md[probe_id].init_complete = true;
		IPA_MPM_FUNC_EXIT();
		return 0;
	}

	/*
	 * Ring initial channel db - Host Side UL and Device side DL channel.
	 * To ring doorbell, write "WP" into doorbell register.
	 * This WP should be set to 1 element less than ring max.
	 */

	/* Ring UL PRODUCER TRANSFER RING (HOST IPA -> DEVICE IPA) Doorbell */
	if (ul_prod != IPA_CLIENT_MAX) {
		IPA_MPM_DBG("Host UL TR PA DB = 0X%0x\n",
			ul_out_params.db_reg_phs_addr_lsb);

		db_addr = ioremap(
			(phys_addr_t)(ul_out_params.db_reg_phs_addr_lsb), 4);

		wp_addr = ipa_mpm_ctx->md[probe_id].ul_prod_ring.tr_pa +
			((IPA_MPM_RING_LEN - 1) * GSI_CHAN_RE_SIZE_16B);

		iowrite32(wp_addr, db_addr);

		IPA_MPM_DBG("Host UL TR  DB = 0X%pK, wp_addr = 0X%0x",
			db_addr, wp_addr);

		iounmap(db_addr);
		ipa_mpm_read_channel(ul_prod);
	}

	/* Ring UL PRODUCER EVENT RING (HOST IPA -> DEVICE IPA) Doorbell
	 * Ring the event DB to a value outside the
	 * ring range such that rp and wp never meet.
	 */
	if (ul_prod != IPA_CLIENT_MAX) {
		ipa_ep_idx = ipa3_get_ep_mapping(ul_prod);

		if (ipa_ep_idx == IPA_EP_NOT_ALLOCATED) {
			IPA_MPM_ERR("fail to alloc EP.\n");
			goto fail_start_channel;
		}
		ep = &ipa3_ctx->ep[ipa_ep_idx];

		IPA_MPM_DBG("for ep_idx %d , gsi_evt_ring_hdl = %ld\n",
			ipa_ep_idx, ep->gsi_evt_ring_hdl);
		gsi_query_evt_ring_db_addr(ep->gsi_evt_ring_hdl,
			&evt_ring_db_addr_low, &evt_ring_db_addr_high);

		IPA_MPM_DBG("Host UL ER PA DB = 0X%0x\n",
			evt_ring_db_addr_low);

		db_addr = ioremap((phys_addr_t)(evt_ring_db_addr_low), 4);

		wp_addr = ipa_mpm_ctx->md[probe_id].ul_prod_ring.er_pa +
			((IPA_MPM_RING_LEN + 1) * GSI_EVT_RING_RE_SIZE_16B);
		IPA_MPM_DBG("Host UL ER  DB = 0X%pK, wp_addr = 0X%0x",
			db_addr, wp_addr);

		iowrite32(wp_addr, db_addr);
		iounmap(db_addr);
	}

	/* Ring DEVICE IPA DL CONSUMER Event Doorbell */
	if (ul_prod != IPA_CLIENT_MAX) {
		db_addr = ioremap((phys_addr_t)
			(ipa_mpm_ctx->md[probe_id].ul_prod.evt_props.device_db),
			4);

		wp_addr = ipa_mpm_ctx->md[probe_id].ul_prod_ring.tr_pa +
			((IPA_MPM_RING_LEN + 1) * GSI_EVT_RING_RE_SIZE_16B);

		iowrite32(wp_addr, db_addr);
		iounmap(db_addr);
	}

	/* Ring DL PRODUCER (DEVICE IPA -> HOST IPA) Doorbell */
	if (dl_cons != IPA_CLIENT_MAX) {
		db_addr = ioremap((phys_addr_t)
		(ipa_mpm_ctx->md[probe_id].dl_cons.chan_props.device_db),
		4);

		wp_addr = ipa_mpm_ctx->md[probe_id].dl_prod_ring.tr_pa +
			((IPA_MPM_RING_LEN - 1) * GSI_CHAN_RE_SIZE_16B);

		IPA_MPM_DBG("Device DL TR  DB = 0X%pK, wp_addr = 0X%0x",
			db_addr, wp_addr);

		iowrite32(wp_addr, db_addr);

		iounmap(db_addr);
	}

	/*
	 * Ring event ring DB on Device side.
	 * ipa_mpm should ring the event DB to a value outside the
	 * ring range such that rp and wp never meet.
	 */
	if (dl_cons != IPA_CLIENT_MAX) {
		db_addr =
		ioremap(
		(phys_addr_t)
		(ipa_mpm_ctx->md[probe_id].dl_cons.evt_props.device_db),
		4);

		wp_addr = ipa_mpm_ctx->md[probe_id].dl_prod_ring.er_pa +
			((IPA_MPM_RING_LEN + 1) * GSI_EVT_RING_RE_SIZE_16B);

		iowrite32(wp_addr, db_addr);
		IPA_MPM_DBG("Device  UL ER  DB = 0X%pK,wp_addr = 0X%0x",
			db_addr, wp_addr);
		iounmap(db_addr);
	}

	/* Ring DL EVENT RING CONSUMER (DEVICE IPA CONSUMER) Doorbell */
	if (dl_cons != IPA_CLIENT_MAX) {
		ipa_ep_idx = ipa3_get_ep_mapping(dl_cons);

		if (ipa_ep_idx == IPA_EP_NOT_ALLOCATED) {
			IPA_MPM_ERR("fail to alloc EP.\n");
			goto fail_start_channel;
		}
		ep = &ipa3_ctx->ep[ipa_ep_idx];

		gsi_query_evt_ring_db_addr(ep->gsi_evt_ring_hdl,
			&evt_ring_db_addr_low, &evt_ring_db_addr_high);
		IPA_MPM_DBG("Host DL ER PA DB = 0X%0x\n",
				evt_ring_db_addr_low);
		db_addr = ioremap((phys_addr_t)(evt_ring_db_addr_low), 4);

		wp_addr = ipa_mpm_ctx->md[probe_id].dl_prod_ring.tr_pa +
			((IPA_MPM_RING_LEN + 1) * GSI_EVT_RING_RE_SIZE_16B);
		iowrite32(wp_addr, db_addr);
		IPA_MPM_DBG("Host  DL ER  DB = 0X%pK, wp_addr = 0X%0x",
			db_addr, wp_addr);
		iounmap(db_addr);
	}

	/* Check if TETH connection is in progress.
	 * If teth isn't started by now, then Stop UL channel.
	 */
	switch (ipa_mpm_ctx->md[probe_id].teth_state) {
	case IPA_MPM_TETH_INIT:
		if (ul_prod != IPA_CLIENT_MAX) {
			/* No teth started yet, disable UL channel */
			ipa_mpm_start_stop_mhip_chan(IPA_MPM_MHIP_CHAN_UL,
						probe_id, MPM_MHIP_STOP);
			/* Disable data path */
			if (ipa_mpm_start_stop_ul_mhip_data_path(probe_id,
				MPM_MHIP_STOP)) {
				IPA_MPM_ERR("MHIP Enable data path failed\n");
				goto fail_start_channel;
			}
		}
		ipa_mpm_vote_unvote_pcie_clk(CLK_OFF, probe_id);
		break;
	case IPA_MPM_TETH_INPROGRESS:
	case IPA_MPM_TETH_CONNECTED:
		IPA_MPM_DBG("UL channel is already started, continue\n");
		ipa_mpm_change_teth_state(probe_id, IPA_MPM_TETH_CONNECTED);

		/* Enable data path */
		if (ul_prod != IPA_CLIENT_MAX) {
			if (ipa_mpm_start_stop_ul_mhip_data_path(probe_id,
				MPM_MHIP_START)) {
				IPA_MPM_ERR("MHIP Enable data path failed\n");
				goto fail_start_channel;
			}
		}
		/* Lift the delay for rmnet USB prod pipe */
		if (probe_id == IPA_MPM_MHIP_CH_ID_1) {
			pipe_idx = ipa3_get_ep_mapping(IPA_CLIENT_USB_PROD);
			ipa3_xdci_ep_delay_rm(pipe_idx);
		}
		break;
	default:
		IPA_MPM_DBG("No op for UL channel, in teth state = %d",
			ipa_mpm_ctx->md[probe_id].teth_state);
		break;
	}

	atomic_inc(&ipa_mpm_ctx->probe_cnt);
	ipa_mpm_ctx->md[probe_id].init_complete = true;
	IPA_MPM_FUNC_EXIT();
	return 0;

fail_gsi_setup:
fail_start_channel:
fail_smmu:
	if (ipa_mpm_ctx->dev_info.ipa_smmu_enabled)
		IPA_MPM_DBG("SMMU failed\n");
	ipa_mpm_vote_unvote_pcie_clk(CLK_OFF, probe_id);
	ipa_mpm_vote_unvote_ipa_clk(CLK_OFF, probe_id);
	ipa_assert();
	return ret;
}

static void ipa_mpm_init_mhip_channel_info(void)
{
	/* IPA_MPM_MHIP_CH_ID_0 => MHIP TETH PIPES  */
	ipa_mpm_pipes[IPA_MPM_MHIP_CH_ID_0].dl_cons.ipa_client =
		IPA_CLIENT_MHI_PRIME_TETH_PROD;
	ipa_mpm_pipes[IPA_MPM_MHIP_CH_ID_0].dl_cons.ep_cfg =
		mhip_dl_teth_ep_cfg;
	ipa_mpm_pipes[IPA_MPM_MHIP_CH_ID_0].ul_prod.ipa_client =
		IPA_CLIENT_MHI_PRIME_TETH_CONS;
	ipa_mpm_pipes[IPA_MPM_MHIP_CH_ID_0].ul_prod.ep_cfg =
		mhip_ul_teth_ep_cfg;
	ipa_mpm_pipes[IPA_MPM_MHIP_CH_ID_0].mhip_client =
		IPA_MPM_MHIP_TETH;

	/* IPA_MPM_MHIP_CH_ID_1 => MHIP RMNET PIPES */
	ipa_mpm_pipes[IPA_MPM_MHIP_CH_ID_1].dl_cons.ipa_client =
		IPA_CLIENT_MHI_PRIME_RMNET_PROD;
	ipa_mpm_pipes[IPA_MPM_MHIP_CH_ID_1].dl_cons.ep_cfg =
		mhip_dl_rmnet_ep_cfg;
	ipa_mpm_pipes[IPA_MPM_MHIP_CH_ID_1].ul_prod.ipa_client =
		IPA_CLIENT_MHI_PRIME_RMNET_CONS;
	ipa_mpm_pipes[IPA_MPM_MHIP_CH_ID_1].ul_prod.ep_cfg =
		mhip_ul_rmnet_ep_cfg;
	ipa_mpm_pipes[IPA_MPM_MHIP_CH_ID_1].mhip_client =
		IPA_MPM_MHIP_USB_RMNET;

	/* IPA_MPM_MHIP_CH_ID_2 => MHIP ADPL PIPE */
	ipa_mpm_pipes[IPA_MPM_MHIP_CH_ID_2].dl_cons.ipa_client =
		IPA_CLIENT_MHI_PRIME_DPL_PROD;
	ipa_mpm_pipes[IPA_MPM_MHIP_CH_ID_2].dl_cons.ep_cfg =
		mhip_dl_dpl_ep_cfg;
	ipa_mpm_pipes[IPA_MPM_MHIP_CH_ID_2].ul_prod.ipa_client =
		IPA_CLIENT_MAX;
	ipa_mpm_pipes[IPA_MPM_MHIP_CH_ID_2].mhip_client =
		IPA_MPM_MHIP_USB_DPL;
}

static void ipa_mpm_mhi_remove_cb(struct mhi_device *mhi_dev)
{
	int mhip_idx;

	IPA_MPM_FUNC_ENTRY();

	for (mhip_idx = 0; mhip_idx < IPA_MPM_MHIP_CH_ID_MAX; mhip_idx++) {
		if (mhi_dev == ipa_mpm_ctx->md[mhip_idx].mhi_dev)
			break;
	}
	if (mhip_idx >= IPA_MPM_MHIP_CH_ID_MAX) {
		IPA_MPM_DBG("remove_cb for mhip_idx = %d not probed before\n",
			mhip_idx);
		return;
	}

	IPA_MPM_DBG("remove_cb for mhip_idx = %d", mhip_idx);

	ipa_mpm_mhip_shutdown(mhip_idx);
	atomic_dec(&ipa_mpm_ctx->probe_cnt);

	if (atomic_read(&ipa_mpm_ctx->probe_cnt) == 0) {
		/* Last probe done, reset Everything here */
		ipa_mpm_ctx->mhi_parent_dev = NULL;
		ipa_mpm_ctx->carved_smmu_cb.next_addr =
			ipa_mpm_ctx->carved_smmu_cb.va_start;
		atomic_set(&ipa_mpm_ctx->pcie_clk_total_cnt, 0);
		for (mhip_idx = 0;
			mhip_idx < IPA_MPM_MHIP_CH_ID_MAX; mhip_idx++) {
			atomic_set(
				&ipa_mpm_ctx->md[mhip_idx].clk_cnt.pcie_clk_cnt,
				0);
		}
	}

	IPA_MPM_FUNC_EXIT();
}

static void ipa_mpm_mhi_status_cb(struct mhi_device *mhi_dev,
				enum MHI_CB mhi_cb)
{
	int mhip_idx;
	enum mhip_status_type status;

	IPA_MPM_DBG("%d\n", mhi_cb);

	for (mhip_idx = 0; mhip_idx < IPA_MPM_MHIP_CH_ID_MAX; mhip_idx++) {
		if (mhi_dev == ipa_mpm_ctx->md[mhip_idx].mhi_dev)
			break;
	}
	if (mhip_idx >= IPA_MPM_MHIP_CH_ID_MAX) {
		IPA_MPM_DBG("ignoring secondary callbacks\n");
		return;
	}

	mutex_lock(&ipa_mpm_ctx->md[mhip_idx].lpm_mutex);
	switch (mhi_cb) {
	case MHI_CB_IDLE:
		break;
	case MHI_CB_LPM_ENTER:
		if (!ipa_mpm_ctx->md[mhip_idx].in_lpm) {
			status = ipa_mpm_start_stop_mhip_chan(
				IPA_MPM_MHIP_CHAN_DL,
				mhip_idx, MPM_MHIP_STOP);
			IPA_MPM_DBG("status = %d\n", status);
			ipa_mpm_vote_unvote_ipa_clk(CLK_OFF, mhip_idx);
			ipa_mpm_ctx->md[mhip_idx].in_lpm = true;
		} else {
			IPA_MPM_DBG("Already in lpm\n");
		}
		break;
	case MHI_CB_LPM_EXIT:
		if (ipa_mpm_ctx->md[mhip_idx].in_lpm) {
			ipa_mpm_vote_unvote_ipa_clk(CLK_ON, mhip_idx);
			status = ipa_mpm_start_stop_mhip_chan(
				IPA_MPM_MHIP_CHAN_DL,
				mhip_idx, MPM_MHIP_START);
			IPA_MPM_DBG("status = %d\n", status);
			ipa_mpm_ctx->md[mhip_idx].in_lpm = false;
		} else {
			IPA_MPM_DBG("Already out of lpm\n");
		}
		break;
	case MHI_CB_EE_RDDM:
	case MHI_CB_PENDING_DATA:
	case MHI_CB_SYS_ERROR:
	case MHI_CB_FATAL_ERROR:
	case MHI_CB_BW_REQ:
	case MHI_CB_EE_MISSION_MODE:
		IPA_MPM_ERR("unexpected event %d\n", mhi_cb);
		break;
	}
	mutex_unlock(&ipa_mpm_ctx->md[mhip_idx].lpm_mutex);
}

static int ipa_mpm_set_dma_mode(enum ipa_client_type src_pipe,
	enum ipa_client_type dst_pipe)
{
	int result = 0;
	struct ipa_ep_cfg ep_cfg = { { 0 } };

	IPA_MPM_FUNC_ENTRY();
	IPA_MPM_DBG("DMA from %d to %d\n", src_pipe, dst_pipe);

	/* Set USB PROD PIPE DMA to MHIP PROD PIPE */
	ep_cfg.mode.mode = IPA_DMA;
	ep_cfg.mode.dst = dst_pipe;
	ep_cfg.seq.set_dynamic = true;

	result = ipa_cfg_ep(ipa_get_ep_mapping(src_pipe), &ep_cfg);
	IPA_MPM_FUNC_EXIT();

	return result;
}

int ipa_mpm_reset_dma_mode(enum ipa_client_type src_pipe,
	enum ipa_client_type dst_pipe)
{
	int result = 0;
	struct ipa_ep_cfg ep_cfg = { { 0 } };

	IPA_MPM_FUNC_ENTRY();
	IPA_MPM_DBG("DMA from %d to %d\n", src_pipe, dst_pipe);

	/* Set USB PROD PIPE DMA to MHIP PROD PIPE */
	ep_cfg.mode.mode = IPA_BASIC;
	ep_cfg.mode.dst = IPA_CLIENT_APPS_LAN_CONS;
	ep_cfg.seq.set_dynamic = true;

	result = ipa_cfg_ep(ipa_get_ep_mapping(src_pipe), &ep_cfg);
	IPA_MPM_FUNC_EXIT();

	return result;
}

static void ipa_mpm_mhip_map_prot(enum ipa_usb_teth_prot prot,
	enum ipa_mpm_mhip_client_type *mhip_client)
{
	switch (prot) {
	case IPA_USB_RNDIS:
		*mhip_client = IPA_MPM_MHIP_TETH;
		break;
	case IPA_USB_RMNET:
		*mhip_client = IPA_MPM_MHIP_USB_RMNET;
		break;
	case IPA_USB_DIAG:
		*mhip_client = IPA_MPM_MHIP_USB_DPL;
		break;
	default:
		*mhip_client = IPA_MPM_MHIP_NONE;
		break;
	}
	IPA_MPM_DBG("Mapped xdci prot %d -> MHIP prot %d\n", prot,
		*mhip_client);
}

int ipa_mpm_mhip_xdci_pipe_enable(enum ipa_usb_teth_prot xdci_teth_prot)
{
	int probe_id = IPA_MPM_MHIP_CH_ID_MAX;
	int i;
	enum ipa_mpm_mhip_client_type mhip_client;
	enum mhip_status_type status;
	int ret = 0;
	int pipe_idx;

	if (ipa_mpm_ctx == NULL) {
		IPA_MPM_ERR("MPM not platform probed yet, returning ..\n");
		return 0;
	}

	ipa_mpm_mhip_map_prot(xdci_teth_prot, &mhip_client);

	for (i = 0; i < IPA_MPM_MHIP_CH_ID_MAX; i++) {
		if (ipa_mpm_pipes[i].mhip_client == mhip_client) {
			probe_id = i;
			break;
		}
	}

	if (probe_id == IPA_MPM_MHIP_CH_ID_MAX) {
		IPA_MPM_ERR("Unknown probe_id\n");
		return 0;
	}

	IPA_MPM_DBG("Connect xdci prot %d -> mhip_client = %d probe_id = %d\n",
			xdci_teth_prot, mhip_client, probe_id);

	ipa_mpm_ctx->md[probe_id].mhip_client = mhip_client;

	ret = ipa_mpm_vote_unvote_pcie_clk(CLK_ON, probe_id);
	if (ret) {
		IPA_MPM_ERR("Error cloking on PCIe clk, err = %d\n", ret);
		return ret;
	}

	switch (mhip_client) {
	case IPA_MPM_MHIP_USB_RMNET:
		ipa_mpm_set_dma_mode(IPA_CLIENT_USB_PROD,
			IPA_CLIENT_MHI_PRIME_RMNET_CONS);
		break;
	case IPA_MPM_MHIP_USB_DPL:
		IPA_MPM_DBG("connecting DPL prot %d\n", mhip_client);
		ipa_mpm_change_teth_state(probe_id, IPA_MPM_TETH_CONNECTED);
		return 0;
	default:
		IPA_MPM_DBG("mhip_client = %d not processed\n", mhip_client);
		ret = ipa_mpm_vote_unvote_pcie_clk(CLK_OFF, probe_id);
		if (ret) {
			IPA_MPM_ERR("Error unvoting on PCIe clk, err = %d\n",
					ret);
			return ret;
		}
		return 0;
	}

	if (mhip_client != IPA_MPM_MHIP_USB_DPL)
		/* Start UL MHIP channel for offloading teth connection */
		status = ipa_mpm_start_stop_mhip_chan(IPA_MPM_MHIP_CHAN_UL,
							probe_id,
							MPM_MHIP_START);
	switch (status) {
	case MHIP_STATUS_SUCCESS:
	case MHIP_STATUS_NO_OP:
		ipa_mpm_change_teth_state(probe_id, IPA_MPM_TETH_CONNECTED);
		ipa_mpm_start_stop_ul_mhip_data_path(probe_id,
						MPM_MHIP_START);

		pipe_idx = ipa3_get_ep_mapping(IPA_CLIENT_USB_PROD);

		/* Lift the delay for rmnet USB prod pipe */
		ipa3_xdci_ep_delay_rm(pipe_idx);
		if (status == MHIP_STATUS_NO_OP) {
			/* Channels already have been started,
			 * we can devote for pcie clocks
			 */
			ipa_mpm_vote_unvote_pcie_clk(CLK_OFF, probe_id);
		}
		break;
	case MHIP_STATUS_EP_NOT_READY:
		ipa_mpm_vote_unvote_pcie_clk(CLK_OFF, probe_id);
		ipa_mpm_change_teth_state(probe_id, IPA_MPM_TETH_INPROGRESS);
		break;
	case MHIP_STATUS_FAIL:
	case MHIP_STATUS_BAD_STATE:
	case MHIP_STATUS_EP_NOT_FOUND:
		IPA_MPM_ERR("UL chan cant be started err =%d\n", status);
		ipa_mpm_vote_unvote_pcie_clk(CLK_OFF, probe_id);
		ret = -EFAULT;
		break;
	default:
		ipa_mpm_vote_unvote_pcie_clk(CLK_OFF, probe_id);
		IPA_MPM_ERR("Err not found\n");
		break;
	}
	return ret;
}

int ipa_mpm_mhip_xdci_pipe_disable(enum ipa_usb_teth_prot xdci_teth_prot)
{
	int probe_id = IPA_MPM_MHIP_CH_ID_MAX;
	int i;
	enum ipa_mpm_mhip_client_type mhip_client;
	enum mhip_status_type status;
	int ret = 0;

	if (ipa_mpm_ctx == NULL) {
		IPA_MPM_ERR("MPM not platform probed, returning ..\n");
		return 0;
	}

	ipa_mpm_mhip_map_prot(xdci_teth_prot, &mhip_client);

	for (i = 0; i < IPA_MPM_MHIP_CH_ID_MAX; i++) {
		if (ipa_mpm_pipes[i].mhip_client == mhip_client) {
			probe_id = i;
			break;
		}
	}

	if (probe_id == IPA_MPM_MHIP_CH_ID_MAX) {
		IPA_MPM_ERR("Invalid probe_id\n");
		return 0;
	}

	IPA_MPM_ERR("xdci disconnect prot %d mhip_client = %d probe_id = %d\n",
			xdci_teth_prot, mhip_client, probe_id);

	switch (mhip_client) {
	case IPA_MPM_MHIP_USB_RMNET:
		ret = ipa_mpm_reset_dma_mode(IPA_CLIENT_USB_PROD,
			IPA_CLIENT_MHI_PRIME_RMNET_CONS);
		if (ret) {
			IPA_MPM_ERR("failed to reset dma mode\n");
			return ret;
		}
		break;
	case IPA_MPM_MHIP_TETH:
		IPA_MPM_DBG("Rndis Disconnect, wait for wan_state ioctl\n");
		return 0;
	case IPA_MPM_MHIP_USB_DPL:
		IPA_MPM_DBG("Teth Disconnecting for DPL\n");
		ipa_mpm_change_teth_state(probe_id, IPA_MPM_TETH_INIT);
		ret = ipa_mpm_vote_unvote_pcie_clk(CLK_OFF, probe_id);
		if (ret)
			IPA_MPM_ERR("Error cloking off PCIe clk, err = %d\n",
				ret);
		ipa_mpm_ctx->md[probe_id].mhip_client = IPA_MPM_MHIP_NONE;
		return ret;
	default:
		IPA_MPM_ERR("mhip_client = %d not supported\n", mhip_client);
		return 0;
	}

	status = ipa_mpm_start_stop_mhip_chan(IPA_MPM_MHIP_CHAN_UL,
		probe_id, MPM_MHIP_STOP);

	switch (status) {
	case MHIP_STATUS_SUCCESS:
	case MHIP_STATUS_NO_OP:
	case MHIP_STATUS_EP_NOT_READY:
		ipa_mpm_change_teth_state(probe_id, IPA_MPM_TETH_INIT);
		ipa_mpm_start_stop_ul_mhip_data_path(probe_id, MPM_MHIP_STOP);
		break;
	case MHIP_STATUS_FAIL:
	case MHIP_STATUS_BAD_STATE:
	case MHIP_STATUS_EP_NOT_FOUND:
		IPA_MPM_ERR("UL chan cant be started err =%d\n", status);
		ipa_mpm_vote_unvote_pcie_clk(CLK_OFF, probe_id);
		return -EFAULT;
		break;
	default:
		IPA_MPM_ERR("Err not found\n");
		break;
	}

	ret = ipa_mpm_vote_unvote_pcie_clk(CLK_OFF, probe_id);

	if (ret) {
		IPA_MPM_ERR("Error cloking off PCIe clk, err = %d\n", ret);
		return ret;
	}

	ipa_mpm_ctx->md[probe_id].mhip_client = IPA_MPM_MHIP_NONE;

	return ret;
}

static int ipa_mpm_populate_smmu_info(struct platform_device *pdev)
{
	struct ipa_smmu_in_params smmu_in;
	struct ipa_smmu_out_params smmu_out;
	u32 carved_iova_ap_mapping[2];
	struct ipa_smmu_cb_ctx *cb;
	struct ipa_smmu_cb_ctx *ap_cb = ipa3_get_smmu_ctx(IPA_SMMU_CB_AP);
	int ret = 0;

	if (ipa_mpm_ctx->carved_smmu_cb.valid) {
		IPA_MPM_DBG("SMMU Context allocated, returning ..\n");
		return ret;
	}

	cb = &ipa_mpm_ctx->carved_smmu_cb;

	/* get IPA SMMU enabled status */
	smmu_in.smmu_client = IPA_SMMU_AP_CLIENT;
	if (ipa_get_smmu_params(&smmu_in, &smmu_out))
		ipa_mpm_ctx->dev_info.ipa_smmu_enabled = false;
	else
		ipa_mpm_ctx->dev_info.ipa_smmu_enabled =
		smmu_out.smmu_enable;

	/* get cache_coherent enable or not */
	ipa_mpm_ctx->dev_info.is_cache_coherent = ap_cb->is_cache_coherent;
	if (of_property_read_u32_array(pdev->dev.of_node, "qcom,iova-mapping",
		carved_iova_ap_mapping, 2)) {
		IPA_MPM_ERR("failed to read of_node %s\n",
			"qcom,mpm-iova-mapping");
		return -EINVAL;
	}
	ipa_mpm_ctx->dev_info.pcie_smmu_enabled = true;

	if (ipa_mpm_ctx->dev_info.ipa_smmu_enabled !=
		ipa_mpm_ctx->dev_info.pcie_smmu_enabled) {
		IPA_MPM_DBG("PCIE/IPA SMMU config mismatch\n");
		return -EINVAL;
	}

	cb->va_start = carved_iova_ap_mapping[0];
	cb->va_size = carved_iova_ap_mapping[1];
	cb->va_end = cb->va_start + cb->va_size;

	if (cb->va_end >= ap_cb->va_start) {
		IPA_MPM_ERR("MPM iommu and AP overlap addr 0x%lx\n",
				cb->va_start);
		ipa_assert();
		return -EFAULT;
	}

	cb->dev = ipa_mpm_ctx->dev_info.dev;
	cb->valid = true;
	cb->next_addr = cb->va_start;

	if (dma_set_mask_and_coherent(ipa_mpm_ctx->dev_info.dev,
		DMA_BIT_MASK(64))) {
		IPA_MPM_ERR("setting DMA mask to 64 failed.\n");
		return -EINVAL;
	}

	return ret;
}

static int ipa_mpm_probe(struct platform_device *pdev)
{
	int ret = 0;
	int i = 0;
	int idx = 0;

	IPA_MPM_FUNC_ENTRY();

	if (ipa_mpm_ctx) {
		IPA_MPM_DBG("MPM is already probed, returning\n");
		return 0;
	}

	ret = ipa_register_ipa_ready_cb(ipa_mpm_ipa3_ready_cb, (void *)pdev);
	/*
	 * If we received -EEXIST, IPA has initialized. So we need
	 * to continue the probing process.
	 */
	if (!ret) {
		IPA_MPM_DBG("IPA not ready yet, registering callback\n");
		return ret;
	}
	IPA_MPM_DBG("IPA is ready, continue with probe\n");

	ipa_mpm_ctx = kzalloc(sizeof(*ipa_mpm_ctx), GFP_KERNEL);

	if (!ipa_mpm_ctx)
		return -ENOMEM;

	for (i = 0; i < IPA_MPM_MHIP_CH_ID_MAX; i++) {
		mutex_init(&ipa_mpm_ctx->md[i].mutex);
		mutex_init(&ipa_mpm_ctx->md[i].lpm_mutex);
	}

	ipa_mpm_ctx->dev_info.pdev = pdev;
	ipa_mpm_ctx->dev_info.dev = &pdev->dev;

	ipa_mpm_init_mhip_channel_info();

	if (of_property_read_u32(pdev->dev.of_node, "qcom,mhi-chdb-base",
		&ipa_mpm_ctx->dev_info.chdb_base)) {
		IPA_MPM_ERR("failed to read qcom,mhi-chdb-base\n");
		goto fail_probe;
	}
	IPA_MPM_DBG("chdb-base=0x%x\n", ipa_mpm_ctx->dev_info.chdb_base);

	if (of_property_read_u32(pdev->dev.of_node, "qcom,mhi-erdb-base",
		&ipa_mpm_ctx->dev_info.erdb_base)) {
		IPA_MPM_ERR("failed to read qcom,mhi-erdb-base\n");
		goto fail_probe;
	}
	IPA_MPM_DBG("erdb-base=0x%x\n", ipa_mpm_ctx->dev_info.erdb_base);

	ret = ipa_mpm_populate_smmu_info(pdev);

	if (ret) {
		IPA_MPM_DBG("SMMU Config failed\n");
		goto fail_probe;
	}

	atomic_set(&ipa_mpm_ctx->ipa_clk_total_cnt, 0);
	atomic_set(&ipa_mpm_ctx->pcie_clk_total_cnt, 0);

	for (idx = 0; idx < IPA_MPM_MHIP_CH_ID_MAX; idx++) {
		ipa_mpm_ctx->md[idx].ul_prod.gsi_state = GSI_INIT;
		ipa_mpm_ctx->md[idx].dl_cons.gsi_state = GSI_INIT;
		atomic_set(&ipa_mpm_ctx->md[idx].clk_cnt.ipa_clk_cnt, 0);
		atomic_set(&ipa_mpm_ctx->md[idx].clk_cnt.pcie_clk_cnt, 0);
	}

	ret = mhi_driver_register(&mhi_driver);
	if (ret) {
		IPA_MPM_ERR("mhi_driver_register failed %d\n", ret);
		goto fail_probe;
	}
	IPA_MPM_FUNC_EXIT();
	return 0;

fail_probe:
	kfree(ipa_mpm_ctx);
	ipa_mpm_ctx = NULL;
	return -EFAULT;
}

static int ipa_mpm_remove(struct platform_device *pdev)
{
	IPA_MPM_FUNC_ENTRY();

	mhi_driver_unregister(&mhi_driver);
	IPA_MPM_FUNC_EXIT();
	return 0;
}

static const struct of_device_id ipa_mpm_dt_match[] = {
	{ .compatible = "qcom,ipa-mpm" },
	{},
};
MODULE_DEVICE_TABLE(of, ipa_mpm_dt_match);

static struct platform_driver ipa_ipa_mpm_driver = {
	.driver = {
		.name = "ipa_mpm",
		.of_match_table = ipa_mpm_dt_match,
	},
	.probe = ipa_mpm_probe,
	.remove = ipa_mpm_remove,
};

/**
 * ipa_mpm_init() - Registers ipa_mpm as a platform device for a APQ
 *
 * This function is called after bootup for APQ device.
 * ipa_mpm will register itself as a platform device, and probe
 * function will get called.
 *
 * Return: None
 */
static int __init ipa_mpm_init(void)
{
	IPA_MPM_DBG("register ipa_mpm platform device\n");
	return platform_driver_register(&ipa_ipa_mpm_driver);
}

/**
 * ipa3_is_mhip_offload_enabled() - check if IPA MPM module was initialized
 * successfully. If it is initialized, MHIP is enabled for teth
 *
 * Return value: 1 for yes; 0 for no
 */
int ipa3_is_mhip_offload_enabled(void)
{
	if (ipa_mpm_ctx == NULL)
		return 0;
	else
		return 1;
}

int ipa_mpm_panic_handler(char *buf, int size)
{
	int i;
	int cnt = 0;

	cnt = scnprintf(buf, size,
			"\n---- MHIP Active Clients Table ----\n");
	cnt += scnprintf(buf + cnt, size - cnt,
			"Total PCIe active clients count: %d\n",
			atomic_read(&ipa_mpm_ctx->pcie_clk_total_cnt));
	cnt += scnprintf(buf + cnt, size - cnt,
			"Total IPA active clients count: %d\n",
			atomic_read(&ipa_mpm_ctx->ipa_clk_total_cnt));

	for (i = 0; i < IPA_MPM_MHIP_CH_ID_MAX; i++) {
		cnt += scnprintf(buf + cnt, size - cnt,
			"client id: %d ipa vote cnt: %d pcie vote cnt\n", i,
			atomic_read(&ipa_mpm_ctx->md[i].clk_cnt.ipa_clk_cnt),
			atomic_read(&ipa_mpm_ctx->md[i].clk_cnt.pcie_clk_cnt));
	}
	return cnt;
}

/**
 * ipa3_get_mhip_gsi_stats() - Query MHIP gsi stats from uc
 * @stats:	[inout] stats blob from client populated by driver
 *
 * Returns:	0 on success, negative on failure
 *
 * @note Cannot be called from atomic context
 *
 */
int ipa3_get_mhip_gsi_stats(struct ipa3_uc_dbg_ring_stats *stats)
{
	int i;

	if (!ipa3_ctx->mhip_ctx.dbg_stats.uc_dbg_stats_mmio) {
		IPAERR("bad parms NULL mhip_gsi_stats_mmio\n");
		return -EINVAL;
	}
	IPA_ACTIVE_CLIENTS_INC_SIMPLE();
	for (i = 0; i < MAX_MHIP_CHANNELS; i++) {
		stats->ring[i].ringFull = ioread32(
			ipa3_ctx->mhip_ctx.dbg_stats.uc_dbg_stats_mmio
			+ i * IPA3_UC_DEBUG_STATS_OFF +
			IPA3_UC_DEBUG_STATS_RINGFULL_OFF);
		stats->ring[i].ringEmpty = ioread32(
			ipa3_ctx->mhip_ctx.dbg_stats.uc_dbg_stats_mmio
			+ i * IPA3_UC_DEBUG_STATS_OFF +
			IPA3_UC_DEBUG_STATS_RINGEMPTY_OFF);
		stats->ring[i].ringUsageHigh = ioread32(
			ipa3_ctx->mhip_ctx.dbg_stats.uc_dbg_stats_mmio
			+ i * IPA3_UC_DEBUG_STATS_OFF +
			IPA3_UC_DEBUG_STATS_RINGUSAGEHIGH_OFF);
		stats->ring[i].ringUsageLow = ioread32(
			ipa3_ctx->mhip_ctx.dbg_stats.uc_dbg_stats_mmio
			+ i * IPA3_UC_DEBUG_STATS_OFF +
			IPA3_UC_DEBUG_STATS_RINGUSAGELOW_OFF);
		stats->ring[i].RingUtilCount = ioread32(
			ipa3_ctx->mhip_ctx.dbg_stats.uc_dbg_stats_mmio
			+ i * IPA3_UC_DEBUG_STATS_OFF +
			IPA3_UC_DEBUG_STATS_RINGUTILCOUNT_OFF);
	}
	IPA_ACTIVE_CLIENTS_DEC_SIMPLE();


	return 0;
}


late_initcall(ipa_mpm_init);
MODULE_LICENSE("GPL v2");
MODULE_DESCRIPTION("MHI Proxy Manager Driver");