msm: ipa: page pool recycling enhancements

Added the following enhancements to page pool recycling logic.
1) Updated page pool implementation to use list instead of array.
2) Added provision to check for configurable number of list
elements when checking for free pages.

Change-Id: I01c43be5a169c1438fb29b8179854985e199d055
Signed-off-by: Chaitanya Pratapa <[email protected]>

drivers/platform/msm/ipa/ipa_v3/ipa.c

@@ -7796,6 +7796,8 @@ static int ipa3_pre_init(const struct ipa3_plat_drv_res *resource_p,
 	ipa3_ctx->stats.page_recycle_stats[0].tmp_alloc = 0;
 	ipa3_ctx->stats.page_recycle_stats[1].total_replenished = 0;
 	ipa3_ctx->stats.page_recycle_stats[1].tmp_alloc = 0;
+	memset(ipa3_ctx->stats.page_recycle_cnt, 0,
+		sizeof(ipa3_ctx->stats.page_recycle_cnt));
 	ipa3_ctx->skip_uc_pipe_reset = resource_p->skip_uc_pipe_reset;
 	ipa3_ctx->tethered_flow_control = resource_p->tethered_flow_control;
 	ipa3_ctx->ee = resource_p->ee;
@@ -7991,6 +7993,9 @@ static int ipa3_pre_init(const struct ipa3_plat_drv_res *resource_p,
 	/* Enable ipa3_ctx->enable_napi_chain */
 	ipa3_ctx->enable_napi_chain = 1;
 
+	/* Initialize Page poll threshold. */
+	ipa3_ctx->page_poll_threshold = IPA_PAGE_POLL_DEFAULT_THRESHOLD;
+
 	/* assume clock is on in virtual/emulation mode */
 	if (ipa3_ctx->ipa3_hw_mode == IPA_HW_MODE_VIRTUAL ||
 	    ipa3_ctx->ipa3_hw_mode == IPA_HW_MODE_EMULATION)
@@ -8650,7 +8655,7 @@ static int get_ipa_dts_configuration(struct platform_device *pdev,
 	ipa_drv_res->ipa_mhi_dynamic_config = false;
 	ipa_drv_res->use_64_bit_dma_mask = false;
 	ipa_drv_res->use_bw_vote = false;
-	ipa_drv_res->wan_rx_ring_size = IPA_GENERIC_RX_POOL_SZ;
+	ipa_drv_res->wan_rx_ring_size = IPA_GENERIC_RX_POOL_SZ_WAN;
 	ipa_drv_res->lan_rx_ring_size = IPA_GENERIC_RX_POOL_SZ;
 	ipa_drv_res->apply_rg10_wa = false;
 	ipa_drv_res->gsi_ch20_wa = false;

drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c

@@ -1476,24 +1476,38 @@ static ssize_t ipa3_read_odlstats(struct file *file, char __user *ubuf,
 	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, cnt);
 }
 
+
 static ssize_t ipa3_read_page_recycle_stats(struct file *file,
 		char __user *ubuf, size_t count, loff_t *ppos)
 {
 	int nbytes;
-	int cnt = 0;
+	int cnt = 0, i = 0, k = 0;
 
 	nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
 			"COAL : Total number of packets replenished =%llu\n"
+			"COAL : Number of page recycled packets  =%llu\n"
 			"COAL : Number of tmp alloc packets  =%llu\n"
 			"DEF  : Total number of packets replenished =%llu\n"
+			"DEF  : Number of page recycled packets =%llu\n"
 			"DEF  : Number of tmp alloc packets  =%llu\n",
 			ipa3_ctx->stats.page_recycle_stats[0].total_replenished,
+			ipa3_ctx->stats.page_recycle_stats[0].page_recycled,
 			ipa3_ctx->stats.page_recycle_stats[0].tmp_alloc,
 			ipa3_ctx->stats.page_recycle_stats[1].total_replenished,
+			ipa3_ctx->stats.page_recycle_stats[1].page_recycled,
 			ipa3_ctx->stats.page_recycle_stats[1].tmp_alloc);
 
 	cnt += nbytes;
 
+	for (k = 0; k < 2; k++) {
+		for (i = 0; i < ipa3_ctx->page_poll_threshold; i++) {
+			nbytes = scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN,
+				"COMMON  : Page replenish efficiency[%d][%d]  =%llu\n",
+				k, i, ipa3_ctx->stats.page_recycle_cnt[k][i]);
+			cnt += nbytes;
+		}
+	}
+
 	return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, cnt);
 }
 static ssize_t ipa3_read_wstats(struct file *file, char __user *ubuf,
@@ -2856,6 +2870,40 @@ static ssize_t ipa3_enable_ipc_low(struct file *file,
 	return count;
 }
 
+static ssize_t ipa3_read_page_poll_threshold(struct file *file,
+	char __user *buf, size_t count, loff_t *ppos) {
+
+	int nbytes;
+	nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN,
+				"Page Poll Threshold = %d\n",
+				ipa3_ctx->page_poll_threshold);
+	return simple_read_from_buffer(buf, count, ppos, dbg_buff, nbytes);
+
+}
+static ssize_t ipa3_write_page_poll_threshold(struct file *file,
+	const char __user *buf, size_t count, loff_t *ppos) {
+
+	int ret;
+	u8 page_poll_threshold =0;
+
+	if (count >= sizeof(dbg_buff))
+		return -EFAULT;
+
+	ret = kstrtou8_from_user(buf, count, 0, &page_poll_threshold);
+	if(ret)
+		return ret;
+
+	if(page_poll_threshold != 0 &&
+		page_poll_threshold <= IPA_PAGE_POLL_THRESHOLD_MAX)
+		ipa3_ctx->page_poll_threshold = page_poll_threshold;
+	else
+		IPAERR("Invalid value \n");
+
+	IPADBG("Updated page poll threshold = %d", ipa3_ctx->page_poll_threshold);
+
+	return count;
+}
+
 static const struct ipa3_debugfs_file debugfs_files[] = {
 	{
 		"gen_reg", IPA_READ_ONLY_MODE, NULL, {
@@ -3033,6 +3081,11 @@ static const struct ipa3_debugfs_file debugfs_files[] = {
 		"app_clk_vote_cnt", IPA_READ_ONLY_MODE, NULL, {
 			.read = ipa3_read_app_clk_vote,
 		}
+	}, {
+		"page_poll_threshold", IPA_READ_WRITE_MODE, NULL, {
+			.read = ipa3_read_page_poll_threshold,
+			.write = ipa3_write_page_poll_threshold,
+		}
 	},
 };
 

drivers/platform/msm/ipa/ipa_v3/ipa_dp.c

@@ -1215,7 +1215,8 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl)
 		snprintf(buff, IPA_RESOURCE_NAME_MAX, "iparepwq%d",
 				sys_in->client);
 		ep->sys->repl_wq = alloc_workqueue(buff,
-				WQ_MEM_RECLAIM | WQ_UNBOUND | WQ_SYSFS, 1);
+				WQ_MEM_RECLAIM | WQ_UNBOUND | WQ_SYSFS | WQ_HIGHPRI,
+				1);
 		if (!ep->sys->repl_wq) {
 			IPAERR("failed to create rep wq for client %d\n",
 					sys_in->client);
@@ -1397,12 +1398,7 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl)
 		atomic_set(&ep->sys->page_recycle_repl->pending, 0);
 		ep->sys->page_recycle_repl->capacity =
 				(ep->sys->rx_pool_sz + 1) * 2;
-
-		ep->sys->page_recycle_repl->cache =
-				kcalloc(ep->sys->page_recycle_repl->capacity,
-				sizeof(void *), GFP_KERNEL);
-		atomic_set(&ep->sys->page_recycle_repl->head_idx, 0);
-		atomic_set(&ep->sys->page_recycle_repl->tail_idx, 0);
+		INIT_LIST_HEAD(&ep->sys->page_recycle_repl->page_repl_head);
 		ep->sys->repl = kzalloc(sizeof(*ep->sys->repl), GFP_KERNEL);
 		if (!ep->sys->repl) {
 			IPAERR("failed to alloc repl for client %d\n",
@@ -2222,8 +2218,10 @@ static void ipa3_replenish_rx_page_cache(struct ipa3_sys_context *sys)
 			ipa_assert();
 			break;
 		}
+		INIT_LIST_HEAD(&rx_pkt->link);
 		rx_pkt->sys = sys;
-		sys->page_recycle_repl->cache[curr] = rx_pkt;
+		list_add_tail(&rx_pkt->link,
+			&sys->page_recycle_repl->page_repl_head);
 	}
 
 	return;
@@ -2291,6 +2289,34 @@ static inline void __trigger_repl_work(struct ipa3_sys_context *sys)
 	}
 }
 
+static struct ipa3_rx_pkt_wrapper * ipa3_get_free_page
+(
+	struct ipa3_sys_context *sys,
+	u32 stats_i
+)
+{
+	struct ipa3_rx_pkt_wrapper *rx_pkt = NULL;
+	struct ipa3_rx_pkt_wrapper *tmp = NULL;
+	struct page *cur_page;
+	int i = 0;
+	u8 LOOP_THRESHOLD = ipa3_ctx->page_poll_threshold;
+
+	list_for_each_entry_safe(rx_pkt, tmp,
+		&sys->page_recycle_repl->page_repl_head, link) {
+		if (i == LOOP_THRESHOLD)
+			break;
+		cur_page = rx_pkt->page_data.page;
+		if (page_ref_count(cur_page) == 1) {
+			/* Found a free page. */
+			page_ref_inc(cur_page);
+			list_del_init(&rx_pkt->link);
+			++ipa3_ctx->stats.page_recycle_cnt[stats_i][i];
+			return rx_pkt;
+		}
+		i++;
+	}
+	return NULL;
+}
 
 static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys)
 {
@@ -2298,10 +2324,8 @@ static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys)
 	int ret;
 	int rx_len_cached = 0;
 	struct gsi_xfer_elem gsi_xfer_elem_array[IPA_REPL_XFER_MAX];
-	u32 curr;
 	u32 curr_wq;
 	int idx = 0;
-	struct page *cur_page;
 	u32 stats_i = 0;
 
 	/* start replenish only when buffers go lower than the threshold */
@@ -2311,17 +2335,13 @@ static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys)
 
 	spin_lock_bh(&sys->spinlock);
 	rx_len_cached = sys->len;
-	curr = atomic_read(&sys->page_recycle_repl->head_idx);
 	curr_wq = atomic_read(&sys->repl->head_idx);
 
 	while (rx_len_cached < sys->rx_pool_sz) {
-		cur_page = sys->page_recycle_repl->cache[curr]->page_data.page;
-		/* Found an idle page that can be used */
-		if (page_ref_count(cur_page) == 1) {
-			page_ref_inc(cur_page);
-			rx_pkt = sys->page_recycle_repl->cache[curr];
-			curr = (++curr == sys->page_recycle_repl->capacity) ?
-								0 : curr;
+		/* check for an idle page that can be used */
+		if ((rx_pkt = ipa3_get_free_page(sys,stats_i)) != NULL) {
+			ipa3_ctx->stats.page_recycle_stats[stats_i].page_recycled++;
+
 		} else {
 			/*
 			 * Could not find idle page at curr index.
@@ -2371,7 +2391,6 @@ static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys)
 		/* ensure write is done before setting head index */
 		mb();
 		atomic_set(&sys->repl->head_idx, curr_wq);
-		atomic_set(&sys->page_recycle_repl->head_idx, curr);
 		sys->len = rx_len_cached;
 	} else {
 		/* we don't expect this will happen */
@@ -2886,19 +2905,14 @@ static void free_rx_page(void *chan_user_data, void *xfer_user_data)
 {
 	struct ipa3_rx_pkt_wrapper *rx_pkt = (struct ipa3_rx_pkt_wrapper *)
 		xfer_user_data;
-	struct ipa3_sys_context *sys = rx_pkt->sys;
-	int i;
 
-	for (i = 0; i < sys->page_recycle_repl->capacity; i++)
-		if (sys->page_recycle_repl->cache[i] == rx_pkt)
-			break;
-	if (i < sys->page_recycle_repl->capacity) {
+	if (!rx_pkt->page_data.is_tmp_alloc) {
+		list_del_init(&rx_pkt->link);
 		page_ref_dec(rx_pkt->page_data.page);
-		sys->page_recycle_repl->cache[i] = NULL;
 	}
 	dma_unmap_page(ipa3_ctx->pdev, rx_pkt->page_data.dma_addr,
 		rx_pkt->len, DMA_FROM_DEVICE);
-	__free_pages(rx_pkt->page_data.page, sys->page_order);
+	__free_pages(rx_pkt->page_data.page, rx_pkt->sys->page_order);
 	kmem_cache_free(ipa3_ctx->rx_pkt_wrapper_cache, rx_pkt);
 }
 
@@ -2912,7 +2926,6 @@ static void ipa3_cleanup_rx(struct ipa3_sys_context *sys)
 	struct ipa3_rx_pkt_wrapper *r;
 	u32 head;
 	u32 tail;
-	int i;
 
 	/*
 	 * buffers not consumed by gsi are cleaned up using cleanup callback
@@ -2960,20 +2973,19 @@ static void ipa3_cleanup_rx(struct ipa3_sys_context *sys)
 		kfree(sys->repl);
 	}
 	if (sys->page_recycle_repl) {
-		for (i = 0; i < sys->page_recycle_repl->capacity; i++) {
-			rx_pkt = sys->page_recycle_repl->cache[i];
-			if (rx_pkt) {
-				dma_unmap_page(ipa3_ctx->pdev,
-					rx_pkt->page_data.dma_addr,
-					rx_pkt->len,
-					DMA_FROM_DEVICE);
-				__free_pages(rx_pkt->page_data.page, sys->page_order);
-				kmem_cache_free(
-					ipa3_ctx->rx_pkt_wrapper_cache,
-					rx_pkt);
-			}
+		list_for_each_entry_safe(rx_pkt, r,
+		&sys->page_recycle_repl->page_repl_head, link) {
+			list_del(&rx_pkt->link);
+			dma_unmap_page(ipa3_ctx->pdev,
+				rx_pkt->page_data.dma_addr,
+				rx_pkt->len,
+				DMA_FROM_DEVICE);
+			__free_pages(rx_pkt->page_data.page,
+				sys->page_order);
+			kmem_cache_free(
+				ipa3_ctx->rx_pkt_wrapper_cache,
+				rx_pkt);
 		}
-		kfree(sys->page_recycle_repl->cache);
 		kfree(sys->page_recycle_repl);
 	}
 }
@@ -3760,6 +3772,11 @@ static struct sk_buff *handle_page_completion(struct gsi_chan_xfer_notify
 		IPAERR("notify->veid > GSI_VEID_MAX\n");
 		if (!rx_page.is_tmp_alloc) {
 			init_page_count(rx_page.page);
+			spin_lock_bh(&rx_pkt->sys->spinlock);
+			/* Add the element to head. */
+			list_add(&rx_pkt->link,
+				&rx_pkt->sys->page_recycle_repl->page_repl_head);
+			spin_unlock_bh(&rx_pkt->sys->spinlock);
 		} else {
 			dma_unmap_page(ipa3_ctx->pdev, rx_page.dma_addr,
 					rx_pkt->len, DMA_FROM_DEVICE);
@@ -3781,9 +3798,14 @@ static struct sk_buff *handle_page_completion(struct gsi_chan_xfer_notify
 		rx_skb = alloc_skb(0, GFP_ATOMIC);
 		if (unlikely(!rx_skb)) {
 			IPAERR("skb alloc failure\n");
-			list_del(&rx_pkt->link);
+			list_del_init(&rx_pkt->link);
 			if (!rx_page.is_tmp_alloc) {
 				init_page_count(rx_page.page);
+				spin_lock_bh(&rx_pkt->sys->spinlock);
+				/* Add the element to head. */
+				list_add(&rx_pkt->link,
+					&rx_pkt->sys->page_recycle_repl->page_repl_head);
+				spin_unlock_bh(&rx_pkt->sys->spinlock);
 			} else {
 				dma_unmap_page(ipa3_ctx->pdev, rx_page.dma_addr,
 					rx_pkt->len, DMA_FROM_DEVICE);
@@ -3797,14 +3819,20 @@ static struct sk_buff *handle_page_completion(struct gsi_chan_xfer_notify
 			rx_page = rx_pkt->page_data;
 			size = rx_pkt->data_len;
 
-			list_del(&rx_pkt->link);
-			if (rx_page.is_tmp_alloc)
+			list_del_init(&rx_pkt->link);
+			if (rx_page.is_tmp_alloc) {
 				dma_unmap_page(ipa3_ctx->pdev, rx_page.dma_addr,
 					rx_pkt->len, DMA_FROM_DEVICE);
-			else
+			} else {
+				spin_lock_bh(&rx_pkt->sys->spinlock);
+				/* Add the element back to tail. */
+				list_add_tail(&rx_pkt->link,
+					&rx_pkt->sys->page_recycle_repl->page_repl_head);
+				spin_unlock_bh(&rx_pkt->sys->spinlock);
 				dma_sync_single_for_cpu(ipa3_ctx->pdev,
 					rx_page.dma_addr,
 					rx_pkt->len, DMA_FROM_DEVICE);
+			}
 			rx_pkt->sys->free_rx_wrapper(rx_pkt);
 
 			skb_add_rx_frag(rx_skb,

drivers/platform/msm/ipa/ipa_v3/ipa_i.h

@@ -65,6 +65,7 @@
 #define IPA_QMAP_HEADER_LENGTH (4)
 #define IPA_DL_CHECKSUM_LENGTH (8)
 #define IPA_NUM_DESC_PER_SW_TX (3)
+#define IPA_GENERIC_RX_POOL_SZ_WAN 224
 #define IPA_GENERIC_RX_POOL_SZ 192
 #define IPA_UC_FINISH_MAX 6
 #define IPA_UC_WAIT_MIN_SLEEP 1000
@@ -102,6 +103,10 @@ enum {
 
 #define IPA_WAN_AGGR_PKT_CNT 1
 
+#define IPA_PAGE_POLL_DEFAULT_THRESHOLD 15
+#define IPA_PAGE_POLL_THRESHOLD_MAX 30
+
+
 #define IPADBG(fmt, args...) \
 	do { \
 		pr_debug(DRV_NAME " %s:%d " fmt, __func__, __LINE__, ## args);\
@@ -1067,6 +1072,12 @@ struct ipa3_repl_ctx {
 	atomic_t pending;
 };
 
+struct ipa3_page_repl_ctx {
+	struct list_head page_repl_head;
+	u32 capacity;
+	atomic_t pending;
+};
+
 /**
  * struct ipa3_sys_context - IPA GPI pipes context
  * @head_desc_list: header descriptors list
@@ -1110,7 +1121,7 @@ struct ipa3_sys_context {
 	struct work_struct repl_work;
 	void (*repl_hdlr)(struct ipa3_sys_context *sys);
 	struct ipa3_repl_ctx *repl;
-	struct ipa3_repl_ctx *page_recycle_repl;
+	struct ipa3_page_repl_ctx *page_recycle_repl;
 	u32 pkt_sent;
 	struct napi_struct *napi_obj;
 	struct list_head pending_pkts[GSI_VEID_MAX];
@@ -1449,8 +1460,10 @@ enum ipa3_config_this_ep {
 
 struct ipa3_page_recycle_stats {
 	u64 total_replenished;
+	u64 page_recycled;
 	u64 tmp_alloc;
 };
+
 struct ipa3_stats {
 	u32 tx_sw_pkts;
 	u32 tx_hw_pkts;
@@ -1475,6 +1488,7 @@ struct ipa3_stats {
 	u32 tx_non_linear;
 	u32 rx_page_drop_cnt;
 	struct ipa3_page_recycle_stats page_recycle_stats[2];
+	u64 page_recycle_cnt[2][IPA_PAGE_POLL_THRESHOLD_MAX];
 };
 
 /* offset for each stats */
@@ -2229,6 +2243,7 @@ struct ipa3_context {
 	u16 ulso_ip_id_min;
 	u16 ulso_ip_id_max;
 	bool use_pm_wrapper;
+	u8 page_poll_threshold;
 };
 
 struct ipa3_plat_drv_res {