diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa.c b/drivers/platform/msm/ipa/ipa_v3/ipa.c index b5621a9f68..f85208c51b 100644 --- a/drivers/platform/msm/ipa/ipa_v3/ipa.c +++ b/drivers/platform/msm/ipa/ipa_v3/ipa.c @@ -7796,6 +7796,8 @@ static int ipa3_pre_init(const struct ipa3_plat_drv_res *resource_p, ipa3_ctx->stats.page_recycle_stats[0].tmp_alloc = 0; ipa3_ctx->stats.page_recycle_stats[1].total_replenished = 0; ipa3_ctx->stats.page_recycle_stats[1].tmp_alloc = 0; + memset(ipa3_ctx->stats.page_recycle_cnt, 0, + sizeof(ipa3_ctx->stats.page_recycle_cnt)); ipa3_ctx->skip_uc_pipe_reset = resource_p->skip_uc_pipe_reset; ipa3_ctx->tethered_flow_control = resource_p->tethered_flow_control; ipa3_ctx->ee = resource_p->ee; @@ -7991,6 +7993,9 @@ static int ipa3_pre_init(const struct ipa3_plat_drv_res *resource_p, /* Enable ipa3_ctx->enable_napi_chain */ ipa3_ctx->enable_napi_chain = 1; + /* Initialize Page poll threshold. */ + ipa3_ctx->page_poll_threshold = IPA_PAGE_POLL_DEFAULT_THRESHOLD; + /* assume clock is on in virtual/emulation mode */ if (ipa3_ctx->ipa3_hw_mode == IPA_HW_MODE_VIRTUAL || ipa3_ctx->ipa3_hw_mode == IPA_HW_MODE_EMULATION) @@ -8650,7 +8655,7 @@ static int get_ipa_dts_configuration(struct platform_device *pdev, ipa_drv_res->ipa_mhi_dynamic_config = false; ipa_drv_res->use_64_bit_dma_mask = false; ipa_drv_res->use_bw_vote = false; - ipa_drv_res->wan_rx_ring_size = IPA_GENERIC_RX_POOL_SZ; + ipa_drv_res->wan_rx_ring_size = IPA_GENERIC_RX_POOL_SZ_WAN; ipa_drv_res->lan_rx_ring_size = IPA_GENERIC_RX_POOL_SZ; ipa_drv_res->apply_rg10_wa = false; ipa_drv_res->gsi_ch20_wa = false; diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c b/drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c index a77ef6cc4c..28c839a1bb 100644 --- a/drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c +++ b/drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c @@ -1476,24 +1476,38 @@ static ssize_t ipa3_read_odlstats(struct file *file, char __user *ubuf, return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, cnt); } + static ssize_t ipa3_read_page_recycle_stats(struct file *file, char __user *ubuf, size_t count, loff_t *ppos) { int nbytes; - int cnt = 0; + int cnt = 0, i = 0, k = 0; nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN, "COAL : Total number of packets replenished =%llu\n" + "COAL : Number of page recycled packets =%llu\n" "COAL : Number of tmp alloc packets =%llu\n" "DEF : Total number of packets replenished =%llu\n" + "DEF : Number of page recycled packets =%llu\n" "DEF : Number of tmp alloc packets =%llu\n", ipa3_ctx->stats.page_recycle_stats[0].total_replenished, + ipa3_ctx->stats.page_recycle_stats[0].page_recycled, ipa3_ctx->stats.page_recycle_stats[0].tmp_alloc, ipa3_ctx->stats.page_recycle_stats[1].total_replenished, + ipa3_ctx->stats.page_recycle_stats[1].page_recycled, ipa3_ctx->stats.page_recycle_stats[1].tmp_alloc); cnt += nbytes; + for (k = 0; k < 2; k++) { + for (i = 0; i < ipa3_ctx->page_poll_threshold; i++) { + nbytes = scnprintf(dbg_buff + cnt, IPA_MAX_MSG_LEN, + "COMMON : Page replenish efficiency[%d][%d] =%llu\n", + k, i, ipa3_ctx->stats.page_recycle_cnt[k][i]); + cnt += nbytes; + } + } + return simple_read_from_buffer(ubuf, count, ppos, dbg_buff, cnt); } static ssize_t ipa3_read_wstats(struct file *file, char __user *ubuf, @@ -2856,6 +2870,40 @@ static ssize_t ipa3_enable_ipc_low(struct file *file, return count; } +static ssize_t ipa3_read_page_poll_threshold(struct file *file, + char __user *buf, size_t count, loff_t *ppos) { + + int nbytes; + nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN, + "Page Poll Threshold = %d\n", + ipa3_ctx->page_poll_threshold); + return simple_read_from_buffer(buf, count, ppos, dbg_buff, nbytes); + +} +static ssize_t ipa3_write_page_poll_threshold(struct file *file, + const char __user *buf, size_t count, loff_t *ppos) { + + int ret; + u8 page_poll_threshold =0; + + if (count >= sizeof(dbg_buff)) + return -EFAULT; + + ret = kstrtou8_from_user(buf, count, 0, &page_poll_threshold); + if(ret) + return ret; + + if(page_poll_threshold != 0 && + page_poll_threshold <= IPA_PAGE_POLL_THRESHOLD_MAX) + ipa3_ctx->page_poll_threshold = page_poll_threshold; + else + IPAERR("Invalid value \n"); + + IPADBG("Updated page poll threshold = %d", ipa3_ctx->page_poll_threshold); + + return count; +} + static const struct ipa3_debugfs_file debugfs_files[] = { { "gen_reg", IPA_READ_ONLY_MODE, NULL, { @@ -3033,6 +3081,11 @@ static const struct ipa3_debugfs_file debugfs_files[] = { "app_clk_vote_cnt", IPA_READ_ONLY_MODE, NULL, { .read = ipa3_read_app_clk_vote, } + }, { + "page_poll_threshold", IPA_READ_WRITE_MODE, NULL, { + .read = ipa3_read_page_poll_threshold, + .write = ipa3_write_page_poll_threshold, + } }, }; diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c b/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c index 7b82542a8d..2d1e19ff14 100644 --- a/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c +++ b/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c @@ -1215,7 +1215,8 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl) snprintf(buff, IPA_RESOURCE_NAME_MAX, "iparepwq%d", sys_in->client); ep->sys->repl_wq = alloc_workqueue(buff, - WQ_MEM_RECLAIM | WQ_UNBOUND | WQ_SYSFS, 1); + WQ_MEM_RECLAIM | WQ_UNBOUND | WQ_SYSFS | WQ_HIGHPRI, + 1); if (!ep->sys->repl_wq) { IPAERR("failed to create rep wq for client %d\n", sys_in->client); @@ -1397,12 +1398,7 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl) atomic_set(&ep->sys->page_recycle_repl->pending, 0); ep->sys->page_recycle_repl->capacity = (ep->sys->rx_pool_sz + 1) * 2; - - ep->sys->page_recycle_repl->cache = - kcalloc(ep->sys->page_recycle_repl->capacity, - sizeof(void *), GFP_KERNEL); - atomic_set(&ep->sys->page_recycle_repl->head_idx, 0); - atomic_set(&ep->sys->page_recycle_repl->tail_idx, 0); + INIT_LIST_HEAD(&ep->sys->page_recycle_repl->page_repl_head); ep->sys->repl = kzalloc(sizeof(*ep->sys->repl), GFP_KERNEL); if (!ep->sys->repl) { IPAERR("failed to alloc repl for client %d\n", @@ -2222,8 +2218,10 @@ static void ipa3_replenish_rx_page_cache(struct ipa3_sys_context *sys) ipa_assert(); break; } + INIT_LIST_HEAD(&rx_pkt->link); rx_pkt->sys = sys; - sys->page_recycle_repl->cache[curr] = rx_pkt; + list_add_tail(&rx_pkt->link, + &sys->page_recycle_repl->page_repl_head); } return; @@ -2291,6 +2289,34 @@ static inline void __trigger_repl_work(struct ipa3_sys_context *sys) } } +static struct ipa3_rx_pkt_wrapper * ipa3_get_free_page +( + struct ipa3_sys_context *sys, + u32 stats_i +) +{ + struct ipa3_rx_pkt_wrapper *rx_pkt = NULL; + struct ipa3_rx_pkt_wrapper *tmp = NULL; + struct page *cur_page; + int i = 0; + u8 LOOP_THRESHOLD = ipa3_ctx->page_poll_threshold; + + list_for_each_entry_safe(rx_pkt, tmp, + &sys->page_recycle_repl->page_repl_head, link) { + if (i == LOOP_THRESHOLD) + break; + cur_page = rx_pkt->page_data.page; + if (page_ref_count(cur_page) == 1) { + /* Found a free page. */ + page_ref_inc(cur_page); + list_del_init(&rx_pkt->link); + ++ipa3_ctx->stats.page_recycle_cnt[stats_i][i]; + return rx_pkt; + } + i++; + } + return NULL; +} static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys) { @@ -2298,10 +2324,8 @@ static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys) int ret; int rx_len_cached = 0; struct gsi_xfer_elem gsi_xfer_elem_array[IPA_REPL_XFER_MAX]; - u32 curr; u32 curr_wq; int idx = 0; - struct page *cur_page; u32 stats_i = 0; /* start replenish only when buffers go lower than the threshold */ @@ -2311,17 +2335,13 @@ static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys) spin_lock_bh(&sys->spinlock); rx_len_cached = sys->len; - curr = atomic_read(&sys->page_recycle_repl->head_idx); curr_wq = atomic_read(&sys->repl->head_idx); while (rx_len_cached < sys->rx_pool_sz) { - cur_page = sys->page_recycle_repl->cache[curr]->page_data.page; - /* Found an idle page that can be used */ - if (page_ref_count(cur_page) == 1) { - page_ref_inc(cur_page); - rx_pkt = sys->page_recycle_repl->cache[curr]; - curr = (++curr == sys->page_recycle_repl->capacity) ? - 0 : curr; + /* check for an idle page that can be used */ + if ((rx_pkt = ipa3_get_free_page(sys,stats_i)) != NULL) { + ipa3_ctx->stats.page_recycle_stats[stats_i].page_recycled++; + } else { /* * Could not find idle page at curr index. @@ -2371,7 +2391,6 @@ static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys) /* ensure write is done before setting head index */ mb(); atomic_set(&sys->repl->head_idx, curr_wq); - atomic_set(&sys->page_recycle_repl->head_idx, curr); sys->len = rx_len_cached; } else { /* we don't expect this will happen */ @@ -2886,19 +2905,14 @@ static void free_rx_page(void *chan_user_data, void *xfer_user_data) { struct ipa3_rx_pkt_wrapper *rx_pkt = (struct ipa3_rx_pkt_wrapper *) xfer_user_data; - struct ipa3_sys_context *sys = rx_pkt->sys; - int i; - for (i = 0; i < sys->page_recycle_repl->capacity; i++) - if (sys->page_recycle_repl->cache[i] == rx_pkt) - break; - if (i < sys->page_recycle_repl->capacity) { + if (!rx_pkt->page_data.is_tmp_alloc) { + list_del_init(&rx_pkt->link); page_ref_dec(rx_pkt->page_data.page); - sys->page_recycle_repl->cache[i] = NULL; } dma_unmap_page(ipa3_ctx->pdev, rx_pkt->page_data.dma_addr, rx_pkt->len, DMA_FROM_DEVICE); - __free_pages(rx_pkt->page_data.page, sys->page_order); + __free_pages(rx_pkt->page_data.page, rx_pkt->sys->page_order); kmem_cache_free(ipa3_ctx->rx_pkt_wrapper_cache, rx_pkt); } @@ -2912,7 +2926,6 @@ static void ipa3_cleanup_rx(struct ipa3_sys_context *sys) struct ipa3_rx_pkt_wrapper *r; u32 head; u32 tail; - int i; /* * buffers not consumed by gsi are cleaned up using cleanup callback @@ -2960,20 +2973,19 @@ static void ipa3_cleanup_rx(struct ipa3_sys_context *sys) kfree(sys->repl); } if (sys->page_recycle_repl) { - for (i = 0; i < sys->page_recycle_repl->capacity; i++) { - rx_pkt = sys->page_recycle_repl->cache[i]; - if (rx_pkt) { - dma_unmap_page(ipa3_ctx->pdev, - rx_pkt->page_data.dma_addr, - rx_pkt->len, - DMA_FROM_DEVICE); - __free_pages(rx_pkt->page_data.page, sys->page_order); - kmem_cache_free( - ipa3_ctx->rx_pkt_wrapper_cache, - rx_pkt); - } + list_for_each_entry_safe(rx_pkt, r, + &sys->page_recycle_repl->page_repl_head, link) { + list_del(&rx_pkt->link); + dma_unmap_page(ipa3_ctx->pdev, + rx_pkt->page_data.dma_addr, + rx_pkt->len, + DMA_FROM_DEVICE); + __free_pages(rx_pkt->page_data.page, + sys->page_order); + kmem_cache_free( + ipa3_ctx->rx_pkt_wrapper_cache, + rx_pkt); } - kfree(sys->page_recycle_repl->cache); kfree(sys->page_recycle_repl); } } @@ -3760,6 +3772,11 @@ static struct sk_buff *handle_page_completion(struct gsi_chan_xfer_notify IPAERR("notify->veid > GSI_VEID_MAX\n"); if (!rx_page.is_tmp_alloc) { init_page_count(rx_page.page); + spin_lock_bh(&rx_pkt->sys->spinlock); + /* Add the element to head. */ + list_add(&rx_pkt->link, + &rx_pkt->sys->page_recycle_repl->page_repl_head); + spin_unlock_bh(&rx_pkt->sys->spinlock); } else { dma_unmap_page(ipa3_ctx->pdev, rx_page.dma_addr, rx_pkt->len, DMA_FROM_DEVICE); @@ -3781,9 +3798,14 @@ static struct sk_buff *handle_page_completion(struct gsi_chan_xfer_notify rx_skb = alloc_skb(0, GFP_ATOMIC); if (unlikely(!rx_skb)) { IPAERR("skb alloc failure\n"); - list_del(&rx_pkt->link); + list_del_init(&rx_pkt->link); if (!rx_page.is_tmp_alloc) { init_page_count(rx_page.page); + spin_lock_bh(&rx_pkt->sys->spinlock); + /* Add the element to head. */ + list_add(&rx_pkt->link, + &rx_pkt->sys->page_recycle_repl->page_repl_head); + spin_unlock_bh(&rx_pkt->sys->spinlock); } else { dma_unmap_page(ipa3_ctx->pdev, rx_page.dma_addr, rx_pkt->len, DMA_FROM_DEVICE); @@ -3797,14 +3819,20 @@ static struct sk_buff *handle_page_completion(struct gsi_chan_xfer_notify rx_page = rx_pkt->page_data; size = rx_pkt->data_len; - list_del(&rx_pkt->link); - if (rx_page.is_tmp_alloc) + list_del_init(&rx_pkt->link); + if (rx_page.is_tmp_alloc) { dma_unmap_page(ipa3_ctx->pdev, rx_page.dma_addr, rx_pkt->len, DMA_FROM_DEVICE); - else + } else { + spin_lock_bh(&rx_pkt->sys->spinlock); + /* Add the element back to tail. */ + list_add_tail(&rx_pkt->link, + &rx_pkt->sys->page_recycle_repl->page_repl_head); + spin_unlock_bh(&rx_pkt->sys->spinlock); dma_sync_single_for_cpu(ipa3_ctx->pdev, rx_page.dma_addr, rx_pkt->len, DMA_FROM_DEVICE); + } rx_pkt->sys->free_rx_wrapper(rx_pkt); skb_add_rx_frag(rx_skb, diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_i.h b/drivers/platform/msm/ipa/ipa_v3/ipa_i.h index 8d54df3ec5..04d5278847 100644 --- a/drivers/platform/msm/ipa/ipa_v3/ipa_i.h +++ b/drivers/platform/msm/ipa/ipa_v3/ipa_i.h @@ -65,6 +65,7 @@ #define IPA_QMAP_HEADER_LENGTH (4) #define IPA_DL_CHECKSUM_LENGTH (8) #define IPA_NUM_DESC_PER_SW_TX (3) +#define IPA_GENERIC_RX_POOL_SZ_WAN 224 #define IPA_GENERIC_RX_POOL_SZ 192 #define IPA_UC_FINISH_MAX 6 #define IPA_UC_WAIT_MIN_SLEEP 1000 @@ -102,6 +103,10 @@ enum { #define IPA_WAN_AGGR_PKT_CNT 1 +#define IPA_PAGE_POLL_DEFAULT_THRESHOLD 15 +#define IPA_PAGE_POLL_THRESHOLD_MAX 30 + + #define IPADBG(fmt, args...) \ do { \ pr_debug(DRV_NAME " %s:%d " fmt, __func__, __LINE__, ## args);\ @@ -1067,6 +1072,12 @@ struct ipa3_repl_ctx { atomic_t pending; }; +struct ipa3_page_repl_ctx { + struct list_head page_repl_head; + u32 capacity; + atomic_t pending; +}; + /** * struct ipa3_sys_context - IPA GPI pipes context * @head_desc_list: header descriptors list @@ -1110,7 +1121,7 @@ struct ipa3_sys_context { struct work_struct repl_work; void (*repl_hdlr)(struct ipa3_sys_context *sys); struct ipa3_repl_ctx *repl; - struct ipa3_repl_ctx *page_recycle_repl; + struct ipa3_page_repl_ctx *page_recycle_repl; u32 pkt_sent; struct napi_struct *napi_obj; struct list_head pending_pkts[GSI_VEID_MAX]; @@ -1449,8 +1460,10 @@ enum ipa3_config_this_ep { struct ipa3_page_recycle_stats { u64 total_replenished; + u64 page_recycled; u64 tmp_alloc; }; + struct ipa3_stats { u32 tx_sw_pkts; u32 tx_hw_pkts; @@ -1475,6 +1488,7 @@ struct ipa3_stats { u32 tx_non_linear; u32 rx_page_drop_cnt; struct ipa3_page_recycle_stats page_recycle_stats[2]; + u64 page_recycle_cnt[2][IPA_PAGE_POLL_THRESHOLD_MAX]; }; /* offset for each stats */ @@ -2229,6 +2243,7 @@ struct ipa3_context { u16 ulso_ip_id_min; u16 ulso_ip_id_max; bool use_pm_wrapper; + u8 page_poll_threshold; }; struct ipa3_plat_drv_res {