diff --git a/dp/inc/cdp_txrx_stats_struct.h b/dp/inc/cdp_txrx_stats_struct.h index f278f4ca01..8329ee1073 100644 --- a/dp/inc/cdp_txrx_stats_struct.h +++ b/dp/inc/cdp_txrx_stats_struct.h @@ -1784,6 +1784,12 @@ struct cdp_pdev_stats { uint64_t tx_ppdu_proc; uint64_t ack_ba_comes_twice; uint64_t ppdu_drop; + + struct { + uint64_t num_bufs_consumed; + uint64_t num_pool_bufs_replenish; + uint64_t num_bufs_alloc_success; + } rx_buffer_pool; }; /* struct cdp_peer_hmwds_ast_add_status - hmwds peer ast add status diff --git a/dp/wifi3.0/dp_rx.c b/dp/wifi3.0/dp_rx.c index d4ec584381..3207a67790 100644 --- a/dp/wifi3.0/dp_rx.c +++ b/dp/wifi3.0/dp_rx.c @@ -33,6 +33,7 @@ #include "dp_txrx_wds.h" #endif #include "dp_hist.h" +#include "dp_rx_buffer_pool.h" #ifdef ATH_RX_PRI_SAVE #define DP_RX_TID_SAVE(_nbuf, _tid) \ @@ -172,7 +173,6 @@ QDF_STATUS __dp_rx_buffers_replenish(struct dp_soc *dp_soc, uint32_t mac_id, union dp_rx_desc_list_elem_t *next; QDF_STATUS ret; uint16_t buf_size = rx_desc_pool->buf_size; - uint8_t buf_alignment = rx_desc_pool->buf_alignment; void *rxdma_srng; @@ -239,11 +239,9 @@ QDF_STATUS __dp_rx_buffers_replenish(struct dp_soc *dp_soc, uint32_t mac_id, count = 0; while (count < num_req_buffers) { - rx_netbuf = qdf_nbuf_alloc(dp_soc->osdev, - buf_size, - RX_BUFFER_RESERVATION, - buf_alignment, - FALSE); + rx_netbuf = dp_rx_buffer_pool_nbuf_alloc(dp_soc, mac_id, + rx_desc_pool, + num_entries_avail); if (qdf_unlikely(!rx_netbuf)) { DP_STATS_INC(dp_pdev, replenish.nbuf_alloc_fail, 1); @@ -254,7 +252,7 @@ QDF_STATUS __dp_rx_buffers_replenish(struct dp_soc *dp_soc, uint32_t mac_id, QDF_DMA_FROM_DEVICE, buf_size); if (qdf_unlikely(QDF_IS_STATUS_ERROR(ret))) { - qdf_nbuf_free(rx_netbuf); + dp_rx_buffer_pool_nbuf_free(dp_soc, rx_netbuf, mac_id); DP_STATS_INC(dp_pdev, replenish.map_err, 1); continue; } @@ -2004,6 +2002,8 @@ uint32_t dp_rx_process(struct dp_intr *int_ctx, hal_ring_handle_t hal_ring_hdl, uint32_t num_entries = 0; struct hal_rx_msdu_metadata msdu_metadata; QDF_STATUS status; + qdf_nbuf_t ebuf_head; + qdf_nbuf_t ebuf_tail; DP_HIST_INIT(); @@ -2025,6 +2025,8 @@ more_data: peer = NULL; vdev = NULL; num_rx_bufs_reaped = 0; + ebuf_head = NULL; + ebuf_tail = NULL; qdf_mem_zero(rx_bufs_reaped, sizeof(rx_bufs_reaped)); qdf_mem_zero(&mpdu_desc_info, sizeof(mpdu_desc_info)); @@ -2088,7 +2090,8 @@ more_data: QDF_DMA_FROM_DEVICE, RX_DATA_BUFFER_SIZE); rx_desc->unmapped = 1; - qdf_nbuf_free(rx_desc->nbuf); + dp_rx_buffer_pool_nbuf_free(soc, rx_desc->nbuf, + rx_desc->pool_id); dp_rx_add_to_free_desc_list( &head[rx_desc->pool_id], &tail[rx_desc->pool_id], @@ -2231,8 +2234,8 @@ more_data: QDF_DMA_FROM_DEVICE, rx_desc_pool->buf_size); rx_desc->unmapped = 1; - DP_RX_LIST_APPEND(nbuf_head, nbuf_tail, rx_desc->nbuf); - + DP_RX_PROCESS_NBUF(soc, nbuf_head, nbuf_tail, ebuf_head, + ebuf_tail, rx_desc); /* * if continuation bit is set then we have MSDU spread * across multiple buffers, let us not decrement quota @@ -2972,6 +2975,11 @@ dp_rx_pdev_buffers_alloc(struct dp_pdev *pdev) rx_desc_pool = &soc->rx_desc_buf[mac_for_pdev]; + /* Initialize RX buffer pool which will be + * used during low memory conditions + */ + dp_rx_buffer_pool_init(soc, mac_for_pdev); + return dp_pdev_rx_buffers_attach(soc, mac_for_pdev, dp_rxdma_srng, rx_desc_pool, rxdma_entries - 1); } @@ -2991,6 +2999,7 @@ dp_rx_pdev_buffers_free(struct dp_pdev *pdev) rx_desc_pool = &soc->rx_desc_buf[mac_for_pdev]; dp_rx_desc_nbuf_free(soc, rx_desc_pool); + dp_rx_buffer_pool_deinit(soc, mac_for_pdev); } /* diff --git a/dp/wifi3.0/dp_rx.h b/dp/wifi3.0/dp_rx.h index f452efd763..669c119247 100644 --- a/dp/wifi3.0/dp_rx.h +++ b/dp/wifi3.0/dp_rx.h @@ -1422,4 +1422,26 @@ static inline void dp_rx_wbm_sg_list_deinit(struct dp_soc *soc) dp_rx_wbm_sg_list_reset(soc); } } + +#ifdef WLAN_FEATURE_RX_PREALLOC_BUFFER_POOL +#define DP_RX_PROCESS_NBUF(soc, head, tail, ebuf_head, ebuf_tail, rx_desc) \ + do { \ + if (!soc->rx_buff_pool[rx_desc->pool_id].is_initialized) { \ + DP_RX_LIST_APPEND(head, tail, rx_desc->nbuf); \ + break; \ + } \ + DP_RX_LIST_APPEND(ebuf_head, ebuf_tail, rx_desc->nbuf); \ + if (!qdf_nbuf_is_rx_chfrag_cont(rx_desc->nbuf)) { \ + if (!dp_rx_buffer_pool_refill(soc, ebuf_head, \ + rx_desc->pool_id)) \ + DP_RX_MERGE_TWO_LIST(head, tail, \ + ebuf_head, ebuf_tail);\ + ebuf_head = NULL; \ + ebuf_tail = NULL; \ + } \ + } while (0) +#else +#define DP_RX_PROCESS_NBUF(soc, head, tail, ebuf_head, ebuf_tail, rx_desc) \ + DP_RX_LIST_APPEND(head, tail, rx_desc->nbuf) +#endif /* WLAN_FEATURE_RX_PREALLOC_BUFFER_POOL */ #endif /* _DP_RX_H */ diff --git a/dp/wifi3.0/dp_rx_defrag.c b/dp/wifi3.0/dp_rx_defrag.c index cddc4aef35..52476c78e6 100644 --- a/dp/wifi3.0/dp_rx_defrag.c +++ b/dp/wifi3.0/dp_rx_defrag.c @@ -28,6 +28,7 @@ #include /* LLC_SNAP_HDR_LEN */ #include "dp_rx_defrag.h" #include "dp_ipa.h" +#include "dp_rx_buffer_pool.h" const struct dp_rx_defrag_cipher dp_f_ccmp = { "AES-CCM", @@ -1434,6 +1435,11 @@ dp_rx_defrag_store_fragment(struct dp_soc *soc, goto discard_frag; } + if (dp_rx_buffer_pool_refill(soc, frag, rx_desc->pool_id)) { + /* fragment queued back to the pool, free the link desc */ + goto err_free_desc; + } + msdu_len = hal_rx_msdu_start_msdu_len_get(rx_desc->rx_buf_start); qdf_nbuf_set_pktlen(frag, (msdu_len + RX_PKT_TLVS_LEN)); @@ -1649,6 +1655,7 @@ dp_rx_defrag_store_fragment(struct dp_soc *soc, discard_frag: qdf_nbuf_free(frag); +err_free_desc: dp_rx_add_to_free_desc_list(head, tail, rx_desc); if (dp_rx_link_desc_return(soc, ring_desc, HAL_BM_ACTION_PUT_IN_IDLE_LIST) != diff --git a/dp/wifi3.0/dp_rx_err.c b/dp/wifi3.0/dp_rx_err.c index 65801f162d..e7c5603c80 100644 --- a/dp/wifi3.0/dp_rx_err.c +++ b/dp/wifi3.0/dp_rx_err.c @@ -31,6 +31,7 @@ #endif #include /* LLC_SNAP_HDR_LEN */ #include "qdf_net_types.h" +#include "dp_rx_buffer_pool.h" /* Max buffer in invalid peer SG list*/ #define DP_MAX_INVALID_BUFFERS 10 @@ -323,7 +324,7 @@ dp_rx_msdus_drop(struct dp_soc *soc, hal_ring_desc_t ring_desc, hal_rx_print_pn(soc->hal_soc, rx_tlv_hdr); /* Just free the buffers */ - qdf_nbuf_free(rx_desc->nbuf); + dp_rx_buffer_pool_nbuf_free(soc, rx_desc->nbuf, *mac_id); dp_rx_add_to_free_desc_list(&pdev->free_list_head, &pdev->free_list_tail, rx_desc); @@ -515,6 +516,12 @@ more_msdu_link_desc: HAL_MSDU_F_MSDU_CONTINUATION)) continue; + if (dp_rx_buffer_pool_refill(soc, head_nbuf, + rx_desc->pool_id)) { + /* MSDU queued back to the pool */ + goto process_next_msdu; + } + rx_tlv_hdr_first = qdf_nbuf_data(head_nbuf); rx_tlv_hdr_last = qdf_nbuf_data(tail_nbuf); @@ -547,6 +554,7 @@ more_msdu_link_desc: qdf_nbuf_free(nbuf); } +process_next_msdu: msdu_processed++; head_nbuf = NULL; tail_nbuf = NULL; @@ -1852,13 +1860,19 @@ dp_rx_wbm_err_process(struct dp_intr *int_ctx, struct dp_soc *soc, soc->wbm_sg_param.wbm_sg_nbuf_tail, nbuf); if (process_sg_buf) { - DP_RX_MERGE_TWO_LIST(nbuf_head, nbuf_tail, - soc->wbm_sg_param.wbm_sg_nbuf_head, - soc->wbm_sg_param.wbm_sg_nbuf_tail); + if (!dp_rx_buffer_pool_refill( + soc, + soc->wbm_sg_param.wbm_sg_nbuf_head, + rx_desc->pool_id)) + DP_RX_MERGE_TWO_LIST( + nbuf_head, nbuf_tail, + soc->wbm_sg_param.wbm_sg_nbuf_head, + soc->wbm_sg_param.wbm_sg_nbuf_tail); dp_rx_wbm_sg_list_reset(soc); process_sg_buf = false; } - } else { + } else if (!dp_rx_buffer_pool_refill(soc, nbuf, + rx_desc->pool_id)) { DP_RX_LIST_APPEND(nbuf_head, nbuf_tail, nbuf); } @@ -2227,7 +2241,8 @@ dp_rx_err_mpdu_pop(struct dp_soc *soc, uint32_t mac_id, "[%s][%d] msdu_nbuf=%pK ", __func__, __LINE__, msdu); - qdf_nbuf_free(msdu); + dp_rx_buffer_pool_nbuf_free(soc, msdu, + rx_desc->pool_id); rx_bufs_used++; dp_rx_add_to_free_desc_list(head, tail, rx_desc); @@ -2375,7 +2390,8 @@ dp_wbm_int_err_mpdu_pop(struct dp_soc *soc, uint32_t mac_id, qdf_nbuf_unmap_single(soc->osdev, msdu, QDF_DMA_FROM_DEVICE); - qdf_nbuf_free(msdu); + dp_rx_buffer_pool_nbuf_free(soc, msdu, + rx_desc->pool_id); rx_bufs_used++; dp_rx_add_to_free_desc_list(head, tail, rx_desc); @@ -2456,7 +2472,8 @@ dp_handle_wbm_internal_error(struct dp_soc *soc, void *hal_desc, rx_desc_pool->buf_size); rx_desc->unmapped = 1; - qdf_nbuf_free(rx_desc->nbuf); + dp_rx_buffer_pool_nbuf_free(soc, rx_desc->nbuf, + rx_desc->pool_id); dp_rx_add_to_free_desc_list(&head, &tail, rx_desc); diff --git a/dp/wifi3.0/dp_rx_mon_dest.c b/dp/wifi3.0/dp_rx_mon_dest.c index 86ad91d89a..b9bb72a7e0 100644 --- a/dp/wifi3.0/dp_rx_mon_dest.c +++ b/dp/wifi3.0/dp_rx_mon_dest.c @@ -28,6 +28,7 @@ #include "dp_rx_mon.h" #include "wlan_cfg.h" #include "dp_internal.h" +#include "dp_rx_buffer_pool.h" #ifdef WLAN_TX_PKT_CAPTURE_ENH #include "dp_rx_mon_feature.h" @@ -277,6 +278,15 @@ dp_rx_mon_mpdu_pop(struct dp_soc *soc, uint32_t mac_id, rx_desc->unmapped = 1; } + if (dp_rx_buffer_pool_refill(soc, msdu, + rx_desc->pool_id)) { + drop_mpdu = true; + msdu = NULL; + dp_pdev->mon_last_linkdesc_paddr = + buf_info.paddr; + goto next_msdu; + } + if (drop_mpdu) { dp_pdev->mon_last_linkdesc_paddr = buf_info.paddr; diff --git a/dp/wifi3.0/dp_stats.c b/dp/wifi3.0/dp_stats.c index bb77d19c30..2ae15fbeb5 100644 --- a/dp/wifi3.0/dp_stats.c +++ b/dp/wifi3.0/dp_stats.c @@ -6368,6 +6368,14 @@ dp_print_pdev_rx_stats(struct dp_pdev *pdev) dp_aggregate_pdev_ctrl_frames_stats(pdev); DP_PRINT_STATS("BAR Received Count: = %u", pdev->stats.rx.bar_recv_cnt); + + DP_PRINT_STATS("RX Buffer Pool Stats:\n"); + DP_PRINT_STATS("\tBuffers consumed during refill = %llu", + pdev->stats.rx_buffer_pool.num_bufs_consumed); + DP_PRINT_STATS("\tSuccessful allocations during refill = %llu", + pdev->stats.rx_buffer_pool.num_bufs_alloc_success); + DP_PRINT_STATS("\tAllocations from the pool during replenish = %llu", + pdev->stats.rx_buffer_pool.num_pool_bufs_replenish); } void diff --git a/dp/wifi3.0/dp_types.h b/dp/wifi3.0/dp_types.h index 52b37834db..94ea6d2a82 100644 --- a/dp/wifi3.0/dp_types.h +++ b/dp/wifi3.0/dp_types.h @@ -967,6 +967,12 @@ struct link_desc_bank { uint32_t size; }; +struct rx_buff_pool { + qdf_nbuf_queue_head_t emerg_nbuf_q; + uint32_t nbuf_fail_cnt; + bool is_initialized; +}; + /* SOC level structure for data path */ struct dp_soc { /** @@ -1362,6 +1368,9 @@ struct dp_soc { } wbm_sg_param; /* Number of msdu exception descriptors */ uint32_t num_msdu_exception_desc; + + /* RX buffer params */ + struct rx_buff_pool rx_buff_pool[MAX_PDEV_CNT]; }; #ifdef IPA_OFFLOAD