diff --git a/dp/inc/cdp_txrx_misc.h b/dp/inc/cdp_txrx_misc.h index 4234eec652..1bee78e060 100644 --- a/dp/inc/cdp_txrx_misc.h +++ b/dp/inc/cdp_txrx_misc.h @@ -665,6 +665,29 @@ cdp_register_rx_mic_error_ind_handler(ol_txrx_soc_handle soc, soc->ol_ops->rx_mic_error = rx_mic_cb; } +typedef void (*rx_refill_thread_sched_cb)(ol_txrx_soc_handle soc); + +/** + * cdp_register_rx_refill_thread_sched_handler() - API to register RX refill + * thread schedule handler + * + * @soc: soc handle + * + * Return: void + */ +static inline void +cdp_register_rx_refill_thread_sched_handler(ol_txrx_soc_handle soc, + rx_refill_thread_sched_cb rx_sched_cb) +{ + if (!soc || !soc->ol_ops) { + QDF_TRACE(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_FATAL, + "%s invalid instance", __func__); + return; + } + + soc->ol_ops->dp_rx_sched_refill_thread = rx_sched_cb; +} + /** * cdp_pdev_reset_driver_del_ack() - reset driver TCP delayed ack flag * @soc: data path soc handle diff --git a/dp/inc/cdp_txrx_ops.h b/dp/inc/cdp_txrx_ops.h index 4045460142..17ee171b04 100644 --- a/dp/inc/cdp_txrx_ops.h +++ b/dp/inc/cdp_txrx_ops.h @@ -1147,6 +1147,7 @@ struct ol_if_ops { QDF_STATUS(*nss_stats_clr)(struct cdp_ctrl_objmgr_psoc *psoc, uint8_t vdev_id); int (*dp_rx_get_pending)(ol_txrx_soc_handle soc); + void (*dp_rx_sched_refill_thread)(ol_txrx_soc_handle soc); /* TODO: Add any other control path calls required to OL_IF/WMA layer */ #ifdef QCA_SUPPORT_WDS_EXTENDED void (*rx_wds_ext_peer_learn)(struct cdp_ctrl_objmgr_psoc *ctrl_psoc, diff --git a/dp/inc/cdp_txrx_stats_struct.h b/dp/inc/cdp_txrx_stats_struct.h index 6f3097204b..e3f5d91ff6 100644 --- a/dp/inc/cdp_txrx_stats_struct.h +++ b/dp/inc/cdp_txrx_stats_struct.h @@ -1968,6 +1968,11 @@ struct cdp_pdev_stats { uint64_t num_pool_bufs_replenish; uint64_t num_bufs_alloc_success; } rx_buffer_pool; + + struct { + uint64_t num_bufs_refilled; + uint64_t num_bufs_allocated; + } rx_refill_buff_pool; }; /* struct cdp_peer_hmwds_ast_add_status - hmwds peer ast add status diff --git a/dp/wifi3.0/dp_rx.c b/dp/wifi3.0/dp_rx.c index 50452f5a6c..cde758728c 100644 --- a/dp/wifi3.0/dp_rx.c +++ b/dp/wifi3.0/dp_rx.c @@ -230,11 +230,8 @@ dp_pdev_nbuf_alloc_and_map_replenish(struct dp_soc *dp_soc, return QDF_STATUS_E_NOMEM; } - ret = qdf_nbuf_map_nbytes_single(dp_soc->osdev, - (nbuf_frag_info_t->virt_addr).nbuf, - QDF_DMA_FROM_DEVICE, - rx_desc_pool->buf_size); - + ret = dp_rx_buffer_pool_nbuf_map(dp_soc, rx_desc_pool, + nbuf_frag_info_t); if (qdf_unlikely(QDF_IS_STATUS_ERROR(ret))) { dp_rx_buffer_pool_nbuf_free(dp_soc, (nbuf_frag_info_t->virt_addr).nbuf, mac_id); @@ -246,11 +243,6 @@ dp_pdev_nbuf_alloc_and_map_replenish(struct dp_soc *dp_soc, nbuf_frag_info_t->paddr = qdf_nbuf_get_frag_paddr((nbuf_frag_info_t->virt_addr).nbuf, 0); - dp_ipa_handle_rx_buf_smmu_mapping(dp_soc, - (qdf_nbuf_t)((nbuf_frag_info_t->virt_addr).nbuf), - rx_desc_pool->buf_size, - true); - ret = dp_check_paddr(dp_soc, &((nbuf_frag_info_t->virt_addr).nbuf), &nbuf_frag_info_t->paddr, rx_desc_pool); @@ -415,6 +407,8 @@ QDF_STATUS __dp_rx_buffers_replenish(struct dp_soc *dp_soc, uint32_t mac_id, hal_srng_access_end(dp_soc->hal_soc, rxdma_srng); + dp_rx_schedule_refill_thread(dp_soc); + dp_verbose_debug("replenished buffers %d, rx desc added back to free list %u", count, num_desc_to_free); diff --git a/dp/wifi3.0/dp_rx_buffer_pool.c b/dp/wifi3.0/dp_rx_buffer_pool.c index 1ff47d0988..f5c860211a 100644 --- a/dp/wifi3.0/dp_rx_buffer_pool.c +++ b/dp/wifi3.0/dp_rx_buffer_pool.c @@ -17,13 +17,22 @@ */ #include "dp_rx_buffer_pool.h" +#include "dp_ipa.h" #ifndef DP_RX_BUFFER_POOL_SIZE #define DP_RX_BUFFER_POOL_SIZE 128 #endif -#ifndef DP_RX_BUFFER_POOL_ALLOC_THRES -#define DP_RX_BUFFER_POOL_ALLOC_THRES 1 +#ifndef DP_RX_REFILL_BUFF_POOL_SIZE +#define DP_RX_REFILL_BUFF_POOL_SIZE 2048 +#endif + +#ifndef DP_RX_REFILL_BUFF_POOL_BURST +#define DP_RX_REFILL_BUFF_POOL_BURST 64 +#endif + +#ifndef DP_RX_BUFF_POOL_ALLOC_THRES +#define DP_RX_BUFF_POOL_ALLOC_THRES 1 #endif #ifdef WLAN_FEATURE_RX_PREALLOC_BUFFER_POOL @@ -109,6 +118,96 @@ void dp_rx_buffer_pool_nbuf_free(struct dp_soc *soc, qdf_nbuf_t nbuf, u8 mac_id) qdf_nbuf_queue_head_enqueue_tail(&buff_pool->emerg_nbuf_q, nbuf); } +void dp_rx_refill_buff_pool_enqueue(struct dp_soc *soc) +{ + struct rx_desc_pool *rx_desc_pool; + struct rx_refill_buff_pool *buff_pool; + struct dp_pdev *dp_pdev; + qdf_nbuf_t nbuf; + QDF_STATUS ret; + int count, i; + qdf_nbuf_t nbuf_head; + qdf_nbuf_t nbuf_tail; + uint32_t num_req_refill; + + if (!soc) + return; + + buff_pool = &soc->rx_refill_buff_pool; + if (!buff_pool->is_initialized) + return; + + rx_desc_pool = &soc->rx_desc_buf[0]; + dp_pdev = dp_get_pdev_for_lmac_id(soc, 0); + + num_req_refill = buff_pool->max_bufq_len - buff_pool->bufq_len; + + while (num_req_refill) { + if (num_req_refill > DP_RX_REFILL_BUFF_POOL_BURST) + num_req_refill = DP_RX_REFILL_BUFF_POOL_BURST; + + count = 0; + nbuf_head = NULL; + nbuf_tail = NULL; + for (i = 0; i < num_req_refill; i++) { + nbuf = qdf_nbuf_alloc(soc->osdev, + rx_desc_pool->buf_size, + RX_BUFFER_RESERVATION, + rx_desc_pool->buf_alignment, + FALSE); + if (!nbuf) + continue; + + ret = qdf_nbuf_map_nbytes_single(soc->osdev, nbuf, + QDF_DMA_FROM_DEVICE, + rx_desc_pool->buf_size); + if (qdf_unlikely(QDF_IS_STATUS_ERROR(ret))) { + qdf_nbuf_free(nbuf); + continue; + } + + dp_ipa_handle_rx_buf_smmu_mapping(soc, nbuf, + rx_desc_pool->buf_size, + true); + DP_RX_LIST_APPEND(nbuf_head, nbuf_tail, nbuf); + count++; + } + if (count) { + qdf_spin_lock_bh(&buff_pool->bufq_lock); + DP_RX_MERGE_TWO_LIST(buff_pool->buf_head, + buff_pool->buf_tail, + nbuf_head, nbuf_tail); + buff_pool->bufq_len += count; + + num_req_refill = buff_pool->max_bufq_len - + buff_pool->bufq_len; + qdf_spin_unlock_bh(&buff_pool->bufq_lock); + + DP_STATS_INC(dp_pdev, + rx_refill_buff_pool.num_bufs_refilled, + count); + } + } +} + +static inline qdf_nbuf_t dp_rx_refill_buff_pool_dequeue_nbuf(struct dp_soc *soc) +{ + qdf_nbuf_t nbuf = NULL; + struct rx_refill_buff_pool *buff_pool = &soc->rx_refill_buff_pool; + + if (!buff_pool->is_initialized || !buff_pool->bufq_len) + return nbuf; + + qdf_spin_lock_bh(&buff_pool->bufq_lock); + nbuf = buff_pool->buf_head; + buff_pool->buf_head = qdf_nbuf_next(buff_pool->buf_head); + qdf_nbuf_set_next(nbuf, NULL); + buff_pool->bufq_len--; + qdf_spin_unlock_bh(&buff_pool->bufq_lock); + + return nbuf; +} + qdf_nbuf_t dp_rx_buffer_pool_nbuf_alloc(struct dp_soc *soc, uint32_t mac_id, struct rx_desc_pool *rx_desc_pool, @@ -119,6 +218,13 @@ dp_rx_buffer_pool_nbuf_alloc(struct dp_soc *soc, uint32_t mac_id, struct dp_srng *dp_rxdma_srng; qdf_nbuf_t nbuf; + nbuf = dp_rx_refill_buff_pool_dequeue_nbuf(soc); + if (nbuf) { + DP_STATS_INC(dp_pdev, + rx_refill_buff_pool.num_bufs_allocated, 1); + return nbuf; + } + if (!wlan_cfg_per_pdev_lmac_ring(soc->wlan_cfg_ctx)) mac_id = dp_pdev->lmac_id; @@ -152,6 +258,81 @@ dp_rx_buffer_pool_nbuf_alloc(struct dp_soc *soc, uint32_t mac_id, return nbuf; } +QDF_STATUS +dp_rx_buffer_pool_nbuf_map(struct dp_soc *soc, + struct rx_desc_pool *rx_desc_pool, + struct dp_rx_nbuf_frag_info *nbuf_frag_info_t) +{ + QDF_STATUS ret = QDF_STATUS_SUCCESS; + + if (!QDF_NBUF_CB_PADDR((nbuf_frag_info_t->virt_addr).nbuf)) { + ret = qdf_nbuf_map_nbytes_single(soc->osdev, + (nbuf_frag_info_t->virt_addr).nbuf, + QDF_DMA_FROM_DEVICE, + rx_desc_pool->buf_size); + + if (qdf_unlikely(QDF_IS_STATUS_ERROR(ret))) + return ret; + + dp_ipa_handle_rx_buf_smmu_mapping(soc, + (qdf_nbuf_t)((nbuf_frag_info_t->virt_addr).nbuf), + rx_desc_pool->buf_size, + true); + } + + return ret; +} + +static void dp_rx_refill_buff_pool_init(struct dp_soc *soc, u8 mac_id) +{ + struct rx_desc_pool *rx_desc_pool = &soc->rx_desc_buf[mac_id]; + qdf_nbuf_t nbuf; + struct rx_refill_buff_pool *buff_pool = &soc->rx_refill_buff_pool; + QDF_STATUS ret; + int i; + + if (!wlan_cfg_is_rx_refill_buffer_pool_enabled(soc->wlan_cfg_ctx)) { + dp_err("RX refill buffer pool support is disabled"); + buff_pool->is_initialized = false; + return; + } + + buff_pool->bufq_len = 0; + buff_pool->buf_head = NULL; + buff_pool->buf_tail = NULL; + buff_pool->max_bufq_len = DP_RX_REFILL_BUFF_POOL_SIZE; + qdf_spinlock_create(&buff_pool->bufq_lock); + + for (i = 0; i < buff_pool->max_bufq_len; i++) { + nbuf = qdf_nbuf_alloc(soc->osdev, rx_desc_pool->buf_size, + RX_BUFFER_RESERVATION, + rx_desc_pool->buf_alignment, FALSE); + if (!nbuf) + continue; + + ret = qdf_nbuf_map_nbytes_single(soc->osdev, nbuf, + QDF_DMA_FROM_DEVICE, + rx_desc_pool->buf_size); + if (qdf_unlikely(QDF_IS_STATUS_ERROR(ret))) { + qdf_nbuf_free(nbuf); + continue; + } + + dp_ipa_handle_rx_buf_smmu_mapping(soc, nbuf, + rx_desc_pool->buf_size, + true); + DP_RX_LIST_APPEND(buff_pool->buf_head, + buff_pool->buf_tail, nbuf); + buff_pool->bufq_len++; + } + + dp_info("RX refill buffer pool required allocation: %u actual allocation: %u", + buff_pool->max_bufq_len, + buff_pool->bufq_len); + + buff_pool->is_initialized = true; +} + void dp_rx_buffer_pool_init(struct dp_soc *soc, u8 mac_id) { struct rx_desc_pool *rx_desc_pool = &soc->rx_desc_buf[mac_id]; @@ -159,6 +340,8 @@ void dp_rx_buffer_pool_init(struct dp_soc *soc, u8 mac_id) qdf_nbuf_t nbuf; int i; + dp_rx_refill_buff_pool_init(soc, mac_id); + if (!wlan_cfg_is_rx_buffer_pool_enabled(soc->wlan_cfg_ctx)) { dp_err("RX buffer pool support is disabled"); buff_pool->is_initialized = false; @@ -187,11 +370,35 @@ void dp_rx_buffer_pool_init(struct dp_soc *soc, u8 mac_id) buff_pool->is_initialized = true; } +static void dp_rx_refill_buff_pool_deinit(struct dp_soc *soc, u8 mac_id) +{ + struct rx_refill_buff_pool *buff_pool = &soc->rx_refill_buff_pool; + struct rx_desc_pool *rx_desc_pool = &soc->rx_desc_buf[mac_id]; + qdf_nbuf_t nbuf; + + if (!buff_pool->is_initialized) + return; + + while ((nbuf = dp_rx_refill_buff_pool_dequeue_nbuf(soc))) { + dp_ipa_handle_rx_buf_smmu_mapping(soc, nbuf, + rx_desc_pool->buf_size, + false); + qdf_nbuf_unmap_nbytes_single(soc->osdev, nbuf, + QDF_DMA_BIDIRECTIONAL, + rx_desc_pool->buf_size); + qdf_nbuf_free(nbuf); + } + + buff_pool->is_initialized = false; +} + void dp_rx_buffer_pool_deinit(struct dp_soc *soc, u8 mac_id) { struct rx_buff_pool *buff_pool = &soc->rx_buff_pool[mac_id]; qdf_nbuf_t nbuf; + dp_rx_refill_buff_pool_deinit(soc, mac_id); + if (!buff_pool->is_initialized) return; diff --git a/dp/wifi3.0/dp_rx_buffer_pool.h b/dp/wifi3.0/dp_rx_buffer_pool.h index bdf887df27..7cc73cbb1a 100644 --- a/dp/wifi3.0/dp_rx_buffer_pool.h +++ b/dp/wifi3.0/dp_rx_buffer_pool.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020 The Linux Foundation. All rights reserved. + * Copyright (c) 2020-2021 The Linux Foundation. All rights reserved. * * Permission to use, copy, modify, and/or distribute this software for * any purpose with or without fee is hereby granted, provided that the @@ -83,6 +83,32 @@ void dp_rx_buffer_pool_nbuf_free(struct dp_soc *soc, qdf_nbuf_t nbuf, qdf_nbuf_t dp_rx_buffer_pool_nbuf_alloc(struct dp_soc *soc, uint32_t mac_id, struct rx_desc_pool *rx_desc_pool, uint32_t num_available_buffers); + +/** + * dp_rx_buffer_pool_nbuf_map() - Map nbuff for buffer replenish + * @soc: SoC handle + * @rx_desc_pool: RX descriptor pool + * @nbuf_frag_info_t: nbuf frag info + * + * Return: nbuf + */ +QDF_STATUS +dp_rx_buffer_pool_nbuf_map(struct dp_soc *soc, + struct rx_desc_pool *rx_desc_pool, + struct dp_rx_nbuf_frag_info *nbuf_frag_info_t); + +/** + * dp_rx_schedule_refill_thread() - Schedule RX refill thread to enqueue + * buffers in refill pool + * @soc: SoC handle + * + */ +static inline void dp_rx_schedule_refill_thread(struct dp_soc *soc) +{ + if (soc->cdp_soc.ol_ops->dp_rx_sched_refill_thread) + soc->cdp_soc.ol_ops->dp_rx_sched_refill_thread(dp_soc_to_cdp_soc_t(soc)); +} + #else /** * dp_rx_buffer_pool_init() - Initialize emergency buffer pool @@ -159,5 +185,26 @@ dp_rx_buffer_pool_nbuf_alloc(struct dp_soc *soc, uint32_t mac_id, RX_BUFFER_RESERVATION, rx_desc_pool->buf_alignment, FALSE); } + +/** + * dp_rx_buffer_pool_nbuf_map() - Map nbuff for buffer replenish + * @soc: SoC handle + * @rx_desc_pool: RX descriptor pool + * @nbuf_frag_info_t: nbuf frag info + * + * Return: nbuf + */ +static inline QDF_STATUS +dp_rx_buffer_pool_nbuf_map(struct dp_soc *soc, + struct rx_desc_pool *rx_desc_pool, + struct dp_rx_nbuf_frag_info *nbuf_frag_info_t) +{ + return qdf_nbuf_map_nbytes_single(soc->osdev, + (nbuf_frag_info_t->virt_addr).nbuf, + QDF_DMA_FROM_DEVICE, + rx_desc_pool->buf_size); +} + +static inline void dp_rx_schedule_refill_thread(struct dp_soc *soc) { } #endif /* WLAN_FEATURE_RX_PREALLOC_BUFFER_POOL */ #endif /* _DP_RX_BUFFER_POOL_H_ */ diff --git a/dp/wifi3.0/dp_types.h b/dp/wifi3.0/dp_types.h index 8dca7698c0..500cac5aad 100644 --- a/dp/wifi3.0/dp_types.h +++ b/dp/wifi3.0/dp_types.h @@ -1138,6 +1138,15 @@ struct rx_buff_pool { bool is_initialized; }; +struct rx_refill_buff_pool { + qdf_nbuf_t buf_head; + qdf_nbuf_t buf_tail; + qdf_spinlock_t bufq_lock; + uint32_t bufq_len; + uint32_t max_bufq_len; + bool is_initialized; +}; + /* * The logic for get current index of these history is dependent on this * value being power of 2. @@ -1727,6 +1736,7 @@ struct dp_soc { /* RX buffer params */ struct rx_buff_pool rx_buff_pool[MAX_PDEV_CNT]; + struct rx_refill_buff_pool rx_refill_buff_pool; /* Save recent operation related variable */ struct dp_last_op_info last_op_info; TAILQ_HEAD(, dp_peer) inactive_peer_list; @@ -3120,4 +3130,9 @@ QDF_STATUS dp_hw_link_desc_pool_banks_alloc(struct dp_soc *soc, uint32_t mac_id); void dp_link_desc_ring_replenish(struct dp_soc *soc, uint32_t mac_id); +#ifdef WLAN_FEATURE_RX_PREALLOC_BUFFER_POOL +void dp_rx_refill_buff_pool_enqueue(struct dp_soc *soc); +#else +static inline void dp_rx_refill_buff_pool_enqueue(struct dp_soc *soc) {} +#endif #endif /* _DP_TYPES_H_ */