diff --git a/dp/wifi3.0/dp_internal.h b/dp/wifi3.0/dp_internal.h index d46d23d988..07311ab74c 100644 --- a/dp/wifi3.0/dp_internal.h +++ b/dp/wifi3.0/dp_internal.h @@ -2435,6 +2435,30 @@ static inline void dp_srng_dst_inv_cached_descs(struct dp_soc *dp_soc, } #endif /* QCA_CACHED_RING_DESC */ +#if defined(QCA_CACHED_RING_DESC) && defined(QCA_DP_RX_HW_SW_NBUF_DESC_PREFETCH) +/** + * dp_srng_dst_prefetch() - Wrapper function to prefetch descs from dest ring + * @hal_soc_hdl: HAL SOC handle + * @hal_ring: opaque pointer to the HAL Rx Destination ring + * @num_entries: Entry count + * + * Return: None + */ +static inline void *dp_srng_dst_prefetch(hal_soc_handle_t hal_soc, + hal_ring_handle_t hal_ring_hdl, + uint32_t num_entries) +{ + return hal_srng_dst_prefetch(hal_soc, hal_ring_hdl, num_entries); +} +#else +static inline void *dp_srng_dst_prefetch(hal_soc_handle_t hal_soc, + hal_ring_handle_t hal_ring_hdl, + uint32_t num_entries) +{ + return NULL; +} +#endif + #ifdef QCA_ENH_V3_STATS_SUPPORT /** * dp_pdev_print_delay_stats(): Print pdev level delay stats diff --git a/dp/wifi3.0/li/dp_li_rx.c b/dp/wifi3.0/li/dp_li_rx.c index 03e82f1238..6e67a3ddb5 100644 --- a/dp/wifi3.0/li/dp_li_rx.c +++ b/dp/wifi3.0/li/dp_li_rx.c @@ -203,8 +203,10 @@ uint32_t dp_rx_process_li(struct dp_intr *int_ctx, uint32_t quota) { hal_ring_desc_t ring_desc; + hal_ring_desc_t last_prefetched_hw_desc; hal_soc_handle_t hal_soc; struct dp_rx_desc *rx_desc = NULL; + struct dp_rx_desc *last_prefetched_sw_desc = NULL; qdf_nbuf_t nbuf, next; bool near_full; union dp_rx_desc_list_elem_t *head[MAX_PDEV_CNT]; @@ -296,6 +298,9 @@ more_data: if (num_pending > quota) num_pending = quota; + last_prefetched_hw_desc = dp_srng_dst_prefetch(hal_soc, hal_ring_hdl, + num_pending); + /* * start reaping the buffers from reo ring and queue * them in per vdev queue. @@ -516,6 +521,12 @@ more_data: dp_rx_add_to_free_desc_list(&head[rx_desc->pool_id], &tail[rx_desc->pool_id], rx_desc); num_rx_bufs_reaped++; + + dp_rx_prefetch_hw_sw_nbuf_desc(soc, hal_soc, num_pending, + hal_ring_hdl, + &last_prefetched_hw_desc, + &last_prefetched_sw_desc); + /* * only if complete msdu is received for scatter case, * then allow break. diff --git a/dp/wifi3.0/li/dp_li_rx.h b/dp/wifi3.0/li/dp_li_rx.h index 1bfca19812..36816b954b 100644 --- a/dp/wifi3.0/li/dp_li_rx.h +++ b/dp/wifi3.0/li/dp_li_rx.h @@ -130,4 +130,75 @@ void dp_rx_prefetch_nbuf_data(qdf_nbuf_t nbuf, qdf_nbuf_t next) { } #endif + +#ifdef QCA_DP_RX_HW_SW_NBUF_DESC_PREFETCH +/** + * dp_rx_cookie_2_va_rxdma_buf_prefetch() - function to prefetch the SW desc + * @soc: Handle to DP Soc structure + * @cookie: cookie used to lookup virtual address + * + * Return: prefetched Rx descriptor virtual address + */ +static inline +void *dp_rx_cookie_2_va_rxdma_buf_prefetch(struct dp_soc *soc, uint32_t cookie) +{ + uint8_t pool_id = DP_RX_DESC_COOKIE_POOL_ID_GET(cookie); + uint16_t index = DP_RX_DESC_COOKIE_INDEX_GET(cookie); + struct rx_desc_pool *rx_desc_pool; + void *prefetch_desc; + + if (qdf_unlikely(pool_id >= MAX_RXDESC_POOLS)) + return NULL; + + rx_desc_pool = &soc->rx_desc_buf[pool_id]; + + if (qdf_unlikely(index >= rx_desc_pool->pool_size)) + return NULL; + + prefetch_desc = &soc->rx_desc_buf[pool_id].array[index].rx_desc; + qdf_prefetch(prefetch_desc); + return prefetch_desc; +} + +/** + * dp_rx_prefetch_hw_sw_nbuf_desc() - function to prefetch HW and SW desc + * @soc: Handle to HAL Soc structure + * @num_entries: valid number of HW descriptors + * @hal_ring_hdl: Destination ring pointer + * @last_prefetched_hw_desc: pointer to the last prefetched HW descriptor + * @last_prefetched_sw_desc: input & output param of last prefetch SW desc + * + * Return: None + */ +static inline +void dp_rx_prefetch_hw_sw_nbuf_desc(struct dp_soc *soc, + hal_soc_handle_t hal_soc, + uint32_t num_entries, + hal_ring_handle_t hal_ring_hdl, + hal_ring_desc_t *last_prefetched_hw_desc, + struct dp_rx_desc **last_prefetched_sw_desc) +{ + if (*last_prefetched_sw_desc) { + qdf_prefetch((uint8_t *)(*last_prefetched_sw_desc)->nbuf); + qdf_prefetch((uint8_t *)(*last_prefetched_sw_desc)->nbuf + 64); + } + + if (num_entries) { + *last_prefetched_sw_desc = dp_rx_cookie_2_va_rxdma_buf_prefetch(soc, HAL_RX_REO_BUF_COOKIE_GET(*last_prefetched_hw_desc)); + *last_prefetched_hw_desc = hal_srng_dst_prefetch_next_cached_desc(hal_soc, + hal_ring_hdl, + (uint8_t *)*last_prefetched_hw_desc); + } +} +#else +static inline +void dp_rx_prefetch_hw_sw_nbuf_desc(struct dp_soc *soc, + hal_soc_handle_t hal_soc, + uint32_t quota, + hal_ring_handle_t hal_ring_hdl, + hal_ring_desc_t *last_prefetched_hw_desc, + struct dp_rx_desc **last_prefetched_sw_desc) +{ +} +#endif #endif diff --git a/hal/wifi3.0/hal_api.h b/hal/wifi3.0/hal_api.h index 3abb5da85e..2b2f4331c9 100644 --- a/hal/wifi3.0/hal_api.h +++ b/hal/wifi3.0/hal_api.h @@ -2969,4 +2969,73 @@ hal_dmac_cmn_src_rxbuf_ring_get(hal_soc_handle_t hal_soc_hdl) return hal_soc->dmac_cmn_src_rxbuf_ring; } + +/** + * hal_srng_dst_prefetch() - function to prefetch 4 destination ring descs + * @hal_soc_hdl: HAL SOC handle + * @hal_ring_hdl: Destination ring pointer + * @num_valid: valid entries in the ring + * + * return: last prefetched destination ring descriptor + */ +static inline +void *hal_srng_dst_prefetch(hal_soc_handle_t hal_soc_hdl, + hal_ring_handle_t hal_ring_hdl, + uint16_t num_valid) +{ + struct hal_srng *srng = (struct hal_srng *)hal_ring_hdl; + uint8_t *desc; + uint32_t cnt; + /* + * prefetching 4 HW descriptors will ensure atleast by the time + * 5th HW descriptor is being processed it is guranteed that the + * 5th HW descriptor, its SW Desc, its nbuf and its nbuf's data + * are in cache line. basically ensuring all the 4 (HW, SW, nbuf + * & nbuf->data) are prefetched. + */ + uint32_t max_prefetch = 4; + + if (srng->u.dst_ring.tp == srng->u.dst_ring.cached_hp) + return NULL; + + desc = (uint8_t *)&srng->ring_base_vaddr[srng->u.dst_ring.tp]; + + if (num_valid < max_prefetch) + max_prefetch = num_valid; + + for (cnt = 0; cnt < max_prefetch; cnt++) { + desc += srng->entry_size * sizeof(uint32_t); + if (desc == ((uint8_t *)srng->ring_vaddr_end)) + desc = (uint8_t *)&srng->ring_base_vaddr[0]; + + qdf_prefetch(desc); + } + return (void *)desc; +} + +/** + * hal_srng_dst_prefetch_next_cached_desc() - function to prefetch next desc + * @hal_soc_hdl: HAL SOC handle + * @hal_ring_hdl: Destination ring pointer + * @last_prefetched_hw_desc: last prefetched HW descriptor + * + * return: next prefetched destination descriptor + */ +static inline +void *hal_srng_dst_prefetch_next_cached_desc(hal_soc_handle_t hal_soc_hdl, + hal_ring_handle_t hal_ring_hdl, + uint8_t *last_prefetched_hw_desc) +{ + struct hal_srng *srng = (struct hal_srng *)hal_ring_hdl; + + if (srng->u.dst_ring.tp == srng->u.dst_ring.cached_hp) + return NULL; + + last_prefetched_hw_desc += srng->entry_size * sizeof(uint32_t); + if (last_prefetched_hw_desc == ((uint8_t *)srng->ring_vaddr_end)) + last_prefetched_hw_desc = (uint8_t *)&srng->ring_base_vaddr[0]; + + qdf_prefetch(last_prefetched_hw_desc); + return (void *)last_prefetched_hw_desc; +} #endif /* _HAL_APIH_ */