xiaomi-sm8450/android_kernel_xiaomi_sm8450
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ixgbe: Break out Rx buffer page management

Browse Source 
We are going to be expanding the number of Rx paths in the driver.  Instead
of duplicating all that code I am pulling it apart into separate functions
so that we don't have so much code duplication.

Signed-off-by: Alexander Duyck <alexander.h.duyck@intel.com>
Tested-by: Andrew Bowers <andrewx.bowers@intel.com>
Signed-off-by: Jeff Kirsher <jeffrey.t.kirsher@intel.com>
This commit is contained in:
Alexander Duyck
2017-01-17 08:36:45 -08:00
committed by Jeff Kirsher
parent c3630cc40b
commit 3fd218767f
1 changed files with 123 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

217
drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
View File

@@ -1847,7 +1847,6 @@ static void ixgbe_dma_sync_frag(struct ixgbe_ring *rx_ring,
ixgbe_rx_pg_size(rx_ring), ixgbe_rx_pg_size(rx_ring),
DMA_FROM_DEVICE, DMA_FROM_DEVICE,
IXGBE_RX_DMA_ATTR); IXGBE_RX_DMA_ATTR);
IXGBE_CB(skb)->page_released = false;
} else { } else {
struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0]; struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
@@ -1857,7 +1856,6 @@ static void ixgbe_dma_sync_frag(struct ixgbe_ring *rx_ring,
skb_frag_size(frag), skb_frag_size(frag),
DMA_FROM_DEVICE); DMA_FROM_DEVICE);
} }
IXGBE_CB(skb)->dma = 0;
} }
/** /**
@@ -1928,8 +1926,14 @@ static void ixgbe_reuse_rx_page(struct ixgbe_ring *rx_ring,
nta++; nta++;
rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
/* transfer page from old buffer to new buffer */ /* Transfer page from old buffer to new buffer.
*new_buff = *old_buff; * Move each member individually to avoid possible store
* forwarding stalls and unnecessary copy of skb.
*/
new_buff->dma = old_buff->dma;
new_buff->page = old_buff->page;
new_buff->page_offset = old_buff->page_offset;
new_buff->pagecnt_bias = old_buff->pagecnt_bias;
} }
static inline bool ixgbe_page_is_reserved(struct page *page) static inline bool ixgbe_page_is_reserved(struct page *page)
@@ -1937,16 +1941,10 @@ static inline bool ixgbe_page_is_reserved(struct page *page)
return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page); return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page);
} }
static bool ixgbe_can_reuse_rx_page(struct ixgbe_ring *rx_ring, static bool ixgbe_can_reuse_rx_page(struct ixgbe_rx_buffer *rx_buffer)
struct ixgbe_rx_buffer *rx_buffer,
struct page *page,
const unsigned int truesize)
{ {
#if (PAGE_SIZE >= 8192) unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
unsigned int last_offset = ixgbe_rx_pg_size(rx_ring) - struct page *page = rx_buffer->page;
ixgbe_rx_bufsz(rx_ring);
#endif
unsigned int pagecnt_bias = rx_buffer->pagecnt_bias--;
/* avoid re-using remote pages */ /* avoid re-using remote pages */
if (unlikely(ixgbe_page_is_reserved(page))) if (unlikely(ixgbe_page_is_reserved(page)))
@@ -1954,16 +1952,17 @@ static bool ixgbe_can_reuse_rx_page(struct ixgbe_ring *rx_ring,
#if (PAGE_SIZE < 8192) #if (PAGE_SIZE < 8192)
/* if we are only owner of page we can reuse it */ /* if we are only owner of page we can reuse it */
if (unlikely(page_count(page) != pagecnt_bias)) if (unlikely((page_ref_count(page) - pagecnt_bias) > 1))
return false; return false;
/* flip page offset to other buffer */
rx_buffer->page_offset ^= truesize;
#else #else
/* move offset up to the next cache line */ /* The last offset is a bit aggressive in that we assume the
rx_buffer->page_offset += truesize; * worst case of FCoE being enabled and using a 3K buffer.
* However this should have minimal impact as the 1K extra is
if (rx_buffer->page_offset > last_offset) * still less than one buffer in size.
*/
#define IXGBE_LAST_OFFSET \
(SKB_WITH_OVERHEAD(PAGE_SIZE) - IXGBE_RXBUFFER_3K)
if (rx_buffer->page_offset > IXGBE_LAST_OFFSET)
return false; return false;
#endif #endif
@@ -1971,7 +1970,7 @@ static bool ixgbe_can_reuse_rx_page(struct ixgbe_ring *rx_ring,
* the pagecnt_bias and page count so that we fully restock the * the pagecnt_bias and page count so that we fully restock the
* number of references the driver holds. * number of references the driver holds.
*/ */
if (unlikely(pagecnt_bias == 1)) { if (unlikely(!pagecnt_bias)) {
page_ref_add(page, USHRT_MAX); page_ref_add(page, USHRT_MAX);
rx_buffer->pagecnt_bias = USHRT_MAX; rx_buffer->pagecnt_bias = USHRT_MAX;
} }
@@ -1994,106 +1993,65 @@ static bool ixgbe_can_reuse_rx_page(struct ixgbe_ring *rx_ring,
* The function will then update the page offset if necessary and return * The function will then update the page offset if necessary and return
* true if the buffer can be reused by the adapter. * true if the buffer can be reused by the adapter.
**/ **/
static bool ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring, static void ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring,
struct ixgbe_rx_buffer *rx_buffer, struct ixgbe_rx_buffer *rx_buffer,
unsigned int size, struct sk_buff *skb,
struct sk_buff *skb) unsigned int size)
{ {
struct page *page = rx_buffer->page;
unsigned char *va = page_address(page) + rx_buffer->page_offset;
#if (PAGE_SIZE < 8192) #if (PAGE_SIZE < 8192)
unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2; unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2;
#else #else
unsigned int truesize = SKB_DATA_ALIGN(size); unsigned int truesize = SKB_DATA_ALIGN(size);
#endif #endif
skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
if (unlikely(skb_is_nonlinear(skb)))
goto add_tail_frag;
if (size <= IXGBE_RX_HDR_SIZE) {
memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long)));
/* page is not reserved, we can reuse buffer as-is */
if (likely(!ixgbe_page_is_reserved(page)))
return true;
/* this page cannot be reused so discard it */
return false;
}
add_tail_frag:
skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
rx_buffer->page_offset, size, truesize); rx_buffer->page_offset, size, truesize);
#if (PAGE_SIZE < 8192)
return ixgbe_can_reuse_rx_page(rx_ring, rx_buffer, page, truesize); rx_buffer->page_offset ^= truesize;
#else
rx_buffer->page_offset += truesize;
#endif
} }
static struct sk_buff *ixgbe_fetch_rx_buffer(struct ixgbe_ring *rx_ring, static struct ixgbe_rx_buffer *ixgbe_get_rx_buffer(struct ixgbe_ring *rx_ring,
union ixgbe_adv_rx_desc *rx_desc) union ixgbe_adv_rx_desc *rx_desc,
struct sk_buff **skb,
const unsigned int size)
{ {
unsigned int size = le16_to_cpu(rx_desc->wb.upper.length);
struct ixgbe_rx_buffer *rx_buffer; struct ixgbe_rx_buffer *rx_buffer;
struct sk_buff *skb;
struct page *page;
rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
page = rx_buffer->page; prefetchw(rx_buffer->page);
prefetchw(page); *skb = rx_buffer->skb;
skb = rx_buffer->skb; /* Delay unmapping of the first packet. It carries the header
* information, HW may still access the header after the writeback.
if (likely(!skb)) { * Only unmap it when EOP is reached
void *page_addr = page_address(page) + */
rx_buffer->page_offset; if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)) {
if (!*skb)
/* prefetch first cache line of first page */ goto skip_sync;
prefetch(page_addr); } else {
#if L1_CACHE_BYTES < 128 if (*skb)
prefetch(page_addr + L1_CACHE_BYTES); ixgbe_dma_sync_frag(rx_ring, *skb);
#endif
/* allocate a skb to store the frags */
skb = napi_alloc_skb(&rx_ring->q_vector->napi,
IXGBE_RX_HDR_SIZE);
if (unlikely(!skb)) {
rx_ring->rx_stats.alloc_rx_buff_failed++;
return NULL;
} }
/*
* we will be copying header into skb->data in
* pskb_may_pull so it is in our interest to prefetch
* it now to avoid a possible cache miss
*/
prefetchw(skb->data);
/*
* Delay unmapping of the first packet. It carries the
* header information, HW may still access the header
* after the writeback. Only unmap it when EOP is
* reached
*/
if (likely(ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)))
goto dma_sync;
IXGBE_CB(skb)->dma = rx_buffer->dma;
} else {
if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))
ixgbe_dma_sync_frag(rx_ring, skb);
dma_sync:
/* we are reusing so sync this buffer for CPU use */ /* we are reusing so sync this buffer for CPU use */
dma_sync_single_range_for_cpu(rx_ring->dev, dma_sync_single_range_for_cpu(rx_ring->dev,
rx_buffer->dma, rx_buffer->dma,
rx_buffer->page_offset, rx_buffer->page_offset,
size, size,
DMA_FROM_DEVICE); DMA_FROM_DEVICE);
skip_sync:
rx_buffer->pagecnt_bias--;
rx_buffer->skb = NULL; return rx_buffer;
} }
/* pull page into skb */ static void ixgbe_put_rx_buffer(struct ixgbe_ring *rx_ring,
if (ixgbe_add_rx_frag(rx_ring, rx_buffer, size, skb)) { struct ixgbe_rx_buffer *rx_buffer,
struct sk_buff *skb)
{
if (ixgbe_can_reuse_rx_page(rx_buffer)) {
/* hand second half of page back to the ring */ /* hand second half of page back to the ring */
ixgbe_reuse_rx_page(rx_ring, rx_buffer); ixgbe_reuse_rx_page(rx_ring, rx_buffer);
} else { } else {
@@ -2107,12 +2065,55 @@ dma_sync:
DMA_FROM_DEVICE, DMA_FROM_DEVICE,
IXGBE_RX_DMA_ATTR); IXGBE_RX_DMA_ATTR);
} }
__page_frag_cache_drain(page, __page_frag_cache_drain(rx_buffer->page,
rx_buffer->pagecnt_bias); rx_buffer->pagecnt_bias);
} }
/* clear contents of buffer_info */ /* clear contents of rx_buffer */
rx_buffer->page = NULL; rx_buffer->page = NULL;
rx_buffer->skb = NULL;
}
static struct sk_buff *ixgbe_construct_skb(struct ixgbe_ring *rx_ring,
struct ixgbe_rx_buffer *rx_buffer,
union ixgbe_adv_rx_desc *rx_desc,
unsigned int size)
{
void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;
#if (PAGE_SIZE < 8192)
unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2;
#else
unsigned int truesize = ALIGN(size, L1_CACHE_BYTES);
#endif
struct sk_buff *skb;
/* prefetch first cache line of first page */
prefetch(va);
#if L1_CACHE_BYTES < 128
prefetch(va + L1_CACHE_BYTES);
#endif
/* allocate a skb to store the frags */
skb = napi_alloc_skb(&rx_ring->q_vector->napi, IXGBE_RX_HDR_SIZE);
if (unlikely(!skb))
return NULL;
if (size > IXGBE_RX_HDR_SIZE) {
if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))
IXGBE_CB(skb)->dma = rx_buffer->dma;
skb_add_rx_frag(skb, 0, rx_buffer->page,
rx_buffer->page_offset,
size, truesize);
#if (PAGE_SIZE < 8192)
rx_buffer->page_offset ^= truesize;
#else
rx_buffer->page_offset += truesize;
#endif
} else {
memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long)));
rx_buffer->pagecnt_bias++;
}
return skb; return skb;
} }
@@ -2144,7 +2145,9 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector,
while (likely(total_rx_packets < budget)) { while (likely(total_rx_packets < budget)) {
union ixgbe_adv_rx_desc *rx_desc; union ixgbe_adv_rx_desc *rx_desc;
struct ixgbe_rx_buffer *rx_buffer;
struct sk_buff *skb; struct sk_buff *skb;
unsigned int size;
/* return some buffers to hardware, one at a time is too slow */ /* return some buffers to hardware, one at a time is too slow */
if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) { if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) {
@@ -2153,8 +2156,8 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector,
} }
rx_desc = IXGBE_RX_DESC(rx_ring, rx_ring->next_to_clean); rx_desc = IXGBE_RX_DESC(rx_ring, rx_ring->next_to_clean);
size = le16_to_cpu(rx_desc->wb.upper.length);
if (!rx_desc->wb.upper.length) if (!size)
break; break;
/* This memory barrier is needed to keep us from reading /* This memory barrier is needed to keep us from reading
@@ -2163,13 +2166,23 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector,
*/ */
dma_rmb(); dma_rmb();
rx_buffer = ixgbe_get_rx_buffer(rx_ring, rx_desc, &skb, size);
/* retrieve a buffer from the ring */ /* retrieve a buffer from the ring */
skb = ixgbe_fetch_rx_buffer(rx_ring, rx_desc); if (skb)
ixgbe_add_rx_frag(rx_ring, rx_buffer, skb, size);
else
skb = ixgbe_construct_skb(rx_ring, rx_buffer,
rx_desc, size);
/* exit if we failed to retrieve a buffer */ /* exit if we failed to retrieve a buffer */
if (!skb) if (!skb) {
rx_ring->rx_stats.alloc_rx_buff_failed++;
rx_buffer->pagecnt_bias++;
break; break;
}
ixgbe_put_rx_buffer(rx_ring, rx_buffer, skb);
cleaned_count++; cleaned_count++;
/* place incomplete frames back on ring for completion */ /* place incomplete frames back on ring for completion */