xiaomi-sm8450/android_kernel_xiaomi_sm8450
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Browse Source 
Daniel Borkmann says:

====================
pull-request: bpf-next 2020-05-23

The following pull-request contains BPF updates for your *net-next* tree.

We've added 50 non-merge commits during the last 8 day(s) which contain
a total of 109 files changed, 2776 insertions(+), 2887 deletions(-).

The main changes are:

1) Add a new AF_XDP buffer allocation API to the core in order to help
   lowering the bar for drivers adopting AF_XDP support. i40e, ice, ixgbe
   as well as mlx5 have been moved over to the new API and also gained a
   small improvement in performance, from Björn Töpel and Magnus Karlsson.

2) Add getpeername()/getsockname() attach types for BPF sock_addr programs
   in order to allow for e.g. reverse translation of load-balancer backend
   to service address/port tuple from a connected peer, from Daniel Borkmann.

3) Improve the BPF verifier is_branch_taken() logic to evaluate pointers
   being non-NULL, e.g. if after an initial test another non-NULL test on
   that pointer follows in a given path, then it can be pruned right away,
   from John Fastabend.

4) Larger rework of BPF sockmap selftests to make output easier to understand
   and to reduce overall runtime as well as adding new BPF kTLS selftests
   that run in combination with sockmap, also from John Fastabend.

5) Batch of misc updates to BPF selftests including fixing up test_align
   to match verifier output again and moving it under test_progs, allowing
   bpf_iter selftest to compile on machines with older vmlinux.h, and
   updating config options for lirc and v6 segment routing helpers, from
   Stanislav Fomichev, Andrii Nakryiko and Alan Maguire.

6) Conversion of BPF tracing samples outdated internal BPF loader to use
   libbpf API instead, from Daniel T. Lee.

7) Follow-up to BPF kernel test infrastructure in order to fix a flake in
   the XDP selftests, from Jesper Dangaard Brouer.

8) Minor improvements to libbpf's internal hashmap implementation, from
   Ian Rogers.
====================

Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
David S. Miller
2020-05-22 18:30:34 -07:00
parent 1e6a705266 a5dfaa2ab9
commit a152b85984
109 changed files with 2814 additions and 2925 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
drivers/net/ethernet/intel/i40e/i40e_main.c
View File

@@ -11,7 +11,7 @@
#include "i40e_diag.h"
#include "i40e_xsk.h"
#include <net/udp_tunnel.h>
#include <net/xdp_sock.h>
#include <net/xdp_sock_drv.h>
/* All i40e tracepoints are defined by the include below, which
* must be included exactly once across the whole kernel with
* CREATE_TRACE_POINTS defined
@@ -3260,26 +3260,31 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring)
if (ring->vsi->type == I40E_VSI_MAIN)
xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq);
kfree(ring->rx_bi);
ring->xsk_umem = i40e_xsk_umem(ring);
if (ring->xsk_umem) {
ring->rx_buf_len = ring->xsk_umem->chunk_size_nohr -
XDP_PACKET_HEADROOM;
ret = i40e_alloc_rx_bi_zc(ring);
if (ret)
return ret;
ring->rx_buf_len = xsk_umem_get_rx_frame_size(ring->xsk_umem);
/* For AF_XDP ZC, we disallow packets to span on
* multiple buffers, thus letting us skip that
* handling in the fast-path.
*/
chain_len = 1;
ring->zca.free = i40e_zca_free;
ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
MEM_TYPE_ZERO_COPY,
&ring->zca);
MEM_TYPE_XSK_BUFF_POOL,
NULL);
if (ret)
return ret;
dev_info(&vsi->back->pdev->dev,
"Registered XDP mem model MEM_TYPE_ZERO_COPY on Rx ring %d\n",
"Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n",
ring->queue_index);
} else {
ret = i40e_alloc_rx_bi(ring);
if (ret)
return ret;
ring->rx_buf_len = vsi->rx_buf_len;
if (ring->vsi->type == I40E_VSI_MAIN) {
ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
@@ -3344,9 +3349,12 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring)
ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
writel(0, ring->tail);
ok = ring->xsk_umem ?
i40e_alloc_rx_buffers_zc(ring, I40E_DESC_UNUSED(ring)) :
!i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
if (ring->xsk_umem) {
xsk_buff_set_rxq_info(ring->xsk_umem, &ring->xdp_rxq);
ok = i40e_alloc_rx_buffers_zc(ring, I40E_DESC_UNUSED(ring));
} else {
ok = !i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
}
if (!ok) {
/* Log this in case the user has forgotten to give the kernel
* any buffers, even later in the application.

134
drivers/net/ethernet/intel/i40e/i40e_txrx.c
View File

@@ -521,28 +521,29 @@ int i40e_add_del_fdir(struct i40e_vsi *vsi,
/**
* i40e_fd_handle_status - check the Programming Status for FD
* @rx_ring: the Rx ring for this descriptor
* @rx_desc: the Rx descriptor for programming Status, not a packet descriptor.
* @qword0_raw: qword0
* @qword1: qword1 after le_to_cpu
* @prog_id: the id originally used for programming
*
* This is used to verify if the FD programming or invalidation
* requested by SW to the HW is successful or not and take actions accordingly.
**/
void i40e_fd_handle_status(struct i40e_ring *rx_ring,
union i40e_rx_desc *rx_desc, u8 prog_id)
static void i40e_fd_handle_status(struct i40e_ring *rx_ring, u64 qword0_raw,
u64 qword1, u8 prog_id)
{
struct i40e_pf *pf = rx_ring->vsi->back;
struct pci_dev *pdev = pf->pdev;
struct i40e_32b_rx_wb_qw0 *qw0;
u32 fcnt_prog, fcnt_avail;
u32 error;
u64 qw;
qw = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
error = (qw & I40E_RX_PROG_STATUS_DESC_QW1_ERROR_MASK) >>
qw0 = (struct i40e_32b_rx_wb_qw0 *)&qword0_raw;
error = (qword1 & I40E_RX_PROG_STATUS_DESC_QW1_ERROR_MASK) >>
I40E_RX_PROG_STATUS_DESC_QW1_ERROR_SHIFT;
if (error == BIT(I40E_RX_PROG_STATUS_DESC_FD_TBL_FULL_SHIFT)) {
pf->fd_inv = le32_to_cpu(rx_desc->wb.qword0.hi_dword.fd_id);
if ((rx_desc->wb.qword0.hi_dword.fd_id != 0) ||
pf->fd_inv = le32_to_cpu(qw0->hi_dword.fd_id);
if (qw0->hi_dword.fd_id != 0 ||
(I40E_DEBUG_FD & pf->hw.debug_mask))
dev_warn(&pdev->dev, "ntuple filter loc = %d, could not be added\n",
pf->fd_inv);
@@ -560,7 +561,7 @@ void i40e_fd_handle_status(struct i40e_ring *rx_ring,
/* store the current atr filter count */
pf->fd_atr_cnt = i40e_get_current_atr_cnt(pf);
if ((rx_desc->wb.qword0.hi_dword.fd_id == 0) &&
if (qw0->hi_dword.fd_id == 0 &&
test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state)) {
/* These set_bit() calls aren't atomic with the
* test_bit() here, but worse case we potentially
@@ -589,7 +590,7 @@ void i40e_fd_handle_status(struct i40e_ring *rx_ring,
} else if (error == BIT(I40E_RX_PROG_STATUS_DESC_NO_FD_ENTRY_SHIFT)) {
if (I40E_DEBUG_FD & pf->hw.debug_mask)
dev_info(&pdev->dev, "ntuple filter fd_id = %d, could not be removed\n",
rx_desc->wb.qword0.hi_dword.fd_id);
qw0->hi_dword.fd_id);
}
}
@@ -1195,6 +1196,11 @@ clear_counts:
rc->total_packets = 0;
}
static struct i40e_rx_buffer *i40e_rx_bi(struct i40e_ring *rx_ring, u32 idx)
{
return &rx_ring->rx_bi[idx];
}
/**
* i40e_reuse_rx_page - page flip buffer and store it back on the ring
* @rx_ring: rx descriptor ring to store buffers on
@@ -1208,7 +1214,7 @@ static void i40e_reuse_rx_page(struct i40e_ring *rx_ring,
struct i40e_rx_buffer *new_buff;
u16 nta = rx_ring->next_to_alloc;
new_buff = &rx_ring->rx_bi[nta];
new_buff = i40e_rx_bi(rx_ring, nta);
/* update, and store next to alloc */
nta++;
@@ -1227,29 +1233,10 @@ static void i40e_reuse_rx_page(struct i40e_ring *rx_ring,
}
/**
* i40e_rx_is_programming_status - check for programming status descriptor
* @qw: qword representing status_error_len in CPU ordering
*
* The value of in the descriptor length field indicate if this
* is a programming status descriptor for flow director or FCoE
* by the value of I40E_RX_PROG_STATUS_DESC_LENGTH, otherwise
* it is a packet descriptor.
**/
static inline bool i40e_rx_is_programming_status(u64 qw)
{
/* The Rx filter programming status and SPH bit occupy the same
* spot in the descriptor. Since we don't support packet split we
* can just reuse the bit as an indication that this is a
* programming status descriptor.
*/
return qw & I40E_RXD_QW1_LENGTH_SPH_MASK;
}
/**
* i40e_clean_programming_status - try clean the programming status descriptor
* i40e_clean_programming_status - clean the programming status descriptor
* @rx_ring: the rx ring that has this descriptor
* @rx_desc: the rx descriptor written back by HW
* @qw: qword representing status_error_len in CPU ordering
* @qword0_raw: qword0
* @qword1: qword1 representing status_error_len in CPU ordering
*
* Flow director should handle FD_FILTER_STATUS to check its filter programming
* status being successful or not and take actions accordingly. FCoE should
@@ -1257,34 +1244,16 @@ static inline bool i40e_rx_is_programming_status(u64 qw)
*
* Returns an i40e_rx_buffer to reuse if the cleanup occurred, otherwise NULL.
**/
struct i40e_rx_buffer *i40e_clean_programming_status(
struct i40e_ring *rx_ring,
union i40e_rx_desc *rx_desc,
u64 qw)
void i40e_clean_programming_status(struct i40e_ring *rx_ring, u64 qword0_raw,
u64 qword1)
{
struct i40e_rx_buffer *rx_buffer;
u32 ntc;
u8 id;
if (!i40e_rx_is_programming_status(qw))
return NULL;
ntc = rx_ring->next_to_clean;
/* fetch, update, and store next to clean */
rx_buffer = &rx_ring->rx_bi[ntc++];
ntc = (ntc < rx_ring->count) ? ntc : 0;
rx_ring->next_to_clean = ntc;
prefetch(I40E_RX_DESC(rx_ring, ntc));
id = (qw & I40E_RX_PROG_STATUS_DESC_QW1_PROGID_MASK) >>
id = (qword1 & I40E_RX_PROG_STATUS_DESC_QW1_PROGID_MASK) >>
I40E_RX_PROG_STATUS_DESC_QW1_PROGID_SHIFT;
if (id == I40E_RX_PROG_STATUS_DESC_FD_FILTER_STATUS)
i40e_fd_handle_status(rx_ring, rx_desc, id);
return rx_buffer;
i40e_fd_handle_status(rx_ring, qword0_raw, qword1, id);
}
/**
@@ -1336,13 +1305,25 @@ err:
return -ENOMEM;
}
int i40e_alloc_rx_bi(struct i40e_ring *rx_ring)
{
unsigned long sz = sizeof(*rx_ring->rx_bi) * rx_ring->count;
rx_ring->rx_bi = kzalloc(sz, GFP_KERNEL);
return rx_ring->rx_bi ? 0 : -ENOMEM;
}
static void i40e_clear_rx_bi(struct i40e_ring *rx_ring)
{
memset(rx_ring->rx_bi, 0, sizeof(*rx_ring->rx_bi) * rx_ring->count);
}
/**
* i40e_clean_rx_ring - Free Rx buffers
* @rx_ring: ring to be cleaned
**/
void i40e_clean_rx_ring(struct i40e_ring *rx_ring)
{
unsigned long bi_size;
u16 i;
/* ring already cleared, nothing to do */
@@ -1361,7 +1342,7 @@ void i40e_clean_rx_ring(struct i40e_ring *rx_ring)
/* Free all the Rx ring sk_buffs */
for (i = 0; i < rx_ring->count; i++) {
struct i40e_rx_buffer *rx_bi = &rx_ring->rx_bi[i];
struct i40e_rx_buffer *rx_bi = i40e_rx_bi(rx_ring, i);
if (!rx_bi->page)
continue;
@@ -1388,8 +1369,10 @@ void i40e_clean_rx_ring(struct i40e_ring *rx_ring)
}
skip_free:
bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
memset(rx_ring->rx_bi, 0, bi_size);
if (rx_ring->xsk_umem)
i40e_clear_rx_bi_zc(rx_ring);
else
i40e_clear_rx_bi(rx_ring);
/* Zero out the descriptor ring */
memset(rx_ring->desc, 0, rx_ring->size);
@@ -1430,15 +1413,7 @@ void i40e_free_rx_resources(struct i40e_ring *rx_ring)
int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring)
{
struct device *dev = rx_ring->dev;
int err = -ENOMEM;
int bi_size;
/* warn if we are about to overwrite the pointer */
WARN_ON(rx_ring->rx_bi);
bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
rx_ring->rx_bi = kzalloc(bi_size, GFP_KERNEL);
if (!rx_ring->rx_bi)
goto err;
int err;
u64_stats_init(&rx_ring->syncp);
@@ -1451,7 +1426,7 @@ int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring)
if (!rx_ring->desc) {
dev_info(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n",
rx_ring->size);
goto err;
return -ENOMEM;
}
rx_ring->next_to_alloc = 0;
@@ -1463,16 +1438,12 @@ int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring)
err = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev,
rx_ring->queue_index);
if (err < 0)
goto err;
return err;
}
rx_ring->xdp_prog = rx_ring->vsi->xdp_prog;
return 0;
err:
kfree(rx_ring->rx_bi);
rx_ring->rx_bi = NULL;
return err;
}
/**
@@ -1592,7 +1563,7 @@ bool i40e_alloc_rx_buffers(struct i40e_ring *rx_ring, u16 cleaned_count)
return false;
rx_desc = I40E_RX_DESC(rx_ring, ntu);
bi = &rx_ring->rx_bi[ntu];
bi = i40e_rx_bi(rx_ring, ntu);
do {
if (!i40e_alloc_mapped_page(rx_ring, bi))
@@ -1614,7 +1585,7 @@ bool i40e_alloc_rx_buffers(struct i40e_ring *rx_ring, u16 cleaned_count)
ntu++;
if (unlikely(ntu == rx_ring->count)) {
rx_desc = I40E_RX_DESC(rx_ring, 0);
bi = rx_ring->rx_bi;
bi = i40e_rx_bi(rx_ring, 0);
ntu = 0;
}
@@ -1981,7 +1952,7 @@ static struct i40e_rx_buffer *i40e_get_rx_buffer(struct i40e_ring *rx_ring,
{
struct i40e_rx_buffer *rx_buffer;
rx_buffer = &rx_ring->rx_bi[rx_ring->next_to_clean];
rx_buffer = i40e_rx_bi(rx_ring, rx_ring->next_to_clean);
prefetchw(rx_buffer->page);
/* we are reusing so sync this buffer for CPU use */
@@ -2382,9 +2353,12 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget)
*/
dma_rmb();
rx_buffer = i40e_clean_programming_status(rx_ring, rx_desc,
qword);
if (unlikely(rx_buffer)) {
if (i40e_rx_is_programming_status(qword)) {
i40e_clean_programming_status(rx_ring,
rx_desc->raw.qword[0],
qword);
rx_buffer = i40e_rx_bi(rx_ring, rx_ring->next_to_clean);
i40e_inc_ntc(rx_ring);
i40e_reuse_rx_page(rx_ring, rx_buffer);
cleaned_count++;
continue;

17
drivers/net/ethernet/intel/i40e/i40e_txrx.h
View File

@@ -296,17 +296,9 @@ struct i40e_tx_buffer {
struct i40e_rx_buffer {
dma_addr_t dma;
union {
struct {
struct page *page;
__u32 page_offset;
__u16 pagecnt_bias;
};
struct {
void *addr;
u64 handle;
};
};
struct page *page;
__u32 page_offset;
__u16 pagecnt_bias;
};
struct i40e_queue_stats {
@@ -358,6 +350,7 @@ struct i40e_ring {
union {
struct i40e_tx_buffer *tx_bi;
struct i40e_rx_buffer *rx_bi;
struct xdp_buff **rx_bi_zc;
};
DECLARE_BITMAP(state, __I40E_RING_STATE_NBITS);
u16 queue_index; /* Queue number of ring */
@@ -419,7 +412,6 @@ struct i40e_ring {
struct i40e_channel *ch;
struct xdp_rxq_info xdp_rxq;
struct xdp_umem *xsk_umem;
struct zero_copy_allocator zca; /* ZC allocator anchor */
} ____cacheline_internodealigned_in_smp;
static inline bool ring_uses_build_skb(struct i40e_ring *ring)
@@ -495,6 +487,7 @@ int __i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size);
bool __i40e_chk_linearize(struct sk_buff *skb);
int i40e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames,
u32 flags);
int i40e_alloc_rx_bi(struct i40e_ring *rx_ring);
/**
* i40e_get_head - Retrieve head from head writeback

40
drivers/net/ethernet/intel/i40e/i40e_txrx_common.h
View File

@@ -4,13 +4,9 @@
#ifndef I40E_TXRX_COMMON_
#define I40E_TXRX_COMMON_
void i40e_fd_handle_status(struct i40e_ring *rx_ring,
union i40e_rx_desc *rx_desc, u8 prog_id);
int i40e_xmit_xdp_tx_ring(struct xdp_buff *xdp, struct i40e_ring *xdp_ring);
struct i40e_rx_buffer *i40e_clean_programming_status(
struct i40e_ring *rx_ring,
union i40e_rx_desc *rx_desc,
u64 qw);
void i40e_clean_programming_status(struct i40e_ring *rx_ring, u64 qword0_raw,
u64 qword1);
void i40e_process_skb_fields(struct i40e_ring *rx_ring,
union i40e_rx_desc *rx_desc, struct sk_buff *skb);
void i40e_xdp_ring_update_tail(struct i40e_ring *xdp_ring);
@@ -84,6 +80,38 @@ static inline void i40e_arm_wb(struct i40e_ring *tx_ring,
}
}
/**
* i40e_rx_is_programming_status - check for programming status descriptor
* @qword1: qword1 representing status_error_len in CPU ordering
*
* The value of in the descriptor length field indicate if this
* is a programming status descriptor for flow director or FCoE
* by the value of I40E_RX_PROG_STATUS_DESC_LENGTH, otherwise
* it is a packet descriptor.
**/
static inline bool i40e_rx_is_programming_status(u64 qword1)
{
/* The Rx filter programming status and SPH bit occupy the same
* spot in the descriptor. Since we don't support packet split we
* can just reuse the bit as an indication that this is a
* programming status descriptor.
*/
return qword1 & I40E_RXD_QW1_LENGTH_SPH_MASK;
}
/**
* i40e_inc_ntc: Advance the next_to_clean index
* @rx_ring: Rx ring
**/
static inline void i40e_inc_ntc(struct i40e_ring *rx_ring)
{
u32 ntc = rx_ring->next_to_clean + 1;
ntc = (ntc < rx_ring->count) ? ntc : 0;
rx_ring->next_to_clean = ntc;
prefetch(I40E_RX_DESC(rx_ring, ntc));
}
void i40e_xsk_clean_rx_ring(struct i40e_ring *rx_ring);
void i40e_xsk_clean_tx_ring(struct i40e_ring *tx_ring);
bool i40e_xsk_any_rx_ring_enabled(struct i40e_vsi *vsi);

5
drivers/net/ethernet/intel/i40e/i40e_type.h
View File

@@ -689,7 +689,7 @@ union i40e_32byte_rx_desc {
__le64 rsvd2;
} read;
struct {
struct {
struct i40e_32b_rx_wb_qw0 {
struct {
union {
__le16 mirroring_status;
@@ -727,6 +727,9 @@ union i40e_32byte_rx_desc {
} hi_dword;
} qword3;
} wb; /* writeback */
struct {
u64 qword[4];
} raw;
};
enum i40e_rx_desc_status_bits {

379
drivers/net/ethernet/intel/i40e/i40e_xsk.c
View File

@@ -2,68 +2,30 @@
/* Copyright(c) 2018 Intel Corporation. */
#include <linux/bpf_trace.h>
#include <net/xdp_sock.h>
#include <net/xdp_sock_drv.h>
#include <net/xdp.h>
#include "i40e.h"
#include "i40e_txrx_common.h"
#include "i40e_xsk.h"
/**
* i40e_xsk_umem_dma_map - DMA maps all UMEM memory for the netdev
* @vsi: Current VSI
* @umem: UMEM to DMA map
*
* Returns 0 on success, <0 on failure
**/
static int i40e_xsk_umem_dma_map(struct i40e_vsi *vsi, struct xdp_umem *umem)
int i40e_alloc_rx_bi_zc(struct i40e_ring *rx_ring)
{
struct i40e_pf *pf = vsi->back;
struct device *dev;
unsigned int i, j;
dma_addr_t dma;
unsigned long sz = sizeof(*rx_ring->rx_bi_zc) * rx_ring->count;
dev = &pf->pdev->dev;
for (i = 0; i < umem->npgs; i++) {
dma = dma_map_page_attrs(dev, umem->pgs[i], 0, PAGE_SIZE,
DMA_BIDIRECTIONAL, I40E_RX_DMA_ATTR);
if (dma_mapping_error(dev, dma))
goto out_unmap;
umem->pages[i].dma = dma;
}
return 0;
out_unmap:
for (j = 0; j < i; j++) {
dma_unmap_page_attrs(dev, umem->pages[i].dma, PAGE_SIZE,
DMA_BIDIRECTIONAL, I40E_RX_DMA_ATTR);
umem->pages[i].dma = 0;
}
return -1;
rx_ring->rx_bi_zc = kzalloc(sz, GFP_KERNEL);
return rx_ring->rx_bi_zc ? 0 : -ENOMEM;
}
/**
* i40e_xsk_umem_dma_unmap - DMA unmaps all UMEM memory for the netdev
* @vsi: Current VSI
* @umem: UMEM to DMA map
**/
static void i40e_xsk_umem_dma_unmap(struct i40e_vsi *vsi, struct xdp_umem *umem)
void i40e_clear_rx_bi_zc(struct i40e_ring *rx_ring)
{
struct i40e_pf *pf = vsi->back;
struct device *dev;
unsigned int i;
memset(rx_ring->rx_bi_zc, 0,
sizeof(*rx_ring->rx_bi_zc) * rx_ring->count);
}
dev = &pf->pdev->dev;
for (i = 0; i < umem->npgs; i++) {
dma_unmap_page_attrs(dev, umem->pages[i].dma, PAGE_SIZE,
DMA_BIDIRECTIONAL, I40E_RX_DMA_ATTR);
umem->pages[i].dma = 0;
}
static struct xdp_buff **i40e_rx_bi(struct i40e_ring *rx_ring, u32 idx)
{
return &rx_ring->rx_bi_zc[idx];
}
/**
@@ -78,7 +40,6 @@ static int i40e_xsk_umem_enable(struct i40e_vsi *vsi, struct xdp_umem *umem,
u16 qid)
{
struct net_device *netdev = vsi->netdev;
struct xdp_umem_fq_reuse *reuseq;
bool if_running;
int err;
@@ -92,13 +53,7 @@ static int i40e_xsk_umem_enable(struct i40e_vsi *vsi, struct xdp_umem *umem,
qid >= netdev->real_num_tx_queues)
return -EINVAL;
reuseq = xsk_reuseq_prepare(vsi->rx_rings[0]->count);
if (!reuseq)
return -ENOMEM;
xsk_reuseq_free(xsk_reuseq_swap(umem, reuseq));
err = i40e_xsk_umem_dma_map(vsi, umem);
err = xsk_buff_dma_map(umem, &vsi->back->pdev->dev, I40E_RX_DMA_ATTR);
if (err)
return err;
@@ -151,7 +106,7 @@ static int i40e_xsk_umem_disable(struct i40e_vsi *vsi, u16 qid)
}
clear_bit(qid, vsi->af_xdp_zc_qps);
i40e_xsk_umem_dma_unmap(vsi, umem);
xsk_buff_dma_unmap(umem, I40E_RX_DMA_ATTR);
if (if_running) {
err = i40e_queue_pair_enable(vsi, qid);
@@ -190,11 +145,9 @@ int i40e_xsk_umem_setup(struct i40e_vsi *vsi, struct xdp_umem *umem,
**/
static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp)
{
struct xdp_umem *umem = rx_ring->xsk_umem;
int err, result = I40E_XDP_PASS;
struct i40e_ring *xdp_ring;
struct bpf_prog *xdp_prog;
u64 offset;
u32 act;
rcu_read_lock();
@@ -203,9 +156,6 @@ static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp)
*/
xdp_prog = READ_ONCE(rx_ring->xdp_prog);
act = bpf_prog_run_xdp(xdp_prog, xdp);
offset = xdp->data - xdp->data_hard_start;
xdp->handle = xsk_umem_adjust_offset(umem, xdp->handle, offset);
switch (act) {
case XDP_PASS:
@@ -232,107 +182,26 @@ static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp)
return result;
}
/**
* i40e_alloc_buffer_zc - Allocates an i40e_rx_buffer
* @rx_ring: Rx ring
* @bi: Rx buffer to populate
*
* This function allocates an Rx buffer. The buffer can come from fill
* queue, or via the recycle queue (next_to_alloc).
*
* Returns true for a successful allocation, false otherwise
**/
static bool i40e_alloc_buffer_zc(struct i40e_ring *rx_ring,
struct i40e_rx_buffer *bi)
{
struct xdp_umem *umem = rx_ring->xsk_umem;
void *addr = bi->addr;
u64 handle, hr;
if (addr) {
rx_ring->rx_stats.page_reuse_count++;
return true;
}
if (!xsk_umem_peek_addr(umem, &handle)) {
rx_ring->rx_stats.alloc_page_failed++;
return false;
}
hr = umem->headroom + XDP_PACKET_HEADROOM;
bi->dma = xdp_umem_get_dma(umem, handle);
bi->dma += hr;
bi->addr = xdp_umem_get_data(umem, handle);
bi->addr += hr;
bi->handle = xsk_umem_adjust_offset(umem, handle, umem->headroom);
xsk_umem_release_addr(umem);
return true;
}
/**
* i40e_alloc_buffer_slow_zc - Allocates an i40e_rx_buffer
* @rx_ring: Rx ring
* @bi: Rx buffer to populate
*
* This function allocates an Rx buffer. The buffer can come from fill
* queue, or via the reuse queue.
*
* Returns true for a successful allocation, false otherwise
**/
static bool i40e_alloc_buffer_slow_zc(struct i40e_ring *rx_ring,
struct i40e_rx_buffer *bi)
{
struct xdp_umem *umem = rx_ring->xsk_umem;
u64 handle, hr;
if (!xsk_umem_peek_addr_rq(umem, &handle)) {
rx_ring->rx_stats.alloc_page_failed++;
return false;
}
handle &= rx_ring->xsk_umem->chunk_mask;
hr = umem->headroom + XDP_PACKET_HEADROOM;
bi->dma = xdp_umem_get_dma(umem, handle);
bi->dma += hr;
bi->addr = xdp_umem_get_data(umem, handle);
bi->addr += hr;
bi->handle = xsk_umem_adjust_offset(umem, handle, umem->headroom);
xsk_umem_release_addr_rq(umem);
return true;
}
static __always_inline bool
__i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 count,
bool alloc(struct i40e_ring *rx_ring,
struct i40e_rx_buffer *bi))
bool i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 count)
{
u16 ntu = rx_ring->next_to_use;
union i40e_rx_desc *rx_desc;
struct i40e_rx_buffer *bi;
struct xdp_buff **bi, *xdp;
dma_addr_t dma;
bool ok = true;
rx_desc = I40E_RX_DESC(rx_ring, ntu);
bi = &rx_ring->rx_bi[ntu];
bi = i40e_rx_bi(rx_ring, ntu);
do {
if (!alloc(rx_ring, bi)) {
xdp = xsk_buff_alloc(rx_ring->xsk_umem);
if (!xdp) {
ok = false;
goto no_buffers;
}
dma_sync_single_range_for_device(rx_ring->dev, bi->dma, 0,
rx_ring->rx_buf_len,
DMA_BIDIRECTIONAL);
rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
*bi = xdp;
dma = xsk_buff_xdp_get_dma(xdp);
rx_desc->read.pkt_addr = cpu_to_le64(dma);
rx_desc->read.hdr_addr = 0;
rx_desc++;
bi++;
@@ -340,11 +209,10 @@ __i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 count,
if (unlikely(ntu == rx_ring->count)) {
rx_desc = I40E_RX_DESC(rx_ring, 0);
bi = rx_ring->rx_bi;
bi = i40e_rx_bi(rx_ring, 0);
ntu = 0;
}
rx_desc->wb.qword1.status_error_len = 0;
count--;
} while (count);
@@ -355,128 +223,9 @@ no_buffers:
return ok;
}
/**
* i40e_alloc_rx_buffers_zc - Allocates a number of Rx buffers
* @rx_ring: Rx ring
* @count: The number of buffers to allocate
*
* This function allocates a number of Rx buffers from the reuse queue
* or fill ring and places them on the Rx ring.
*
* Returns true for a successful allocation, false otherwise
**/
bool i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 count)
{
return __i40e_alloc_rx_buffers_zc(rx_ring, count,
i40e_alloc_buffer_slow_zc);
}
/**
* i40e_alloc_rx_buffers_fast_zc - Allocates a number of Rx buffers
* @rx_ring: Rx ring
* @count: The number of buffers to allocate
*
* This function allocates a number of Rx buffers from the fill ring
* or the internal recycle mechanism and places them on the Rx ring.
*
* Returns true for a successful allocation, false otherwise
**/
static bool i40e_alloc_rx_buffers_fast_zc(struct i40e_ring *rx_ring, u16 count)
{
return __i40e_alloc_rx_buffers_zc(rx_ring, count,
i40e_alloc_buffer_zc);
}
/**
* i40e_get_rx_buffer_zc - Return the current Rx buffer
* @rx_ring: Rx ring
* @size: The size of the rx buffer (read from descriptor)
*
* This function returns the current, received Rx buffer, and also
* does DMA synchronization. the Rx ring.
*
* Returns the received Rx buffer
**/
static struct i40e_rx_buffer *i40e_get_rx_buffer_zc(struct i40e_ring *rx_ring,
const unsigned int size)
{
struct i40e_rx_buffer *bi;
bi = &rx_ring->rx_bi[rx_ring->next_to_clean];
/* we are reusing so sync this buffer for CPU use */
dma_sync_single_range_for_cpu(rx_ring->dev,
bi->dma, 0,
size,
DMA_BIDIRECTIONAL);
return bi;
}
/**
* i40e_reuse_rx_buffer_zc - Recycle an Rx buffer
* @rx_ring: Rx ring
* @old_bi: The Rx buffer to recycle
*
* This function recycles a finished Rx buffer, and places it on the
* recycle queue (next_to_alloc).
**/
static void i40e_reuse_rx_buffer_zc(struct i40e_ring *rx_ring,
struct i40e_rx_buffer *old_bi)
{
struct i40e_rx_buffer *new_bi = &rx_ring->rx_bi[rx_ring->next_to_alloc];
u16 nta = rx_ring->next_to_alloc;
/* update, and store next to alloc */
nta++;
rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
/* transfer page from old buffer to new buffer */
new_bi->dma = old_bi->dma;
new_bi->addr = old_bi->addr;
new_bi->handle = old_bi->handle;
old_bi->addr = NULL;
}
/**
* i40e_zca_free - Free callback for MEM_TYPE_ZERO_COPY allocations
* @alloc: Zero-copy allocator
* @handle: Buffer handle
**/
void i40e_zca_free(struct zero_copy_allocator *alloc, unsigned long handle)
{
struct i40e_rx_buffer *bi;
struct i40e_ring *rx_ring;
u64 hr, mask;
u16 nta;
rx_ring = container_of(alloc, struct i40e_ring, zca);
hr = rx_ring->xsk_umem->headroom + XDP_PACKET_HEADROOM;
mask = rx_ring->xsk_umem->chunk_mask;
nta = rx_ring->next_to_alloc;
bi = &rx_ring->rx_bi[nta];
nta++;
rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
handle &= mask;
bi->dma = xdp_umem_get_dma(rx_ring->xsk_umem, handle);
bi->dma += hr;
bi->addr = xdp_umem_get_data(rx_ring->xsk_umem, handle);
bi->addr += hr;
bi->handle = xsk_umem_adjust_offset(rx_ring->xsk_umem, (u64)handle,
rx_ring->xsk_umem->headroom);
}
/**
* i40e_construct_skb_zc - Create skbufff from zero-copy Rx buffer
* @rx_ring: Rx ring
* @bi: Rx buffer
* @xdp: xdp_buff
*
* This functions allocates a new skb from a zero-copy Rx buffer.
@@ -484,7 +233,6 @@ void i40e_zca_free(struct zero_copy_allocator *alloc, unsigned long handle)
* Returns the skb, or NULL on failure.
**/
static struct sk_buff *i40e_construct_skb_zc(struct i40e_ring *rx_ring,
struct i40e_rx_buffer *bi,
struct xdp_buff *xdp)
{
unsigned int metasize = xdp->data - xdp->data_meta;
@@ -503,23 +251,10 @@ static struct sk_buff *i40e_construct_skb_zc(struct i40e_ring *rx_ring,
if (metasize)
skb_metadata_set(skb, metasize);
i40e_reuse_rx_buffer_zc(rx_ring, bi);
xsk_buff_free(xdp);
return skb;
}
/**
* i40e_inc_ntc: Advance the next_to_clean index
* @rx_ring: Rx ring
**/
static void i40e_inc_ntc(struct i40e_ring *rx_ring)
{
u32 ntc = rx_ring->next_to_clean + 1;
ntc = (ntc < rx_ring->count) ? ntc : 0;
rx_ring->next_to_clean = ntc;
prefetch(I40E_RX_DESC(rx_ring, ntc));
}
/**
* i40e_clean_rx_irq_zc - Consumes Rx packets from the hardware ring
* @rx_ring: Rx ring
@@ -531,25 +266,20 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget)
{
unsigned int total_rx_bytes = 0, total_rx_packets = 0;
u16 cleaned_count = I40E_DESC_UNUSED(rx_ring);
struct xdp_umem *umem = rx_ring->xsk_umem;
unsigned int xdp_res, xdp_xmit = 0;
bool failure = false;
struct sk_buff *skb;
struct xdp_buff xdp;
xdp.rxq = &rx_ring->xdp_rxq;
xdp.frame_sz = xsk_umem_xdp_frame_sz(umem);
while (likely(total_rx_packets < (unsigned int)budget)) {
struct i40e_rx_buffer *bi;
union i40e_rx_desc *rx_desc;
struct xdp_buff **bi;
unsigned int size;
u64 qword;
if (cleaned_count >= I40E_RX_BUFFER_WRITE) {
failure = failure ||
!i40e_alloc_rx_buffers_fast_zc(rx_ring,
cleaned_count);
!i40e_alloc_rx_buffers_zc(rx_ring,
cleaned_count);
cleaned_count = 0;
}
@@ -562,35 +292,36 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget)
*/
dma_rmb();
bi = i40e_clean_programming_status(rx_ring, rx_desc,
qword);
if (unlikely(bi)) {
i40e_reuse_rx_buffer_zc(rx_ring, bi);
if (i40e_rx_is_programming_status(qword)) {
i40e_clean_programming_status(rx_ring,
rx_desc->raw.qword[0],
qword);
bi = i40e_rx_bi(rx_ring, rx_ring->next_to_clean);
xsk_buff_free(*bi);
*bi = NULL;
cleaned_count++;
i40e_inc_ntc(rx_ring);
continue;
}
bi = i40e_rx_bi(rx_ring, rx_ring->next_to_clean);
size = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>
I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
if (!size)
break;
bi = i40e_get_rx_buffer_zc(rx_ring, size);
xdp.data = bi->addr;
xdp.data_meta = xdp.data;
xdp.data_hard_start = xdp.data - XDP_PACKET_HEADROOM;
xdp.data_end = xdp.data + size;
xdp.handle = bi->handle;
bi = i40e_rx_bi(rx_ring, rx_ring->next_to_clean);
(*bi)->data_end = (*bi)->data + size;
xsk_buff_dma_sync_for_cpu(*bi);
xdp_res = i40e_run_xdp_zc(rx_ring, &xdp);
xdp_res = i40e_run_xdp_zc(rx_ring, *bi);
if (xdp_res) {
if (xdp_res & (I40E_XDP_TX | I40E_XDP_REDIR)) {
if (xdp_res & (I40E_XDP_TX | I40E_XDP_REDIR))
xdp_xmit |= xdp_res;
bi->addr = NULL;
} else {
i40e_reuse_rx_buffer_zc(rx_ring, bi);
}
else
xsk_buff_free(*bi);
*bi = NULL;
total_rx_bytes += size;
total_rx_packets++;
@@ -606,7 +337,8 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget)
* BIT(I40E_RXD_QW1_ERROR_SHIFT). This is due to that
* SBP is *not* set in PRT_SBPVSI (default not set).
*/
skb = i40e_construct_skb_zc(rx_ring, bi, &xdp);
skb = i40e_construct_skb_zc(rx_ring, *bi);
*bi = NULL;
if (!skb) {
rx_ring->rx_stats.alloc_buff_failed++;
break;
@@ -664,10 +396,9 @@ static bool i40e_xmit_zc(struct i40e_ring *xdp_ring, unsigned int budget)
if (!xsk_umem_consume_tx(xdp_ring->xsk_umem, &desc))
break;
dma = xdp_umem_get_dma(xdp_ring->xsk_umem, desc.addr);
dma_sync_single_for_device(xdp_ring->dev, dma, desc.len,
DMA_BIDIRECTIONAL);
dma = xsk_buff_raw_get_dma(xdp_ring->xsk_umem, desc.addr);
xsk_buff_raw_dma_sync_for_device(xdp_ring->xsk_umem, dma,
desc.len);
tx_bi = &xdp_ring->tx_bi[xdp_ring->next_to_use];
tx_bi->bytecount = desc.len;
@@ -826,13 +557,13 @@ void i40e_xsk_clean_rx_ring(struct i40e_ring *rx_ring)
u16 i;
for (i = 0; i < rx_ring->count; i++) {
struct i40e_rx_buffer *rx_bi = &rx_ring->rx_bi[i];
struct xdp_buff *rx_bi = *i40e_rx_bi(rx_ring, i);
if (!rx_bi->addr)
if (!rx_bi)
continue;
xsk_umem_fq_reuse(rx_ring->xsk_umem, rx_bi->handle);
rx_bi->addr = NULL;
xsk_buff_free(rx_bi);
rx_bi = NULL;
}
}

3
drivers/net/ethernet/intel/i40e/i40e_xsk.h
View File

@@ -12,12 +12,13 @@ int i40e_queue_pair_disable(struct i40e_vsi *vsi, int queue_pair);
int i40e_queue_pair_enable(struct i40e_vsi *vsi, int queue_pair);
int i40e_xsk_umem_setup(struct i40e_vsi *vsi, struct xdp_umem *umem,
u16 qid);
void i40e_zca_free(struct zero_copy_allocator *alloc, unsigned long handle);
bool i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 cleaned_count);
int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget);
bool i40e_clean_xdp_tx_irq(struct i40e_vsi *vsi,
struct i40e_ring *tx_ring, int napi_budget);
int i40e_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags);
int i40e_alloc_rx_bi_zc(struct i40e_ring *rx_ring);
void i40e_clear_rx_bi_zc(struct i40e_ring *rx_ring);
#endif /* _I40E_XSK_H_ */

16
drivers/net/ethernet/intel/ice/ice_base.c
View File

@@ -1,6 +1,7 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2019, Intel Corporation. */
#include <net/xdp_sock_drv.h>
#include "ice_base.h"
#include "ice_dcb_lib.h"
@@ -308,24 +309,23 @@ int ice_setup_rx_ctx(struct ice_ring *ring)
if (ring->xsk_umem) {
xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq);
ring->rx_buf_len = ring->xsk_umem->chunk_size_nohr -
XDP_PACKET_HEADROOM;
ring->rx_buf_len =
xsk_umem_get_rx_frame_size(ring->xsk_umem);
/* For AF_XDP ZC, we disallow packets to span on
* multiple buffers, thus letting us skip that
* handling in the fast-path.
*/
chain_len = 1;
ring->zca.free = ice_zca_free;
err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
MEM_TYPE_ZERO_COPY,
&ring->zca);
MEM_TYPE_XSK_BUFF_POOL,
NULL);
if (err)
return err;
xsk_buff_set_rxq_info(ring->xsk_umem, &ring->xdp_rxq);
dev_info(ice_pf_to_dev(vsi->back), "Registered XDP mem model MEM_TYPE_ZERO_COPY on Rx ring %d\n",
dev_info(ice_pf_to_dev(vsi->back), "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n",
ring->q_index);
} else {
ring->zca.free = NULL;
if (!xdp_rxq_info_is_reg(&ring->xdp_rxq))
/* coverity[check_return] */
xdp_rxq_info_reg(&ring->xdp_rxq,
@@ -426,7 +426,7 @@ int ice_setup_rx_ctx(struct ice_ring *ring)
writel(0, ring->tail);
err = ring->xsk_umem ?
ice_alloc_rx_bufs_slow_zc(ring, ICE_DESC_UNUSED(ring)) :
ice_alloc_rx_bufs_zc(ring, ICE_DESC_UNUSED(ring)) :
ice_alloc_rx_bufs(ring, ICE_DESC_UNUSED(ring));
if (err)
dev_info(ice_pf_to_dev(vsi->back), "Failed allocate some buffers on %sRx ring %d (pf_q %d)\n",

8
drivers/net/ethernet/intel/ice/ice_txrx.h
View File

@@ -158,17 +158,16 @@ struct ice_tx_offload_params {
};
struct ice_rx_buf {
struct sk_buff *skb;
dma_addr_t dma;
union {
struct {
struct sk_buff *skb;
dma_addr_t dma;
struct page *page;
unsigned int page_offset;
u16 pagecnt_bias;
};
struct {
void *addr;
u64 handle;
struct xdp_buff *xdp;
};
};
};
@@ -292,7 +291,6 @@ struct ice_ring {
struct rcu_head rcu; /* to avoid race on free */
struct bpf_prog *xdp_prog;
struct xdp_umem *xsk_umem;
struct zero_copy_allocator zca;
/* CL3 - 3rd cacheline starts here */
struct xdp_rxq_info xdp_rxq;
/* CLX - the below items are only accessed infrequently and should be

380
drivers/net/ethernet/intel/ice/ice_xsk.c
View File

@@ -2,7 +2,7 @@
/* Copyright (c) 2019, Intel Corporation. */
#include <linux/bpf_trace.h>
#include <net/xdp_sock.h>
#include <net/xdp_sock_drv.h>
#include <net/xdp.h>
#include "ice.h"
#include "ice_base.h"
@@ -279,28 +279,6 @@ static int ice_xsk_alloc_umems(struct ice_vsi *vsi)
return 0;
}
/**
* ice_xsk_add_umem - add a UMEM region for XDP sockets
* @vsi: VSI to which the UMEM will be added
* @umem: pointer to a requested UMEM region
* @qid: queue ID
*
* Returns 0 on success, negative on error
*/
static int ice_xsk_add_umem(struct ice_vsi *vsi, struct xdp_umem *umem, u16 qid)
{
int err;
err = ice_xsk_alloc_umems(vsi);
if (err)
return err;
vsi->xsk_umems[qid] = umem;
vsi->num_xsk_umems_used++;
return 0;
}
/**
* ice_xsk_remove_umem - Remove an UMEM for a certain ring/qid
* @vsi: VSI from which the VSI will be removed
@@ -318,65 +296,6 @@ static void ice_xsk_remove_umem(struct ice_vsi *vsi, u16 qid)
}
}
/**
* ice_xsk_umem_dma_map - DMA map UMEM region for XDP sockets
* @vsi: VSI to map the UMEM region
* @umem: UMEM to map
*
* Returns 0 on success, negative on error
*/
static int ice_xsk_umem_dma_map(struct ice_vsi *vsi, struct xdp_umem *umem)
{
struct ice_pf *pf = vsi->back;
struct device *dev;
unsigned int i;
dev = ice_pf_to_dev(pf);
for (i = 0; i < umem->npgs; i++) {
dma_addr_t dma = dma_map_page_attrs(dev, umem->pgs[i], 0,
PAGE_SIZE,
DMA_BIDIRECTIONAL,
ICE_RX_DMA_ATTR);
if (dma_mapping_error(dev, dma)) {
dev_dbg(dev, "XSK UMEM DMA mapping error on page num %d\n",
i);
goto out_unmap;
}
umem->pages[i].dma = dma;
}
return 0;
out_unmap:
for (; i > 0; i--) {
dma_unmap_page_attrs(dev, umem->pages[i].dma, PAGE_SIZE,
DMA_BIDIRECTIONAL, ICE_RX_DMA_ATTR);
umem->pages[i].dma = 0;
}
return -EFAULT;
}
/**
* ice_xsk_umem_dma_unmap - DMA unmap UMEM region for XDP sockets
* @vsi: VSI from which the UMEM will be unmapped
* @umem: UMEM to unmap
*/
static void ice_xsk_umem_dma_unmap(struct ice_vsi *vsi, struct xdp_umem *umem)
{
struct ice_pf *pf = vsi->back;
struct device *dev;
unsigned int i;
dev = ice_pf_to_dev(pf);
for (i = 0; i < umem->npgs; i++) {
dma_unmap_page_attrs(dev, umem->pages[i].dma, PAGE_SIZE,
DMA_BIDIRECTIONAL, ICE_RX_DMA_ATTR);
umem->pages[i].dma = 0;
}
}
/**
* ice_xsk_umem_disable - disable a UMEM region
@@ -391,7 +310,7 @@ static int ice_xsk_umem_disable(struct ice_vsi *vsi, u16 qid)
!vsi->xsk_umems[qid])
return -EINVAL;
ice_xsk_umem_dma_unmap(vsi, vsi->xsk_umems[qid]);
xsk_buff_dma_unmap(vsi->xsk_umems[qid], ICE_RX_DMA_ATTR);
ice_xsk_remove_umem(vsi, qid);
return 0;
@@ -408,7 +327,6 @@ static int ice_xsk_umem_disable(struct ice_vsi *vsi, u16 qid)
static int
ice_xsk_umem_enable(struct ice_vsi *vsi, struct xdp_umem *umem, u16 qid)
{
struct xdp_umem_fq_reuse *reuseq;
int err;
if (vsi->type != ICE_VSI_PF)
@@ -419,20 +337,18 @@ ice_xsk_umem_enable(struct ice_vsi *vsi, struct xdp_umem *umem, u16 qid)
if (qid >= vsi->num_xsk_umems)
return -EINVAL;
if (vsi->xsk_umems && vsi->xsk_umems[qid])
return -EBUSY;
reuseq = xsk_reuseq_prepare(vsi->rx_rings[0]->count);
if (!reuseq)
return -ENOMEM;
xsk_reuseq_free(xsk_reuseq_swap(umem, reuseq));
err = ice_xsk_umem_dma_map(vsi, umem);
err = ice_xsk_alloc_umems(vsi);
if (err)
return err;
err = ice_xsk_add_umem(vsi, umem, qid);
if (vsi->xsk_umems && vsi->xsk_umems[qid])
return -EBUSY;
vsi->xsk_umems[qid] = umem;
vsi->num_xsk_umems_used++;
err = xsk_buff_dma_map(vsi->xsk_umems[qid], ice_pf_to_dev(vsi->back),
ICE_RX_DMA_ATTR);
if (err)
return err;
@@ -483,138 +399,23 @@ xsk_umem_if_up:
return ret;
}
/**
* ice_zca_free - Callback for MEM_TYPE_ZERO_COPY allocations
* @zca: zero-cpoy allocator
* @handle: Buffer handle
*/
void ice_zca_free(struct zero_copy_allocator *zca, unsigned long handle)
{
struct ice_rx_buf *rx_buf;
struct ice_ring *rx_ring;
struct xdp_umem *umem;
u64 hr, mask;
u16 nta;
rx_ring = container_of(zca, struct ice_ring, zca);
umem = rx_ring->xsk_umem;
hr = umem->headroom + XDP_PACKET_HEADROOM;
mask = umem->chunk_mask;
nta = rx_ring->next_to_alloc;
rx_buf = &rx_ring->rx_buf[nta];
nta++;
rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
handle &= mask;
rx_buf->dma = xdp_umem_get_dma(umem, handle);
rx_buf->dma += hr;
rx_buf->addr = xdp_umem_get_data(umem, handle);
rx_buf->addr += hr;
rx_buf->handle = (u64)handle + umem->headroom;
}
/**
* ice_alloc_buf_fast_zc - Retrieve buffer address from XDP umem
* @rx_ring: ring with an xdp_umem bound to it
* @rx_buf: buffer to which xsk page address will be assigned
*
* This function allocates an Rx buffer in the hot path.
* The buffer can come from fill queue or recycle queue.
*
* Returns true if an assignment was successful, false if not.
*/
static __always_inline bool
ice_alloc_buf_fast_zc(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf)
{
struct xdp_umem *umem = rx_ring->xsk_umem;
void *addr = rx_buf->addr;
u64 handle, hr;
if (addr) {
rx_ring->rx_stats.page_reuse_count++;
return true;
}
if (!xsk_umem_peek_addr(umem, &handle)) {
rx_ring->rx_stats.alloc_page_failed++;
return false;
}
hr = umem->headroom + XDP_PACKET_HEADROOM;
rx_buf->dma = xdp_umem_get_dma(umem, handle);
rx_buf->dma += hr;
rx_buf->addr = xdp_umem_get_data(umem, handle);
rx_buf->addr += hr;
rx_buf->handle = handle + umem->headroom;
xsk_umem_release_addr(umem);
return true;
}
/**
* ice_alloc_buf_slow_zc - Retrieve buffer address from XDP umem
* @rx_ring: ring with an xdp_umem bound to it
* @rx_buf: buffer to which xsk page address will be assigned
*
* This function allocates an Rx buffer in the slow path.
* The buffer can come from fill queue or recycle queue.
*
* Returns true if an assignment was successful, false if not.
*/
static __always_inline bool
ice_alloc_buf_slow_zc(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf)
{
struct xdp_umem *umem = rx_ring->xsk_umem;
u64 handle, headroom;
if (!xsk_umem_peek_addr_rq(umem, &handle)) {
rx_ring->rx_stats.alloc_page_failed++;
return false;
}
handle &= umem->chunk_mask;
headroom = umem->headroom + XDP_PACKET_HEADROOM;
rx_buf->dma = xdp_umem_get_dma(umem, handle);
rx_buf->dma += headroom;
rx_buf->addr = xdp_umem_get_data(umem, handle);
rx_buf->addr += headroom;
rx_buf->handle = handle + umem->headroom;
xsk_umem_release_addr_rq(umem);
return true;
}
/**
* ice_alloc_rx_bufs_zc - allocate a number of Rx buffers
* @rx_ring: Rx ring
* @count: The number of buffers to allocate
* @alloc: the function pointer to call for allocation
*
* This function allocates a number of Rx buffers from the fill ring
* or the internal recycle mechanism and places them on the Rx ring.
*
* Returns false if all allocations were successful, true if any fail.
*/
static bool
ice_alloc_rx_bufs_zc(struct ice_ring *rx_ring, int count,
bool (*alloc)(struct ice_ring *, struct ice_rx_buf *))
bool ice_alloc_rx_bufs_zc(struct ice_ring *rx_ring, u16 count)
{
union ice_32b_rx_flex_desc *rx_desc;
u16 ntu = rx_ring->next_to_use;
struct ice_rx_buf *rx_buf;
bool ret = false;
dma_addr_t dma;
if (!count)
return false;
@@ -623,16 +424,14 @@ ice_alloc_rx_bufs_zc(struct ice_ring *rx_ring, int count,
rx_buf = &rx_ring->rx_buf[ntu];
do {
if (!alloc(rx_ring, rx_buf)) {
rx_buf->xdp = xsk_buff_alloc(rx_ring->xsk_umem);
if (!rx_buf->xdp) {
ret = true;
break;
}
dma_sync_single_range_for_device(rx_ring->dev, rx_buf->dma, 0,
rx_ring->rx_buf_len,
DMA_BIDIRECTIONAL);
rx_desc->read.pkt_addr = cpu_to_le64(rx_buf->dma);
dma = xsk_buff_xdp_get_dma(rx_buf->xdp);
rx_desc->read.pkt_addr = cpu_to_le64(dma);
rx_desc->wb.status_error0 = 0;
rx_desc++;
@@ -652,32 +451,6 @@ ice_alloc_rx_bufs_zc(struct ice_ring *rx_ring, int count,
return ret;
}
/**
* ice_alloc_rx_bufs_fast_zc - allocate zero copy bufs in the hot path
* @rx_ring: Rx ring
* @count: number of bufs to allocate
*
* Returns false on success, true on failure.
*/
static bool ice_alloc_rx_bufs_fast_zc(struct ice_ring *rx_ring, u16 count)
{
return ice_alloc_rx_bufs_zc(rx_ring, count,
ice_alloc_buf_fast_zc);
}
/**
* ice_alloc_rx_bufs_slow_zc - allocate zero copy bufs in the slow path
* @rx_ring: Rx ring
* @count: number of bufs to allocate
*
* Returns false on success, true on failure.
*/
bool ice_alloc_rx_bufs_slow_zc(struct ice_ring *rx_ring, u16 count)
{
return ice_alloc_rx_bufs_zc(rx_ring, count,
ice_alloc_buf_slow_zc);
}
/**
* ice_bump_ntc - Bump the next_to_clean counter of an Rx ring
* @rx_ring: Rx ring
@@ -691,77 +464,22 @@ static void ice_bump_ntc(struct ice_ring *rx_ring)
prefetch(ICE_RX_DESC(rx_ring, ntc));
}
/**
* ice_get_rx_buf_zc - Fetch the current Rx buffer
* @rx_ring: Rx ring
* @size: size of a buffer
*
* This function returns the current, received Rx buffer and does
* DMA synchronization.
*
* Returns a pointer to the received Rx buffer.
*/
static struct ice_rx_buf *ice_get_rx_buf_zc(struct ice_ring *rx_ring, int size)
{
struct ice_rx_buf *rx_buf;
rx_buf = &rx_ring->rx_buf[rx_ring->next_to_clean];
dma_sync_single_range_for_cpu(rx_ring->dev, rx_buf->dma, 0,
size, DMA_BIDIRECTIONAL);
return rx_buf;
}
/**
* ice_reuse_rx_buf_zc - reuse an Rx buffer
* @rx_ring: Rx ring
* @old_buf: The buffer to recycle
*
* This function recycles a finished Rx buffer, and places it on the recycle
* queue (next_to_alloc).
*/
static void
ice_reuse_rx_buf_zc(struct ice_ring *rx_ring, struct ice_rx_buf *old_buf)
{
unsigned long mask = (unsigned long)rx_ring->xsk_umem->chunk_mask;
u64 hr = rx_ring->xsk_umem->headroom + XDP_PACKET_HEADROOM;
u16 nta = rx_ring->next_to_alloc;
struct ice_rx_buf *new_buf;
new_buf = &rx_ring->rx_buf[nta++];
rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
new_buf->dma = old_buf->dma & mask;
new_buf->dma += hr;
new_buf->addr = (void *)((unsigned long)old_buf->addr & mask);
new_buf->addr += hr;
new_buf->handle = old_buf->handle & mask;
new_buf->handle += rx_ring->xsk_umem->headroom;
old_buf->addr = NULL;
}
/**
* ice_construct_skb_zc - Create an sk_buff from zero-copy buffer
* @rx_ring: Rx ring
* @rx_buf: zero-copy Rx buffer
* @xdp: XDP buffer
*
* This function allocates a new skb from a zero-copy Rx buffer.
*
* Returns the skb on success, NULL on failure.
*/
static struct sk_buff *
ice_construct_skb_zc(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf,
struct xdp_buff *xdp)
ice_construct_skb_zc(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf)
{
unsigned int metasize = xdp->data - xdp->data_meta;
unsigned int datasize = xdp->data_end - xdp->data;
unsigned int datasize_hard = xdp->data_end -
xdp->data_hard_start;
unsigned int metasize = rx_buf->xdp->data - rx_buf->xdp->data_meta;
unsigned int datasize = rx_buf->xdp->data_end - rx_buf->xdp->data;
unsigned int datasize_hard = rx_buf->xdp->data_end -
rx_buf->xdp->data_hard_start;
struct sk_buff *skb;
skb = __napi_alloc_skb(&rx_ring->q_vector->napi, datasize_hard,
@@ -769,13 +487,13 @@ ice_construct_skb_zc(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf,
if (unlikely(!skb))
return NULL;
skb_reserve(skb, xdp->data - xdp->data_hard_start);
memcpy(__skb_put(skb, datasize), xdp->data, datasize);
skb_reserve(skb, rx_buf->xdp->data - rx_buf->xdp->data_hard_start);
memcpy(__skb_put(skb, datasize), rx_buf->xdp->data, datasize);
if (metasize)
skb_metadata_set(skb, metasize);
ice_reuse_rx_buf_zc(rx_ring, rx_buf);
xsk_buff_free(rx_buf->xdp);
rx_buf->xdp = NULL;
return skb;
}
@@ -802,7 +520,6 @@ ice_run_xdp_zc(struct ice_ring *rx_ring, struct xdp_buff *xdp)
}
act = bpf_prog_run_xdp(xdp_prog, xdp);
xdp->handle += xdp->data - xdp->data_hard_start;
switch (act) {
case XDP_PASS:
break;
@@ -840,13 +557,8 @@ int ice_clean_rx_irq_zc(struct ice_ring *rx_ring, int budget)
{
unsigned int total_rx_bytes = 0, total_rx_packets = 0;
u16 cleaned_count = ICE_DESC_UNUSED(rx_ring);
struct xdp_umem *umem = rx_ring->xsk_umem;
unsigned int xdp_xmit = 0;
bool failure = false;
struct xdp_buff xdp;
xdp.rxq = &rx_ring->xdp_rxq;
xdp.frame_sz = xsk_umem_xdp_frame_sz(umem);
while (likely(total_rx_packets < (unsigned int)budget)) {
union ice_32b_rx_flex_desc *rx_desc;
@@ -858,8 +570,8 @@ int ice_clean_rx_irq_zc(struct ice_ring *rx_ring, int budget)
u8 rx_ptype;
if (cleaned_count >= ICE_RX_BUF_WRITE) {
failure |= ice_alloc_rx_bufs_fast_zc(rx_ring,
cleaned_count);
failure |= ice_alloc_rx_bufs_zc(rx_ring,
cleaned_count);
cleaned_count = 0;
}
@@ -880,25 +592,19 @@ int ice_clean_rx_irq_zc(struct ice_ring *rx_ring, int budget)
if (!size)
break;
rx_buf = ice_get_rx_buf_zc(rx_ring, size);
if (!rx_buf->addr)
break;
xdp.data = rx_buf->addr;
xdp.data_meta = xdp.data;
xdp.data_hard_start = xdp.data - XDP_PACKET_HEADROOM;
xdp.data_end = xdp.data + size;
xdp.handle = rx_buf->handle;
rx_buf = &rx_ring->rx_buf[rx_ring->next_to_clean];
rx_buf->xdp->data_end = rx_buf->xdp->data + size;
xsk_buff_dma_sync_for_cpu(rx_buf->xdp);
xdp_res = ice_run_xdp_zc(rx_ring, &xdp);
xdp_res = ice_run_xdp_zc(rx_ring, rx_buf->xdp);
if (xdp_res) {
if (xdp_res & (ICE_XDP_TX | ICE_XDP_REDIR)) {
if (xdp_res & (ICE_XDP_TX | ICE_XDP_REDIR))
xdp_xmit |= xdp_res;
rx_buf->addr = NULL;
} else {
ice_reuse_rx_buf_zc(rx_ring, rx_buf);
}
else
xsk_buff_free(rx_buf->xdp);
rx_buf->xdp = NULL;
total_rx_bytes += size;
total_rx_packets++;
cleaned_count++;
@@ -908,7 +614,7 @@ int ice_clean_rx_irq_zc(struct ice_ring *rx_ring, int budget)
}
/* XDP_PASS path */
skb = ice_construct_skb_zc(rx_ring, rx_buf, &xdp);
skb = ice_construct_skb_zc(rx_ring, rx_buf);
if (!skb) {
rx_ring->rx_stats.alloc_buf_failed++;
break;
@@ -979,10 +685,9 @@ static bool ice_xmit_zc(struct ice_ring *xdp_ring, int budget)
if (!xsk_umem_consume_tx(xdp_ring->xsk_umem, &desc))
break;
dma = xdp_umem_get_dma(xdp_ring->xsk_umem, desc.addr);
dma_sync_single_for_device(xdp_ring->dev, dma, desc.len,
DMA_BIDIRECTIONAL);
dma = xsk_buff_raw_get_dma(xdp_ring->xsk_umem, desc.addr);
xsk_buff_raw_dma_sync_for_device(xdp_ring->xsk_umem, dma,
desc.len);
tx_buf->bytecount = desc.len;
@@ -1165,11 +870,10 @@ void ice_xsk_clean_rx_ring(struct ice_ring *rx_ring)
for (i = 0; i < rx_ring->count; i++) {
struct ice_rx_buf *rx_buf = &rx_ring->rx_buf[i];
if (!rx_buf->addr)
if (!rx_buf->xdp)
continue;
xsk_umem_fq_reuse(rx_ring->xsk_umem, rx_buf->handle);
rx_buf->addr = NULL;
rx_buf->xdp = NULL;
}
}

13
drivers/net/ethernet/intel/ice/ice_xsk.h
View File

@@ -10,11 +10,10 @@ struct ice_vsi;
#ifdef CONFIG_XDP_SOCKETS
int ice_xsk_umem_setup(struct ice_vsi *vsi, struct xdp_umem *umem, u16 qid);
void ice_zca_free(struct zero_copy_allocator *zca, unsigned long handle);
int ice_clean_rx_irq_zc(struct ice_ring *rx_ring, int budget);
bool ice_clean_tx_irq_zc(struct ice_ring *xdp_ring, int budget);
int ice_xsk_wakeup(struct net_device *netdev, u32 queue_id, u32 flags);
bool ice_alloc_rx_bufs_slow_zc(struct ice_ring *rx_ring, u16 count);
bool ice_alloc_rx_bufs_zc(struct ice_ring *rx_ring, u16 count);
bool ice_xsk_any_rx_ring_ena(struct ice_vsi *vsi);
void ice_xsk_clean_rx_ring(struct ice_ring *rx_ring);
void ice_xsk_clean_xdp_ring(struct ice_ring *xdp_ring);
@@ -27,12 +26,6 @@ ice_xsk_umem_setup(struct ice_vsi __always_unused *vsi,
return -EOPNOTSUPP;
}
static inline void
ice_zca_free(struct zero_copy_allocator __always_unused *zca,
unsigned long __always_unused handle)
{
}
static inline int
ice_clean_rx_irq_zc(struct ice_ring __always_unused *rx_ring,
int __always_unused budget)
@@ -48,8 +41,8 @@ ice_clean_tx_irq_zc(struct ice_ring __always_unused *xdp_ring,
}
static inline bool
ice_alloc_rx_bufs_slow_zc(struct ice_ring __always_unused *rx_ring,
u16 __always_unused count)
ice_alloc_rx_bufs_zc(struct ice_ring __always_unused *rx_ring,
u16 __always_unused count)
{
return false;
}

9
drivers/net/ethernet/intel/ixgbe/ixgbe.h
View File

@@ -224,17 +224,17 @@ struct ixgbe_tx_buffer {
};
struct ixgbe_rx_buffer {
struct sk_buff *skb;
dma_addr_t dma;
union {
struct {
struct sk_buff *skb;
dma_addr_t dma;
struct page *page;
__u32 page_offset;
__u16 pagecnt_bias;
};
struct {
void *addr;
u64 handle;
bool discard;
struct xdp_buff *xdp;
};
};
};
@@ -351,7 +351,6 @@ struct ixgbe_ring {
};
struct xdp_rxq_info xdp_rxq;
struct xdp_umem *xsk_umem;
struct zero_copy_allocator zca; /* ZC allocator anchor */
u16 ring_idx; /* {rx,tx,xdp}_ring back reference idx */
u16 rx_buf_len;
} ____cacheline_internodealigned_in_smp;

15
drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
View File

@@ -35,7 +35,7 @@
#include <net/tc_act/tc_mirred.h>
#include <net/vxlan.h>
#include <net/mpls.h>
#include <net/xdp_sock.h>
#include <net/xdp_sock_drv.h>
#include <net/xfrm.h>
#include "ixgbe.h"
@@ -3745,8 +3745,7 @@ static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter,
/* configure the packet buffer length */
if (rx_ring->xsk_umem) {
u32 xsk_buf_len = rx_ring->xsk_umem->chunk_size_nohr -
XDP_PACKET_HEADROOM;
u32 xsk_buf_len = xsk_umem_get_rx_frame_size(rx_ring->xsk_umem);
/* If the MAC support setting RXDCTL.RLPML, the
* SRRCTL[n].BSIZEPKT is set to PAGE_SIZE and
@@ -4093,11 +4092,10 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter,
xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq);
ring->xsk_umem = ixgbe_xsk_umem(adapter, ring);
if (ring->xsk_umem) {
ring->zca.free = ixgbe_zca_free;
WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
MEM_TYPE_ZERO_COPY,
&ring->zca));
MEM_TYPE_XSK_BUFF_POOL,
NULL));
xsk_buff_set_rxq_info(ring->xsk_umem, &ring->xdp_rxq);
} else {
WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
MEM_TYPE_PAGE_SHARED, NULL));
@@ -4153,8 +4151,7 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter,
}
if (ring->xsk_umem && hw->mac.type != ixgbe_mac_82599EB) {
u32 xsk_buf_len = ring->xsk_umem->chunk_size_nohr -
XDP_PACKET_HEADROOM;
u32 xsk_buf_len = xsk_umem_get_rx_frame_size(ring->xsk_umem);
rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK |
IXGBE_RXDCTL_RLPML_EN);

2
drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h
View File

@@ -35,7 +35,7 @@ int ixgbe_xsk_umem_setup(struct ixgbe_adapter *adapter, struct xdp_umem *umem,
void ixgbe_zca_free(struct zero_copy_allocator *alloc, unsigned long handle);
void ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 cleaned_count);
bool ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 cleaned_count);
int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector,
struct ixgbe_ring *rx_ring,
const int budget);

309
drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c
View File

@@ -2,7 +2,7 @@
/* Copyright(c) 2018 Intel Corporation. */
#include <linux/bpf_trace.h>
#include <net/xdp_sock.h>
#include <net/xdp_sock_drv.h>
#include <net/xdp.h>
#include "ixgbe.h"
@@ -20,54 +20,11 @@ struct xdp_umem *ixgbe_xsk_umem(struct ixgbe_adapter *adapter,
return xdp_get_umem_from_qid(adapter->netdev, qid);
}
static int ixgbe_xsk_umem_dma_map(struct ixgbe_adapter *adapter,
struct xdp_umem *umem)
{
struct device *dev = &adapter->pdev->dev;
unsigned int i, j;
dma_addr_t dma;
for (i = 0; i < umem->npgs; i++) {
dma = dma_map_page_attrs(dev, umem->pgs[i], 0, PAGE_SIZE,
DMA_BIDIRECTIONAL, IXGBE_RX_DMA_ATTR);
if (dma_mapping_error(dev, dma))
goto out_unmap;
umem->pages[i].dma = dma;
}
return 0;
out_unmap:
for (j = 0; j < i; j++) {
dma_unmap_page_attrs(dev, umem->pages[i].dma, PAGE_SIZE,
DMA_BIDIRECTIONAL, IXGBE_RX_DMA_ATTR);
umem->pages[i].dma = 0;
}
return -1;
}
static void ixgbe_xsk_umem_dma_unmap(struct ixgbe_adapter *adapter,
struct xdp_umem *umem)
{
struct device *dev = &adapter->pdev->dev;
unsigned int i;
for (i = 0; i < umem->npgs; i++) {
dma_unmap_page_attrs(dev, umem->pages[i].dma, PAGE_SIZE,
DMA_BIDIRECTIONAL, IXGBE_RX_DMA_ATTR);
umem->pages[i].dma = 0;
}
}
static int ixgbe_xsk_umem_enable(struct ixgbe_adapter *adapter,
struct xdp_umem *umem,
u16 qid)
{
struct net_device *netdev = adapter->netdev;
struct xdp_umem_fq_reuse *reuseq;
bool if_running;
int err;
@@ -78,13 +35,7 @@ static int ixgbe_xsk_umem_enable(struct ixgbe_adapter *adapter,
qid >= netdev->real_num_tx_queues)
return -EINVAL;
reuseq = xsk_reuseq_prepare(adapter->rx_ring[0]->count);
if (!reuseq)
return -ENOMEM;
xsk_reuseq_free(xsk_reuseq_swap(umem, reuseq));
err = ixgbe_xsk_umem_dma_map(adapter, umem);
err = xsk_buff_dma_map(umem, &adapter->pdev->dev, IXGBE_RX_DMA_ATTR);
if (err)
return err;
@@ -124,7 +75,7 @@ static int ixgbe_xsk_umem_disable(struct ixgbe_adapter *adapter, u16 qid)
ixgbe_txrx_ring_disable(adapter, qid);
clear_bit(qid, adapter->af_xdp_zc_qps);
ixgbe_xsk_umem_dma_unmap(adapter, umem);
xsk_buff_dma_unmap(umem, IXGBE_RX_DMA_ATTR);
if (if_running)
ixgbe_txrx_ring_enable(adapter, qid);
@@ -143,19 +94,14 @@ static int ixgbe_run_xdp_zc(struct ixgbe_adapter *adapter,
struct ixgbe_ring *rx_ring,
struct xdp_buff *xdp)
{
struct xdp_umem *umem = rx_ring->xsk_umem;
int err, result = IXGBE_XDP_PASS;
struct bpf_prog *xdp_prog;
struct xdp_frame *xdpf;
u64 offset;
u32 act;
rcu_read_lock();
xdp_prog = READ_ONCE(rx_ring->xdp_prog);
act = bpf_prog_run_xdp(xdp_prog, xdp);
offset = xdp->data - xdp->data_hard_start;
xdp->handle = xsk_umem_adjust_offset(umem, xdp->handle, offset);
switch (act) {
case XDP_PASS:
@@ -186,140 +132,16 @@ static int ixgbe_run_xdp_zc(struct ixgbe_adapter *adapter,
return result;
}
static struct
ixgbe_rx_buffer *ixgbe_get_rx_buffer_zc(struct ixgbe_ring *rx_ring,
unsigned int size)
{
struct ixgbe_rx_buffer *bi;
bi = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
/* we are reusing so sync this buffer for CPU use */
dma_sync_single_range_for_cpu(rx_ring->dev,
bi->dma, 0,
size,
DMA_BIDIRECTIONAL);
return bi;
}
static void ixgbe_reuse_rx_buffer_zc(struct ixgbe_ring *rx_ring,
struct ixgbe_rx_buffer *obi)
{
u16 nta = rx_ring->next_to_alloc;
struct ixgbe_rx_buffer *nbi;
nbi = &rx_ring->rx_buffer_info[rx_ring->next_to_alloc];
/* update, and store next to alloc */
nta++;
rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
/* transfer page from old buffer to new buffer */
nbi->dma = obi->dma;
nbi->addr = obi->addr;
nbi->handle = obi->handle;
obi->addr = NULL;
obi->skb = NULL;
}
void ixgbe_zca_free(struct zero_copy_allocator *alloc, unsigned long handle)
{
struct ixgbe_rx_buffer *bi;
struct ixgbe_ring *rx_ring;
u64 hr, mask;
u16 nta;
rx_ring = container_of(alloc, struct ixgbe_ring, zca);
hr = rx_ring->xsk_umem->headroom + XDP_PACKET_HEADROOM;
mask = rx_ring->xsk_umem->chunk_mask;
nta = rx_ring->next_to_alloc;
bi = rx_ring->rx_buffer_info;
nta++;
rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
handle &= mask;
bi->dma = xdp_umem_get_dma(rx_ring->xsk_umem, handle);
bi->dma += hr;
bi->addr = xdp_umem_get_data(rx_ring->xsk_umem, handle);
bi->addr += hr;
bi->handle = xsk_umem_adjust_offset(rx_ring->xsk_umem, (u64)handle,
rx_ring->xsk_umem->headroom);
}
static bool ixgbe_alloc_buffer_zc(struct ixgbe_ring *rx_ring,
struct ixgbe_rx_buffer *bi)
{
struct xdp_umem *umem = rx_ring->xsk_umem;
void *addr = bi->addr;
u64 handle, hr;
if (addr)
return true;
if (!xsk_umem_peek_addr(umem, &handle)) {
rx_ring->rx_stats.alloc_rx_page_failed++;
return false;
}
hr = umem->headroom + XDP_PACKET_HEADROOM;
bi->dma = xdp_umem_get_dma(umem, handle);
bi->dma += hr;
bi->addr = xdp_umem_get_data(umem, handle);
bi->addr += hr;
bi->handle = xsk_umem_adjust_offset(umem, handle, umem->headroom);
xsk_umem_release_addr(umem);
return true;
}
static bool ixgbe_alloc_buffer_slow_zc(struct ixgbe_ring *rx_ring,
struct ixgbe_rx_buffer *bi)
{
struct xdp_umem *umem = rx_ring->xsk_umem;
u64 handle, hr;
if (!xsk_umem_peek_addr_rq(umem, &handle)) {
rx_ring->rx_stats.alloc_rx_page_failed++;
return false;
}
handle &= rx_ring->xsk_umem->chunk_mask;
hr = umem->headroom + XDP_PACKET_HEADROOM;
bi->dma = xdp_umem_get_dma(umem, handle);
bi->dma += hr;
bi->addr = xdp_umem_get_data(umem, handle);
bi->addr += hr;
bi->handle = xsk_umem_adjust_offset(umem, handle, umem->headroom);
xsk_umem_release_addr_rq(umem);
return true;
}
static __always_inline bool
__ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 cleaned_count,
bool alloc(struct ixgbe_ring *rx_ring,
struct ixgbe_rx_buffer *bi))
bool ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 count)
{
union ixgbe_adv_rx_desc *rx_desc;
struct ixgbe_rx_buffer *bi;
u16 i = rx_ring->next_to_use;
dma_addr_t dma;
bool ok = true;
/* nothing to do */
if (!cleaned_count)
if (!count)
return true;
rx_desc = IXGBE_RX_DESC(rx_ring, i);
@@ -327,21 +149,18 @@ __ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 cleaned_count,
i -= rx_ring->count;
do {
if (!alloc(rx_ring, bi)) {
bi->xdp = xsk_buff_alloc(rx_ring->xsk_umem);
if (!bi->xdp) {
ok = false;
break;
}
/* sync the buffer for use by the device */
dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
bi->page_offset,
rx_ring->rx_buf_len,
DMA_BIDIRECTIONAL);
dma = xsk_buff_xdp_get_dma(bi->xdp);
/* Refresh the desc even if buffer_addrs didn't change
* because each write-back erases this info.
*/
rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
rx_desc->read.pkt_addr = cpu_to_le64(dma);
rx_desc++;
bi++;
@@ -355,17 +174,14 @@ __ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 cleaned_count,
/* clear the length for the next_to_use descriptor */
rx_desc->wb.upper.length = 0;
cleaned_count--;
} while (cleaned_count);
count--;
} while (count);
i += rx_ring->count;
if (rx_ring->next_to_use != i) {
rx_ring->next_to_use = i;
/* update next to alloc since we have filled the ring */
rx_ring->next_to_alloc = i;
/* Force memory writes to complete before letting h/w
* know there are new descriptors to fetch. (Only
* applicable for weak-ordered memory model archs,
@@ -378,40 +194,27 @@ __ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 cleaned_count,
return ok;
}
void ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 count)
{
__ixgbe_alloc_rx_buffers_zc(rx_ring, count,
ixgbe_alloc_buffer_slow_zc);
}
static bool ixgbe_alloc_rx_buffers_fast_zc(struct ixgbe_ring *rx_ring,
u16 count)
{
return __ixgbe_alloc_rx_buffers_zc(rx_ring, count,
ixgbe_alloc_buffer_zc);
}
static struct sk_buff *ixgbe_construct_skb_zc(struct ixgbe_ring *rx_ring,
struct ixgbe_rx_buffer *bi,
struct xdp_buff *xdp)
struct ixgbe_rx_buffer *bi)
{
unsigned int metasize = xdp->data - xdp->data_meta;
unsigned int datasize = xdp->data_end - xdp->data;
unsigned int metasize = bi->xdp->data - bi->xdp->data_meta;
unsigned int datasize = bi->xdp->data_end - bi->xdp->data;
struct sk_buff *skb;
/* allocate a skb to store the frags */
skb = __napi_alloc_skb(&rx_ring->q_vector->napi,
xdp->data_end - xdp->data_hard_start,
bi->xdp->data_end - bi->xdp->data_hard_start,
GFP_ATOMIC | __GFP_NOWARN);
if (unlikely(!skb))
return NULL;
skb_reserve(skb, xdp->data - xdp->data_hard_start);
memcpy(__skb_put(skb, datasize), xdp->data, datasize);
skb_reserve(skb, bi->xdp->data - bi->xdp->data_hard_start);
memcpy(__skb_put(skb, datasize), bi->xdp->data, datasize);
if (metasize)
skb_metadata_set(skb, metasize);
ixgbe_reuse_rx_buffer_zc(rx_ring, bi);
xsk_buff_free(bi->xdp);
bi->xdp = NULL;
return skb;
}
@@ -431,14 +234,9 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector,
unsigned int total_rx_bytes = 0, total_rx_packets = 0;
struct ixgbe_adapter *adapter = q_vector->adapter;
u16 cleaned_count = ixgbe_desc_unused(rx_ring);
struct xdp_umem *umem = rx_ring->xsk_umem;
unsigned int xdp_res, xdp_xmit = 0;
bool failure = false;
struct sk_buff *skb;
struct xdp_buff xdp;
xdp.rxq = &rx_ring->xdp_rxq;
xdp.frame_sz = xsk_umem_xdp_frame_sz(umem);
while (likely(total_rx_packets < budget)) {
union ixgbe_adv_rx_desc *rx_desc;
@@ -448,8 +246,8 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector,
/* return some buffers to hardware, one at a time is too slow */
if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) {
failure = failure ||
!ixgbe_alloc_rx_buffers_fast_zc(rx_ring,
cleaned_count);
!ixgbe_alloc_rx_buffers_zc(rx_ring,
cleaned_count);
cleaned_count = 0;
}
@@ -464,42 +262,40 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector,
*/
dma_rmb();
bi = ixgbe_get_rx_buffer_zc(rx_ring, size);
bi = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
if (unlikely(!ixgbe_test_staterr(rx_desc,
IXGBE_RXD_STAT_EOP))) {
struct ixgbe_rx_buffer *next_bi;
ixgbe_reuse_rx_buffer_zc(rx_ring, bi);
xsk_buff_free(bi->xdp);
bi->xdp = NULL;
ixgbe_inc_ntc(rx_ring);
next_bi =
&rx_ring->rx_buffer_info[rx_ring->next_to_clean];
next_bi->skb = ERR_PTR(-EINVAL);
next_bi->discard = true;
continue;
}
if (unlikely(bi->skb)) {
ixgbe_reuse_rx_buffer_zc(rx_ring, bi);
if (unlikely(bi->discard)) {
xsk_buff_free(bi->xdp);
bi->xdp = NULL;
bi->discard = false;
ixgbe_inc_ntc(rx_ring);
continue;
}
xdp.data = bi->addr;
xdp.data_meta = xdp.data;
xdp.data_hard_start = xdp.data - XDP_PACKET_HEADROOM;
xdp.data_end = xdp.data + size;
xdp.handle = bi->handle;
xdp_res = ixgbe_run_xdp_zc(adapter, rx_ring, &xdp);
bi->xdp->data_end = bi->xdp->data + size;
xsk_buff_dma_sync_for_cpu(bi->xdp);
xdp_res = ixgbe_run_xdp_zc(adapter, rx_ring, bi->xdp);
if (xdp_res) {
if (xdp_res & (IXGBE_XDP_TX | IXGBE_XDP_REDIR)) {
if (xdp_res & (IXGBE_XDP_TX | IXGBE_XDP_REDIR))
xdp_xmit |= xdp_res;
bi->addr = NULL;
bi->skb = NULL;
} else {
ixgbe_reuse_rx_buffer_zc(rx_ring, bi);
}
else
xsk_buff_free(bi->xdp);
bi->xdp = NULL;
total_rx_packets++;
total_rx_bytes += size;
@@ -509,7 +305,7 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector,
}
/* XDP_PASS path */
skb = ixgbe_construct_skb_zc(rx_ring, bi, &xdp);
skb = ixgbe_construct_skb_zc(rx_ring, bi);
if (!skb) {
rx_ring->rx_stats.alloc_rx_buff_failed++;
break;
@@ -561,17 +357,17 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector,
void ixgbe_xsk_clean_rx_ring(struct ixgbe_ring *rx_ring)
{
u16 i = rx_ring->next_to_clean;
struct ixgbe_rx_buffer *bi = &rx_ring->rx_buffer_info[i];
struct ixgbe_rx_buffer *bi;
u16 i;
while (i != rx_ring->next_to_alloc) {
xsk_umem_fq_reuse(rx_ring->xsk_umem, bi->handle);
i++;
bi++;
if (i == rx_ring->count) {
i = 0;
bi = rx_ring->rx_buffer_info;
}
for (i = 0; i < rx_ring->count; i++) {
bi = &rx_ring->rx_buffer_info[i];
if (!bi->xdp)
continue;
xsk_buff_free(bi->xdp);
bi->xdp = NULL;
}
}
@@ -594,10 +390,9 @@ static bool ixgbe_xmit_zc(struct ixgbe_ring *xdp_ring, unsigned int budget)
if (!xsk_umem_consume_tx(xdp_ring->xsk_umem, &desc))
break;
dma = xdp_umem_get_dma(xdp_ring->xsk_umem, desc.addr);
dma_sync_single_for_device(xdp_ring->dev, dma, desc.len,
DMA_BIDIRECTIONAL);
dma = xsk_buff_raw_get_dma(xdp_ring->xsk_umem, desc.addr);
xsk_buff_raw_dma_sync_for_device(xdp_ring->xsk_umem, dma,
desc.len);
tx_bi = &xdp_ring->tx_buffer_info[xdp_ring->next_to_use];
tx_bi->bytecount = desc.len;