123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109 |
- // SPDX-License-Identifier: GPL-2.0
- /*
- * Copyright (C) 2015-2019 Jason A. Donenfeld <[email protected]>. All Rights Reserved.
- */
- #include "queueing.h"
- #include <linux/skb_array.h>
- struct multicore_worker __percpu *
- wg_packet_percpu_multicore_worker_alloc(work_func_t function, void *ptr)
- {
- int cpu;
- struct multicore_worker __percpu *worker = alloc_percpu(struct multicore_worker);
- if (!worker)
- return NULL;
- for_each_possible_cpu(cpu) {
- per_cpu_ptr(worker, cpu)->ptr = ptr;
- INIT_WORK(&per_cpu_ptr(worker, cpu)->work, function);
- }
- return worker;
- }
- int wg_packet_queue_init(struct crypt_queue *queue, work_func_t function,
- unsigned int len)
- {
- int ret;
- memset(queue, 0, sizeof(*queue));
- queue->last_cpu = -1;
- ret = ptr_ring_init(&queue->ring, len, GFP_KERNEL);
- if (ret)
- return ret;
- queue->worker = wg_packet_percpu_multicore_worker_alloc(function, queue);
- if (!queue->worker) {
- ptr_ring_cleanup(&queue->ring, NULL);
- return -ENOMEM;
- }
- return 0;
- }
- void wg_packet_queue_free(struct crypt_queue *queue, bool purge)
- {
- free_percpu(queue->worker);
- WARN_ON(!purge && !__ptr_ring_empty(&queue->ring));
- ptr_ring_cleanup(&queue->ring, purge ? __skb_array_destroy_skb : NULL);
- }
- #define NEXT(skb) ((skb)->prev)
- #define STUB(queue) ((struct sk_buff *)&queue->empty)
- void wg_prev_queue_init(struct prev_queue *queue)
- {
- NEXT(STUB(queue)) = NULL;
- queue->head = queue->tail = STUB(queue);
- queue->peeked = NULL;
- atomic_set(&queue->count, 0);
- BUILD_BUG_ON(
- offsetof(struct sk_buff, next) != offsetof(struct prev_queue, empty.next) -
- offsetof(struct prev_queue, empty) ||
- offsetof(struct sk_buff, prev) != offsetof(struct prev_queue, empty.prev) -
- offsetof(struct prev_queue, empty));
- }
- static void __wg_prev_queue_enqueue(struct prev_queue *queue, struct sk_buff *skb)
- {
- WRITE_ONCE(NEXT(skb), NULL);
- WRITE_ONCE(NEXT(xchg_release(&queue->head, skb)), skb);
- }
- bool wg_prev_queue_enqueue(struct prev_queue *queue, struct sk_buff *skb)
- {
- if (!atomic_add_unless(&queue->count, 1, MAX_QUEUED_PACKETS))
- return false;
- __wg_prev_queue_enqueue(queue, skb);
- return true;
- }
- struct sk_buff *wg_prev_queue_dequeue(struct prev_queue *queue)
- {
- struct sk_buff *tail = queue->tail, *next = smp_load_acquire(&NEXT(tail));
- if (tail == STUB(queue)) {
- if (!next)
- return NULL;
- queue->tail = next;
- tail = next;
- next = smp_load_acquire(&NEXT(next));
- }
- if (next) {
- queue->tail = next;
- atomic_dec(&queue->count);
- return tail;
- }
- if (tail != READ_ONCE(queue->head))
- return NULL;
- __wg_prev_queue_enqueue(queue, STUB(queue));
- next = smp_load_acquire(&NEXT(tail));
- if (next) {
- queue->tail = next;
- atomic_dec(&queue->count);
- return tail;
- }
- return NULL;
- }
- #undef NEXT
- #undef STUB
|