rmnet_core: Add Low Latency framework

Packets are now sent over a dedicated MHI channel when indicated by the
DFC driver.

New dedicated channel is controlled by rmnet driver. Buffers are allocated
and supplied to it as needed from a recyclable pool for RX on the channel,
and packets will be sent to it and freed manually once the channel
indicates that they have been sent.

Low latency packets can be aggregated like standard QMAP packets, but have
their own aggregation state to prevent mixing default and low latency
flows, and to allow each type of flow to use their own send functions
(i.e. dev_queue_xmit() versus rmnet_ll_send_skb()).

Low latency packets also have their own load-balancing scheme, and do not
need to use the SHS module for balancing. To facilitate this, we mark the
low latency packets with a non-zero priority value upon receipt from the
MHI chainnel and avoid sending any such marked packets to the SHS ingress
hook.

DFC has been updated with a new netlink message type to handle swapping a
list of bearers from one channel to another. The actual swap is performed
asynchronously, and separate netlink ACKs will be sent to the userspace
socket when the switch has been completed.

Change-Id: I93861d4b004f399ba203d76a71b2f01fa5c0d5d2
Signed-off-by: Sean Tranchetti <[email protected]>

core/Kbuild

@@ -8,7 +8,26 @@ endif
 
 obj-m += rmnet_core.o
 obj-m += rmnet_ctl.o
-rmnet_core-y := rmnet_config.o rmnet_handlers.o rmnet_descriptor.o \
-	rmnet_genl.o rmnet_map_command.o rmnet_map_data.o rmnet_vnd.o\
-	qmi_rmnet.o wda_qmi.o dfc_qmi.o dfc_qmap.o
-rmnet_ctl-y := rmnet_ctl_client.o rmnet_ctl_ipa.o
+
+#core sources
+rmnet_core-y := \
+	rmnet_config.o \
+	rmnet_handlers.o \
+	rmnet_descriptor.o \
+	rmnet_genl.o \
+	rmnet_map_command.o \
+	rmnet_map_data.o \
+	rmnet_vnd.o
+rmnet_core-$(CONFIG_MHI_BUS) += \
+	rmnet_ll.o
+
+#DFC sources
+rmnet_core-y += \
+	qmi_rmnet.o \
+	wda_qmi.o \
+	dfc_qmi.o \
+	dfc_qmap.o
+
+rmnet_ctl-y := \
+	rmnet_ctl_client.o \
+	rmnet_ctl_ipa.o

core/qmi_rmnet.c

@@ -32,6 +32,7 @@
 #define NLMSG_CLIENT_DELETE 5
 #define NLMSG_SCALE_FACTOR 6
 #define NLMSG_WQ_FREQUENCY 7
+#define NLMSG_CHANNEL_SWITCH 8
 
 #define FLAG_DFC_MASK 0x000F
 #define FLAG_POWERSAVE_MASK 0x0010
@@ -704,7 +705,8 @@ qmi_rmnet_delete_client(void *port, struct qmi_info *qmi, struct tcmsg *tcm)
 	__qmi_rmnet_delete_client(port, qmi, idx);
 }
 
-void qmi_rmnet_change_link(struct net_device *dev, void *port, void *tcm_pt)
+void qmi_rmnet_change_link(struct net_device *dev, void *port, void *tcm_pt,
+			   int attr_len)
 {
 	struct qmi_info *qmi = (struct qmi_info *)rmnet_get_qmi_pt(port);
 	struct tcmsg *tcm = (struct tcmsg *)tcm_pt;
@@ -741,7 +743,11 @@ void qmi_rmnet_change_link(struct net_device *dev, void *port, void *tcm_pt)
 				rmnet_reset_qmi_pt(port);
 				kfree(qmi);
 			}
-		} else if (tcm->tcm_ifindex & FLAG_POWERSAVE_MASK) {
+
+			return;
+		}
+
+		if (tcm->tcm_ifindex & FLAG_POWERSAVE_MASK) {
 			qmi_rmnet_work_init(port);
 			rmnet_set_powersave_format(port);
 		}
@@ -768,6 +774,11 @@ void qmi_rmnet_change_link(struct net_device *dev, void *port, void *tcm_pt)
 	case NLMSG_WQ_FREQUENCY:
 		rmnet_wq_frequency = tcm->tcm_ifindex;
 		break;
+	case NLMSG_CHANNEL_SWITCH:
+		if (!qmi || !DFC_SUPPORTED_MODE(dfc_mode) ||
+		    !qmi_rmnet_has_dfc_client(qmi))
+			return;
+		break;
 	default:
 		pr_debug("%s(): No handler\n", __func__);
 		break;
@@ -869,6 +880,35 @@ bool qmi_rmnet_all_flows_enabled(struct net_device *dev)
 EXPORT_SYMBOL(qmi_rmnet_all_flows_enabled);
 
 #ifdef CONFIG_QTI_QMI_DFC
+bool qmi_rmnet_flow_is_low_latency(struct net_device *dev, int ip_type,
+				   u32 mark)
+{
+	struct qos_info *qos = rmnet_get_qos_pt(dev);
+	struct rmnet_bearer_map *bearer;
+	struct rmnet_flow_map *itm;
+	bool ret = false;
+
+	if (!qos)
+		goto out;
+
+	spin_lock_bh(&qos->qos_lock);
+	itm = qmi_rmnet_get_flow_map(qos, mark, ip_type);
+	if (!itm)
+		goto out_unlock;
+
+	bearer = itm->bearer;
+	if (!bearer)
+		goto out_unlock;
+
+	ret = bearer->is_low_latency;
+
+out_unlock:
+	spin_unlock_bh(&qos->qos_lock);
+out:
+	return ret;
+}
+EXPORT_SYMBOL(qmi_rmnet_flow_is_low_latency);
+
 void qmi_rmnet_burst_fc_check(struct net_device *dev,
 			      int ip_type, u32 mark, unsigned int len)
 {

core/qmi_rmnet.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018-2020, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2018-2021, The Linux Foundation. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 and
@@ -29,7 +29,8 @@ struct qmi_rmnet_ps_ind {
 
 #ifdef CONFIG_QTI_QMI_RMNET
 void qmi_rmnet_qmi_exit(void *qmi_pt, void *port);
-void qmi_rmnet_change_link(struct net_device *dev, void *port, void *tcm_pt);
+void qmi_rmnet_change_link(struct net_device *dev, void *port, void *tcm_pt,
+			   int attr_len);
 void qmi_rmnet_enable_all_flows(struct net_device *dev);
 bool qmi_rmnet_all_flows_enabled(struct net_device *dev);
 #else
@@ -38,7 +39,8 @@ static inline void qmi_rmnet_qmi_exit(void *qmi_pt, void *port)
 }
 
 static inline void
-qmi_rmnet_change_link(struct net_device *dev, void *port, void *tcm_pt)
+qmi_rmnet_change_link(struct net_device *dev, void *port, void *tcm_pt,
+		      int attr_len)
 {
 }
 
@@ -59,6 +61,8 @@ void *qmi_rmnet_qos_init(struct net_device *real_dev,
 			 struct net_device *vnd_dev, u8 mux_id);
 void qmi_rmnet_qos_exit_pre(void *qos);
 void qmi_rmnet_qos_exit_post(void);
+bool qmi_rmnet_flow_is_low_latency(struct net_device *dev, int ip_type,
+				   u32 mark);
 void qmi_rmnet_burst_fc_check(struct net_device *dev,
 			      int ip_type, u32 mark, unsigned int len);
 int qmi_rmnet_get_queue(struct net_device *dev, struct sk_buff *skb);
@@ -78,6 +82,12 @@ static inline void qmi_rmnet_qos_exit_post(void)
 {
 }
 
+static inline bool qmi_rmnet_flow_is_low_latency(struct net_device *dev,
+						 int ip_type, u32 mark)
+{
+	return false;
+}
+
 static inline void
 qmi_rmnet_burst_fc_check(struct net_device *dev,
 			 int ip_type, u32 mark, unsigned int len)

core/qmi_rmnet_i.h

@@ -51,6 +51,7 @@ struct rmnet_bearer_map {
 	u16 last_seq;
 	u32 bytes_in_flight;
 	u32 last_adjusted_grant;
+	bool is_low_latency;
 	bool tcp_bidir;
 	bool rat_switch;
 	bool tx_off;

core/rmnet_config.c

@@ -23,6 +23,7 @@
 #include "rmnet_private.h"
 #include "rmnet_map.h"
 #include "rmnet_descriptor.h"
+#include "rmnet_ll.h"
 #include "rmnet_genl.h"
 #include "rmnet_qmi.h"
 #include "qmi_rmnet.h"
@@ -417,10 +418,11 @@ static int rmnet_changelink(struct net_device *dev, struct nlattr *tb[],
 	}
 
 	if (data[IFLA_RMNET_DFC_QOS]) {
+		struct nlattr *qos = data[IFLA_RMNET_DFC_QOS];
 		struct tcmsg *tcm;
 
-		tcm = nla_data(data[IFLA_RMNET_DFC_QOS]);
-		qmi_rmnet_change_link(dev, port, tcm);
+		tcm = nla_data(qos);
+		qmi_rmnet_change_link(dev, port, tcm, nla_len(qos));
 	}
 
 	if (data[IFLA_RMNET_UL_AGG_PARAMS]) {
@@ -770,6 +772,13 @@ static int __init rmnet_init(void)
 		return rc;
 	}
 
+	rc = rmnet_ll_init();
+	if (rc != 0) {
+		unregister_netdevice_notifier(&rmnet_dev_notifier);
+		rtnl_link_unregister(&rmnet_link_ops);
+		return rc;
+	}
+
 	rmnet_core_genl_init();
 
 	try_module_get(THIS_MODULE);
@@ -780,6 +789,7 @@ static void __exit rmnet_exit(void)
 {
 	unregister_netdevice_notifier(&rmnet_dev_notifier);
 	rtnl_link_unregister(&rmnet_link_ops);
+	rmnet_ll_exit();
 	rmnet_core_genl_deinit();
 
 	module_put(THIS_MODULE);

core/rmnet_config.h

@@ -69,6 +69,26 @@ struct rmnet_egress_agg_params {
 	u32 agg_time;
 };
 
+enum {
+	RMNET_DEFAULT_AGG_STATE,
+	RMNET_LL_AGG_STATE,
+	RMNET_MAX_AGG_STATE,
+};
+
+struct rmnet_aggregation_state {
+	struct timespec64 agg_time;
+	struct timespec64 agg_last;
+	struct hrtimer hrtimer;
+	struct work_struct agg_wq;
+	/* Pointer back to the main lock for use in the workqueue */
+	spinlock_t *agg_lock;
+	struct sk_buff *agg_skb;
+	int (*send_agg_skb)(struct sk_buff *skb);
+	int agg_state;
+	u8 agg_count;
+};
+
+
 struct rmnet_agg_page {
 	struct list_head list;
 	struct page *page;
@@ -92,13 +112,7 @@ struct rmnet_port {
 	/* Protect aggregation related elements */
 	spinlock_t agg_lock;
 
-	struct sk_buff *agg_skb;
-	int agg_state;
-	u8 agg_count;
-	struct timespec64 agg_time;
-	struct timespec64 agg_last;
-	struct hrtimer hrtimer;
-	struct work_struct agg_wq;
+	struct rmnet_aggregation_state agg_state[RMNET_MAX_AGG_STATE];
 	u8 agg_size_order;
 	struct list_head agg_list;
 	struct rmnet_agg_page *agg_head;

core/rmnet_descriptor.c

@@ -565,7 +565,7 @@ EXPORT_SYMBOL(rmnet_frag_flow_command);
 static int rmnet_frag_deaggregate_one(struct sk_buff *skb,
 				      struct rmnet_port *port,
 				      struct list_head *list,
-				      u32 start)
+				      u32 start, u32 priority)
 {
 	struct skb_shared_info *shinfo = skb_shinfo(skb);
 	struct rmnet_frag_descriptor *frag_desc;
@@ -619,6 +619,7 @@ static int rmnet_frag_deaggregate_one(struct sk_buff *skb,
 	if (!frag_desc)
 		return -1;
 
+	frag_desc->priority = priority;
 	pkt_len += sizeof(*maph);
 	if (port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4) {
 		pkt_len += sizeof(struct rmnet_map_dl_csum_trailer);
@@ -697,13 +698,14 @@ static int rmnet_frag_deaggregate_one(struct sk_buff *skb,
 }
 
 void rmnet_frag_deaggregate(struct sk_buff *skb, struct rmnet_port *port,
-			    struct list_head *list)
+			    struct list_head *list, u32 priority)
 {
 	u32 start = 0;
 	int rc;
 
 	while (start < skb->len) {
-		rc = rmnet_frag_deaggregate_one(skb, port, list, start);
+		rc = rmnet_frag_deaggregate_one(skb, port, list, start,
+						priority);
 		if (rc < 0)
 			return;
 
@@ -967,6 +969,8 @@ skip_frags:
 	if (frag_desc->gso_segs > 1)
 		rmnet_frag_gso_stamp(head_skb, frag_desc);
 
+	/* Propagate original priority value */
+	head_skb->priority = frag_desc->priority;
 	return head_skb;
 }
 
@@ -1608,7 +1612,7 @@ void rmnet_frag_ingress_handler(struct sk_buff *skb,
 	while (skb) {
 		struct sk_buff *skb_frag;
 
-		rmnet_frag_deaggregate(skb, port, &desc_list);
+		rmnet_frag_deaggregate(skb, port, &desc_list, skb->priority);
 		if (!list_empty(&desc_list)) {
 			struct rmnet_frag_descriptor *frag_desc, *tmp;
 

core/rmnet_descriptor.h

@@ -38,6 +38,7 @@ struct rmnet_frag_descriptor {
 	struct net_device *dev;
 	u32 len;
 	u32 hash;
+	u32 priority;
 	__be32 tcp_seq;
 	__be16 ip_id;
 	__be16 tcp_flags;
@@ -85,7 +86,7 @@ int rmnet_frag_flow_command(struct rmnet_frag_descriptor *frag_desc,
 
 /* Ingress data handlers */
 void rmnet_frag_deaggregate(struct sk_buff *skb, struct rmnet_port *port,
-			    struct list_head *list);
+			    struct list_head *list, u32 priority);
 void rmnet_frag_deliver(struct rmnet_frag_descriptor *frag_desc,
 			struct rmnet_port *port);
 int rmnet_frag_process_next_hdr_packet(struct rmnet_frag_descriptor *frag_desc,

core/rmnet_handlers.c

@@ -26,6 +26,7 @@
 #include "rmnet_map.h"
 #include "rmnet_handlers.h"
 #include "rmnet_descriptor.h"
+#include "rmnet_ll.h"
 
 #include "rmnet_qmi.h"
 #include "qmi_rmnet.h"
@@ -118,6 +119,10 @@ rmnet_deliver_skb(struct sk_buff *skb, struct rmnet_port *port)
 	skb->pkt_type = PACKET_HOST;
 	skb_set_mac_header(skb, 0);
 
+	/* Low latency packets use a different balancing scheme */
+	if (skb->priority == 0xda1a)
+		goto skip_shs;
+
 	rcu_read_lock();
 	rmnet_shs_stamp = rcu_dereference(rmnet_shs_skb_entry);
 	if (rmnet_shs_stamp) {
@@ -127,6 +132,7 @@ rmnet_deliver_skb(struct sk_buff *skb, struct rmnet_port *port)
 	}
 	rcu_read_unlock();
 
+skip_shs:
 	netif_receive_skb(skb);
 }
 EXPORT_SYMBOL(rmnet_deliver_skb);
@@ -323,7 +329,8 @@ next_skb:
 
 static int rmnet_map_egress_handler(struct sk_buff *skb,
 				    struct rmnet_port *port, u8 mux_id,
-				    struct net_device *orig_dev)
+				    struct net_device *orig_dev,
+				    bool low_latency)
 {
 	int required_headroom, additional_header_len, csum_type, tso = 0;
 	struct rmnet_map_header *map_header;
@@ -354,10 +361,13 @@ static int rmnet_map_egress_handler(struct sk_buff *skb,
 	if (csum_type &&
 	    (skb_shinfo(skb)->gso_type & (SKB_GSO_UDP_L4 | SKB_GSO_TCPV4 | SKB_GSO_TCPV6)) &&
 	     skb_shinfo(skb)->gso_size) {
+		struct rmnet_aggregation_state *state;
 		unsigned long flags;
 
+		state = &port->agg_state[(low_latency) ? RMNET_LL_AGG_STATE :
+					 RMNET_DEFAULT_AGG_STATE];
 		spin_lock_irqsave(&port->agg_lock, flags);
-		rmnet_map_send_agg_skb(port, flags);
+		rmnet_map_send_agg_skb(state, flags);
 
 		if (rmnet_map_add_tso_header(skb, port, orig_dev))
 			return -EINVAL;
@@ -380,7 +390,7 @@ static int rmnet_map_egress_handler(struct sk_buff *skb,
 		if (rmnet_map_tx_agg_skip(skb, required_headroom) || tso)
 			goto done;
 
-		rmnet_map_tx_aggregate(skb, port);
+		rmnet_map_tx_aggregate(skb, port, low_latency);
 		return -EINPROGRESS;
 	}
 
@@ -457,7 +467,7 @@ EXPORT_SYMBOL(rmnet_rx_handler);
  * for egress device configured in logical endpoint. Packet is then transmitted
  * on the egress device.
  */
-void rmnet_egress_handler(struct sk_buff *skb)
+void rmnet_egress_handler(struct sk_buff *skb, bool low_latency)
 {
 	struct net_device *orig_dev;
 	struct rmnet_port *port;
@@ -480,7 +490,8 @@ void rmnet_egress_handler(struct sk_buff *skb)
 		goto drop;
 
 	skb_len = skb->len;
-	err = rmnet_map_egress_handler(skb, port, mux_id, orig_dev);
+	err = rmnet_map_egress_handler(skb, port, mux_id, orig_dev,
+				       low_latency);
 	if (err == -ENOMEM || err == -EINVAL) {
 		goto drop;
 	} else if (err == -EINPROGRESS) {
@@ -490,7 +501,13 @@ void rmnet_egress_handler(struct sk_buff *skb)
 
 	rmnet_vnd_tx_fixup(orig_dev, skb_len);
 
-	dev_queue_xmit(skb);
+	if (low_latency) {
+		if (rmnet_ll_send_skb(skb))
+			goto drop;
+	} else {
+		dev_queue_xmit(skb);
+	}
+
 	return;
 
 drop:

core/rmnet_handlers.h

@@ -1,4 +1,4 @@
-/* Copyright (c) 2013, 2016-2017, 2019-2020, The Linux Foundation. All rights reserved.
+/* Copyright (c) 2013, 2016-2017, 2019-2021, The Linux Foundation. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 and
@@ -23,7 +23,7 @@ enum rmnet_packet_context {
 	RMNET_WQ_CTX,
 };
 
-void rmnet_egress_handler(struct sk_buff *skb);
+void rmnet_egress_handler(struct sk_buff *skb, bool low_latency);
 void rmnet_deliver_skb(struct sk_buff *skb, struct rmnet_port *port);
 void rmnet_deliver_skb_wq(struct sk_buff *skb, struct rmnet_port *port,
 			  enum rmnet_packet_context ctx);

core/rmnet_ll.c

@@ -0,0 +1,379 @@
+/* Copyright (c) 2020-2021 The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * RmNet Low Latency channel handlers
+ */
+
+#include <linux/device.h>
+#include <linux/netdevice.h>
+#include <linux/of.h>
+#include <linux/skbuff.h>
+#include <linux/mhi.h>
+#include <linux/if_ether.h>
+#include <linux/mm.h>
+#include <linux/list.h>
+#include <linux/version.h>
+#include "rmnet_ll.h"
+
+#define RMNET_LL_DEFAULT_MRU 0x8000
+#define RMNET_LL_MAX_RECYCLE_ITER 16
+
+struct rmnet_ll_buffer {
+	struct list_head list;
+	struct page *page;
+	bool temp_alloc;
+	bool submitted;
+};
+
+struct rmnet_ll_buffer_pool {
+	struct list_head buf_list;
+	/* Protect access to the recycle buffer pool */
+	spinlock_t pool_lock;
+	struct list_head *last;
+	u32 pool_size;
+};
+
+struct rmnet_ll_endpoint {
+	struct rmnet_ll_buffer_pool buf_pool;
+	struct mhi_device *mhi_dev;
+	struct net_device *mhi_netdev;
+	u32 dev_mru;
+	u32 page_order;
+	u32 buf_len;
+};
+
+static struct rmnet_ll_endpoint *rmnet_ll_ep;
+static struct rmnet_ll_stats rmnet_ll_stats;
+/* For TX synch with MHI via mhi_queue_transfer() */
+static DEFINE_SPINLOCK(rmnet_ll_tx_lock);
+
+static void rmnet_ll_buffers_submit(struct rmnet_ll_endpoint *ll_ep,
+				    struct list_head *buf_list)
+{
+	struct rmnet_ll_buffer *ll_buf;
+	int rc;
+
+	list_for_each_entry(ll_buf, buf_list, list) {
+		if (ll_buf->submitted)
+			continue;
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)
+		rc = mhi_queue_transfer(ll_ep->mhi_dev, DMA_FROM_DEVICE,
+					page_address(ll_buf->page),
+					ll_ep->buf_len, MHI_EOT);
+#else
+		rc = mhi_queue_buf(ll_ep->mhi_dev, DMA_FROM_DEVICE,
+				   page_address(ll_buf->page),
+				   ll_ep->buf_len, MHI_EOT);
+#endif
+		if (rc) {
+			rmnet_ll_stats.rx_queue_err++;
+			/* Don't leak the page if we're not storing it */
+			if (ll_buf->temp_alloc)
+				put_page(ll_buf->page);
+		} else {
+			ll_buf->submitted = true;
+			rmnet_ll_stats.rx_queue++;
+		}
+	}
+}
+
+static struct rmnet_ll_buffer *
+rmnet_ll_buffer_alloc(struct rmnet_ll_endpoint *ll_ep, gfp_t gfp)
+{
+	struct rmnet_ll_buffer *ll_buf;
+	struct page *page;
+	void *addr;
+
+	page = __dev_alloc_pages(gfp, ll_ep->page_order);
+	if (!page)
+		return NULL;
+
+	/* Store the buffer information at the end */
+	addr = page_address(page);
+	ll_buf = addr + ll_ep->buf_len;
+	ll_buf->page = page;
+	ll_buf->submitted = false;
+	INIT_LIST_HEAD(&ll_buf->list);
+	return ll_buf;
+}
+
+static int rmnet_ll_buffer_pool_alloc(struct rmnet_ll_endpoint *ll_ep)
+{
+	spin_lock_init(&ll_ep->buf_pool.pool_lock);
+	INIT_LIST_HEAD(&ll_ep->buf_pool.buf_list);
+	ll_ep->buf_pool.last = ll_ep->buf_pool.buf_list.next;
+	ll_ep->buf_pool.pool_size = 0;
+	return 0;
+}
+
+static void rmnet_ll_buffer_pool_free(struct rmnet_ll_endpoint *ll_ep)
+{
+	struct rmnet_ll_buffer *ll_buf, *tmp;
+	list_for_each_entry_safe(ll_buf, tmp, &ll_ep->buf_pool.buf_list, list) {
+		list_del(&ll_buf->list);
+		put_page(ll_buf->page);
+	}
+
+	ll_ep->buf_pool.last = NULL;
+}
+
+static void rmnet_ll_buffers_recycle(struct rmnet_ll_endpoint *ll_ep)
+{
+	struct rmnet_ll_buffer *ll_buf, *tmp;
+	LIST_HEAD(buf_list);
+	int num_tre, count = 0, iter = 0;
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)
+	num_tre = mhi_get_no_free_descriptors(ll_ep->mhi_dev, DMA_FROM_DEVICE);
+#else
+	num_tre = mhi_get_free_desc_count(ll_ep->mhi_dev, DMA_FROM_DEVICE);
+#endif
+	if (!num_tre)
+		goto out;
+
+	list_for_each_entry_safe(ll_buf, tmp, ll_ep->buf_pool.last, list) {
+		if (++iter > RMNET_LL_MAX_RECYCLE_ITER || count == num_tre)
+			break;
+
+		if (ll_buf->submitted)
+			continue;
+
+		count++;
+		list_move_tail(&ll_buf->list, &buf_list);
+	}
+
+	/* Mark where we left off */
+	ll_ep->buf_pool.last = &ll_buf->list;
+	/* Submit any pool buffers to the HW if we found some */
+	if (count) {
+		rmnet_ll_buffers_submit(ll_ep, &buf_list);
+		/* requeue immediately BEFORE the last checked element */
+		list_splice_tail_init(&buf_list, ll_ep->buf_pool.last);
+	}
+
+	/* Do any temporary allocations needed to fill the rest */
+	for (; count < num_tre; count++) {
+		ll_buf = rmnet_ll_buffer_alloc(ll_ep, GFP_ATOMIC);
+		if (!ll_buf)
+			break;
+
+		list_add_tail(&ll_buf->list, &buf_list);
+		ll_buf->temp_alloc = true;
+		rmnet_ll_stats.rx_tmp_allocs++;
+	}
+
+	if (!list_empty(&buf_list))
+		rmnet_ll_buffers_submit(ll_ep, &buf_list);
+
+out:
+	return;
+}
+
+static void rmnet_ll_rx(struct mhi_device *mhi_dev, struct mhi_result *res)
+{
+	struct rmnet_ll_endpoint *ll_ep = dev_get_drvdata(&mhi_dev->dev);
+	struct rmnet_ll_buffer *ll_buf;
+	struct sk_buff *skb;
+
+	/* Get the buffer struct back for our page information */
+	ll_buf = res->buf_addr + ll_ep->buf_len;
+	ll_buf->submitted = false;
+	if (res->transaction_status) {
+		rmnet_ll_stats.rx_status_err++;
+		goto err;
+	} else if (!res->bytes_xferd) {
+		rmnet_ll_stats.rx_null++;
+		goto err;
+	}
+
+	/* Store this away so we don't have to look it up every time */
+	if (!ll_ep->mhi_netdev) {
+		ll_ep->mhi_netdev = dev_get_by_name(&init_net, "rmnet_mhi0");
+		if (!ll_ep->mhi_netdev)
+			goto err;
+	}
+
+	skb = alloc_skb(0, GFP_ATOMIC);
+	if (!skb) {
+		rmnet_ll_stats.rx_oom++;
+		goto err;
+	}
+
+	/* Build the SKB and pass it off to the stack */
+	skb_add_rx_frag(skb, 0, ll_buf->page, 0, res->bytes_xferd,
+			ll_ep->buf_len);
+	if (!ll_buf->temp_alloc)
+		get_page(ll_buf->page);
+
+	skb->dev = ll_ep->mhi_netdev;
+	skb->protocol = htons(ETH_P_MAP);
+	/* Mark this as arriving on the LL channel. Allows rmnet to skip
+	 * module handling as needed.
+	 */
+	skb->priority = 0xda1a;
+	rmnet_ll_stats.rx_pkts++;
+	netif_rx(skb);
+	rmnet_ll_buffers_recycle(ll_ep);
+	return;
+
+err:
+	/* Go, and never darken my towels again! */
+	if (ll_buf->temp_alloc)
+		put_page(ll_buf->page);
+}
+
+static void rmnet_ll_tx_complete(struct mhi_device *mhi_dev,
+				 struct mhi_result *res)
+{
+	struct sk_buff *skb = res->buf_addr;
+
+	/* Check the result and free the SKB */
+	if (res->transaction_status)
+		rmnet_ll_stats.tx_complete_err++;
+	else
+		rmnet_ll_stats.tx_complete++;
+
+	dev_kfree_skb_any(skb);
+}
+
+static int rmnet_ll_probe(struct mhi_device *mhi_dev,
+			  const struct mhi_device_id *id)
+{
+	struct rmnet_ll_endpoint *ll_ep;
+	int rc;
+
+	/* Allocate space for our state from the managed pool tied to the life
+	 * of the mhi device.
+	 */
+	ll_ep = devm_kzalloc(&mhi_dev->dev, sizeof(*ll_ep), GFP_KERNEL);
+	if (!ll_ep)
+		return -ENOMEM;
+
+	/* Hold on to the mhi_dev so we can send data to it later */
+	ll_ep->mhi_dev = mhi_dev;
+
+	/* Grab the MRU of the device so we know the size of the pages we need
+	 * to allocate for the pool.
+	 */
+	rc = of_property_read_u32(mhi_dev->dev.of_node, "mhi,mru",
+				  &ll_ep->dev_mru);
+	if (rc || !ll_ep->dev_mru)
+		/* Use our default mru */
+		ll_ep->dev_mru = RMNET_LL_DEFAULT_MRU;
+
+	ll_ep->page_order = get_order(ll_ep->dev_mru);
+	/* We store some stuff at the end of the page, so don't let the HW
+	 * use that part of it.
+	 */
+	ll_ep->buf_len = ll_ep->dev_mru - sizeof(struct rmnet_ll_buffer);
+
+	/* Tell MHI to initialize the UL/DL channels for transfer */
+	rc = mhi_prepare_for_transfer(mhi_dev);
+	if (rc) {
+		pr_err("%s(): Failed to prepare device for transfer: 0x%x\n",
+		       __func__, rc);
+		return rc;
+	}
+
+	rc = rmnet_ll_buffer_pool_alloc(ll_ep);
+	if (rc) {
+		pr_err("%s(): Failed to allocate buffer pool: %d\n", __func__,
+		       rc);
+		mhi_unprepare_from_transfer(mhi_dev);
+		return rc;
+	}
+
+	rmnet_ll_buffers_recycle(ll_ep);
+
+	/* Not a fan of storing this pointer in two locations, but I've yet to
+	 * come up with any other good way of accessing it on the TX path from
+	 * rmnet otherwise, since we won't have any references to the mhi_dev.
+	 */
+	dev_set_drvdata(&mhi_dev->dev, ll_ep);
+	rmnet_ll_ep = ll_ep;
+	return 0;
+}
+
+static void rmnet_ll_remove(struct mhi_device *mhi_dev)
+{
+	struct rmnet_ll_endpoint *ll_ep;
+
+	ll_ep = dev_get_drvdata(&mhi_dev->dev);
+	/* Remove our private data form the device. No need to free it though.
+	 * It will be freed once the mhi_dev is released since it was alloced
+	 * from a managed pool.
+	 */
+	dev_set_drvdata(&mhi_dev->dev, NULL);
+	rmnet_ll_ep = NULL;
+	rmnet_ll_buffer_pool_free(ll_ep);
+}
+
+static const struct mhi_device_id rmnet_ll_channel_table[] = {
+	{
+		.chan = "RMNET_DATA_LL",
+	},
+	{},
+};
+
+static struct mhi_driver rmnet_ll_driver = {
+	.probe = rmnet_ll_probe,
+	.remove = rmnet_ll_remove,
+	.dl_xfer_cb = rmnet_ll_rx,
+	.ul_xfer_cb = rmnet_ll_tx_complete,
+	.id_table = rmnet_ll_channel_table,
+	.driver = {
+		.name = "rmnet_ll",
+		.owner = THIS_MODULE,
+	},
+};
+
+int rmnet_ll_send_skb(struct sk_buff *skb)
+{
+	struct rmnet_ll_endpoint *ll_ep = rmnet_ll_ep;
+	int rc = -ENODEV;
+
+	/* Lock to prevent multiple sends at the same time. mhi_queue_transfer()
+	 * cannot be called in parallel for the same DMA direction.
+	 */
+	spin_lock_bh(&rmnet_ll_tx_lock);
+	if (ll_ep)
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)
+		rc = mhi_queue_transfer(ll_ep->mhi_dev, DMA_TO_DEVICE, skb,
+					skb->len, MHI_EOT);
+#else
+		rc = mhi_queue_skb(ll_ep->mhi_dev, DMA_TO_DEVICE, skb,
+				   skb->len, MHI_EOT);
+#endif
+
+	spin_unlock_bh(&rmnet_ll_tx_lock);
+	if (rc)
+		rmnet_ll_stats.tx_queue_err++;
+	else
+		rmnet_ll_stats.tx_queue++;
+
+	return rc;
+}
+
+struct rmnet_ll_stats *rmnet_ll_get_stats(void)
+{
+	return &rmnet_ll_stats;
+}
+
+int rmnet_ll_init(void)
+{
+	return mhi_driver_register(&rmnet_ll_driver);
+}
+
+void rmnet_ll_exit(void)
+{
+	mhi_driver_unregister(&rmnet_ll_driver);
+}

core/rmnet_ll.h

@@ -0,0 +1,70 @@
+/* Copyright (c) 2020-2021 The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * RmNet Low Latency channel handlers
+ */
+
+#ifndef __RMNET_LL_H__
+#define __RMNET_LL_H__
+
+#include <linux/skbuff.h>
+
+struct rmnet_ll_stats {
+		u64 tx_queue;
+		u64 tx_queue_err;
+		u64 tx_complete;
+		u64 tx_complete_err;
+		u64 rx_queue;
+		u64 rx_queue_err;
+		u64 rx_status_err;
+		u64 rx_null;
+		u64 rx_oom;
+		u64 rx_pkts;
+		u64 rx_tmp_allocs;
+};
+
+#if IS_ENABLED(CONFIG_MHI_BUS)
+
+int rmnet_ll_send_skb(struct sk_buff *skb);
+struct rmnet_ll_stats *rmnet_ll_get_stats(void);
+int rmnet_ll_init(void);
+void rmnet_ll_exit(void);
+
+#else
+
+static struct rmnet_ll_stats rmnet_ll_dummy_stats;
+
+static inline int rmnet_ll_send_skb(struct sk_buff *skb)
+{
+	return -EINVAL;
+}
+
+static inline struct rmnet_ll_stats *rmnet_ll_get_stats(void)
+{
+	return &rmnet_ll_dummy_stats;
+}
+
+static inline int rmnet_ll_init(void)
+{
+	/* Allow configuration to continue. Nothing else will happen since all
+	 * this does is register the driver with the mhi framework, and if the
+	 * channel never comes up, we don't do anything.
+	 */
+	return 0;
+}
+
+static inline void rmnet_ll_exit(void)
+{
+}
+
+#endif
+
+#endif

core/rmnet_map.h

@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2020, The Linux Foundation. All rights reserved.
+/* Copyright (c) 2013-2021, The Linux Foundation. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 and
@@ -271,7 +271,8 @@ int rmnet_map_process_next_hdr_packet(struct sk_buff *skb,
 				      struct sk_buff_head *list,
 				      u16 len);
 int rmnet_map_tx_agg_skip(struct sk_buff *skb, int offset);
-void rmnet_map_tx_aggregate(struct sk_buff *skb, struct rmnet_port *port);
+void rmnet_map_tx_aggregate(struct sk_buff *skb, struct rmnet_port *port,
+			    bool low_latency);
 void rmnet_map_tx_aggregate_init(struct rmnet_port *port);
 void rmnet_map_tx_aggregate_exit(struct rmnet_port *port);
 void rmnet_map_update_ul_agg_config(struct rmnet_port *port, u16 size,
@@ -291,7 +292,8 @@ int rmnet_map_dl_ind_register(struct rmnet_port *port,
 int rmnet_map_dl_ind_deregister(struct rmnet_port *port,
 				struct rmnet_map_dl_ind *dl_ind);
 void rmnet_map_cmd_exit(struct rmnet_port *port);
-void rmnet_map_send_agg_skb(struct rmnet_port *port, unsigned long flags);
+void rmnet_map_send_agg_skb(struct rmnet_aggregation_state *state,
+			    unsigned long flags);
 int rmnet_map_add_tso_header(struct sk_buff *skb, struct rmnet_port *port,
 			      struct net_device *orig_dev);
 #endif /* _RMNET_MAP_H_ */

core/rmnet_map_data.c

@@ -21,6 +21,7 @@
 #include "rmnet_map.h"
 #include "rmnet_private.h"
 #include "rmnet_handlers.h"
+#include "rmnet_ll.h"
 
 #define RMNET_MAP_PKT_COPY_THRESHOLD 64
 #define RMNET_MAP_DEAGGR_SPACING  64
@@ -397,6 +398,7 @@ struct sk_buff *rmnet_map_deaggregate(struct sk_buff *skb,
 		memcpy(skbn->data, data, packet_len);
 	}
 
+	skbn->priority = skb->priority;
 	pskb_pull(skb, packet_len);
 
 	return skbn;
@@ -855,6 +857,9 @@ __rmnet_map_segment_coal_skb(struct sk_buff *coal_skb,
 	if (coal_meta->pkt_count > 1)
 		rmnet_map_gso_stamp(skbn, coal_meta);
 
+	/* Propagate priority value */
+	skbn->priority = coal_skb->priority;
+
 	__skb_queue_tail(list, skbn);
 
 	/* Update meta information to move past the data we just segmented */
@@ -1249,35 +1254,35 @@ int rmnet_map_tx_agg_skip(struct sk_buff *skb, int offset)
 static void rmnet_map_flush_tx_packet_work(struct work_struct *work)
 {
 	struct sk_buff *skb = NULL;
-	struct rmnet_port *port;
+	struct rmnet_aggregation_state *state;
 	unsigned long flags;
 
-	port = container_of(work, struct rmnet_port, agg_wq);
+	state = container_of(work, struct rmnet_aggregation_state, agg_wq);
 
-	spin_lock_irqsave(&port->agg_lock, flags);
-	if (likely(port->agg_state == -EINPROGRESS)) {
+	spin_lock_irqsave(state->agg_lock, flags);
+	if (likely(state->agg_state == -EINPROGRESS)) {
 		/* Buffer may have already been shipped out */
-		if (likely(port->agg_skb)) {
-			skb = port->agg_skb;
-			port->agg_skb = NULL;
-			port->agg_count = 0;
-			memset(&port->agg_time, 0, sizeof(port->agg_time));
+		if (likely(state->agg_skb)) {
+			skb = state->agg_skb;
+			state->agg_skb = NULL;
+			state->agg_count = 0;
+			memset(&state->agg_time, 0, sizeof(state->agg_time));
 		}
-		port->agg_state = 0;
+		state->agg_state = 0;
 	}
 
-	spin_unlock_irqrestore(&port->agg_lock, flags);
+	spin_unlock_irqrestore(state->agg_lock, flags);
 	if (skb)
-		dev_queue_xmit(skb);
+		state->send_agg_skb(skb);
 }
 
 enum hrtimer_restart rmnet_map_flush_tx_packet_queue(struct hrtimer *t)
 {
-	struct rmnet_port *port;
+	struct rmnet_aggregation_state *state;
 
-	port = container_of(t, struct rmnet_port, hrtimer);
+	state = container_of(t, struct rmnet_aggregation_state, hrtimer);
 
-	schedule_work(&port->agg_wq);
+	schedule_work(&state->agg_wq);
 	return HRTIMER_NORESTART;
 }
 
@@ -1423,99 +1428,105 @@ static struct sk_buff *rmnet_map_build_skb(struct rmnet_port *port)
 	return skb;
 }
 
-void rmnet_map_send_agg_skb(struct rmnet_port *port, unsigned long flags)
+void rmnet_map_send_agg_skb(struct rmnet_aggregation_state *state,
+			    unsigned long flags)
 {
 	struct sk_buff *agg_skb;
 
-	if (!port->agg_skb) {
-		spin_unlock_irqrestore(&port->agg_lock, flags);
+	if (!state->agg_skb) {
+		spin_unlock_irqrestore(state->agg_lock, flags);
 		return;
 	}
 
-	agg_skb = port->agg_skb;
+	agg_skb = state->agg_skb;
 	/* Reset the aggregation state */
-	port->agg_skb = NULL;
-	port->agg_count = 0;
-	memset(&port->agg_time, 0, sizeof(port->agg_time));
-	port->agg_state = 0;
-	spin_unlock_irqrestore(&port->agg_lock, flags);
-	hrtimer_cancel(&port->hrtimer);
-	dev_queue_xmit(agg_skb);
+	state->agg_skb = NULL;
+	state->agg_count = 0;
+	memset(&state->agg_time, 0, sizeof(state->agg_time));
+	state->agg_state = 0;
+	spin_unlock_irqrestore(state->agg_lock, flags);
+	hrtimer_cancel(&state->hrtimer);
+	state->send_agg_skb(agg_skb);
 }
 
-void rmnet_map_tx_aggregate(struct sk_buff *skb, struct rmnet_port *port)
+void rmnet_map_tx_aggregate(struct sk_buff *skb, struct rmnet_port *port,
+			    bool low_latency)
 {
+	struct rmnet_aggregation_state *state;
 	struct timespec64 diff, last;
 	int size;
 	unsigned long flags;
 
+	state = &port->agg_state[(low_latency) ? RMNET_LL_AGG_STATE :
+						 RMNET_DEFAULT_AGG_STATE];
+
 new_packet:
 	spin_lock_irqsave(&port->agg_lock, flags);
-	memcpy(&last, &port->agg_last, sizeof(last));
-	ktime_get_real_ts64(&port->agg_last);
+	memcpy(&last, &state->agg_last, sizeof(last));
+	ktime_get_real_ts64(&state->agg_last);
 
 	if ((port->data_format & RMNET_EGRESS_FORMAT_PRIORITY) &&
 	    skb->priority) {
 		/* Send out any aggregated SKBs we have */
-		rmnet_map_send_agg_skb(port, flags);
+		rmnet_map_send_agg_skb(state, flags);
 		/* Send out the priority SKB. Not holding agg_lock anymore */
 		skb->protocol = htons(ETH_P_MAP);
-		dev_queue_xmit(skb);
+		state->send_agg_skb(skb);
 		return;
 	}
 
-	if (!port->agg_skb) {
+	if (!state->agg_skb) {
 		/* Check to see if we should agg first. If the traffic is very
 		 * sparse, don't aggregate. We will need to tune this later
 		 */
-		diff = timespec64_sub(port->agg_last, last);
+		diff = timespec64_sub(state->agg_last, last);
 		size = port->egress_agg_params.agg_size - skb->len;
 
 		if (diff.tv_sec > 0 || diff.tv_nsec > rmnet_agg_bypass_time ||
 		    size <= 0) {
 			spin_unlock_irqrestore(&port->agg_lock, flags);
 			skb->protocol = htons(ETH_P_MAP);
-			dev_queue_xmit(skb);
+			state->send_agg_skb(skb);
 			return;
 		}
 
-		port->agg_skb = rmnet_map_build_skb(port);
-		if (!port->agg_skb) {
-			port->agg_skb = 0;
-			port->agg_count = 0;
-			memset(&port->agg_time, 0, sizeof(port->agg_time));
+		state->agg_skb = rmnet_map_build_skb(port);
+		if (!state->agg_skb) {
+			state->agg_skb = NULL;
+			state->agg_count = 0;
+			memset(&state->agg_time, 0, sizeof(state->agg_time));
 			spin_unlock_irqrestore(&port->agg_lock, flags);
 			skb->protocol = htons(ETH_P_MAP);
-			dev_queue_xmit(skb);
+			state->send_agg_skb(skb);
 			return;
 		}
 
-		rmnet_map_linearize_copy(port->agg_skb, skb);
-		port->agg_skb->dev = skb->dev;
-		port->agg_skb->protocol = htons(ETH_P_MAP);
-		port->agg_count = 1;
-		ktime_get_real_ts64(&port->agg_time);
+		rmnet_map_linearize_copy(state->agg_skb, skb);
+		state->agg_skb->dev = skb->dev;
+		state->agg_skb->protocol = htons(ETH_P_MAP);
+		state->agg_count = 1;
+		ktime_get_real_ts64(&state->agg_time);
 		dev_kfree_skb_any(skb);
 		goto schedule;
 	}
-	diff = timespec64_sub(port->agg_last, port->agg_time);
-	size = skb_tailroom(port->agg_skb);
+	diff = timespec64_sub(state->agg_last, state->agg_time);
+	size = skb_tailroom(state->agg_skb);
 
 	if (skb->len > size ||
-	    port->agg_count >= port->egress_agg_params.agg_count ||
+	    state->agg_count >= port->egress_agg_params.agg_count ||
 	    diff.tv_sec > 0 || diff.tv_nsec > rmnet_agg_time_limit) {
-		rmnet_map_send_agg_skb(port, flags);
+		rmnet_map_send_agg_skb(state, flags);
 		goto new_packet;
 	}
 
-	rmnet_map_linearize_copy(port->agg_skb, skb);
-	port->agg_count++;
+	rmnet_map_linearize_copy(state->agg_skb, skb);
+	state->agg_count++;
 	dev_kfree_skb_any(skb);
 
 schedule:
-	if (port->agg_state != -EINPROGRESS) {
-		port->agg_state = -EINPROGRESS;
-		hrtimer_start(&port->hrtimer,
+	if (state->agg_state != -EINPROGRESS) {
+		state->agg_state = -EINPROGRESS;
+		hrtimer_start(&state->hrtimer,
 			      ns_to_ktime(port->egress_agg_params.agg_time),
 			      HRTIMER_MODE_REL);
 	}
@@ -1556,8 +1567,8 @@ done:
 
 void rmnet_map_tx_aggregate_init(struct rmnet_port *port)
 {
-	hrtimer_init(&port->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
-	port->hrtimer.function = rmnet_map_flush_tx_packet_queue;
+	unsigned int i;
+
 	spin_lock_init(&port->agg_lock);
 	INIT_LIST_HEAD(&port->agg_list);
 
@@ -1568,26 +1579,48 @@ void rmnet_map_tx_aggregate_init(struct rmnet_port *port)
 	 */
 	rmnet_map_update_ul_agg_config(port, PAGE_SIZE - 1, 20, 0, 3000000);
 
-	INIT_WORK(&port->agg_wq, rmnet_map_flush_tx_packet_work);
+	for (i = RMNET_DEFAULT_AGG_STATE; i < RMNET_MAX_AGG_STATE; i++) {
+		struct rmnet_aggregation_state *state = &port->agg_state[i];
+
+		hrtimer_init(&state->hrtimer, CLOCK_MONOTONIC,
+			     HRTIMER_MODE_REL);
+		state->hrtimer.function = rmnet_map_flush_tx_packet_queue;
+		INIT_WORK(&state->agg_wq, rmnet_map_flush_tx_packet_work);
+		state->agg_lock = &port->agg_lock;
+	}
+
+	/* Set delivery functions for each aggregation state */
+	port->agg_state[RMNET_DEFAULT_AGG_STATE].send_agg_skb = dev_queue_xmit;
+	port->agg_state[RMNET_LL_AGG_STATE].send_agg_skb = rmnet_ll_send_skb;
 }
 
 void rmnet_map_tx_aggregate_exit(struct rmnet_port *port)
 {
 	unsigned long flags;
+	unsigned int i;
 
-	hrtimer_cancel(&port->hrtimer);
-	cancel_work_sync(&port->agg_wq);
+	for (i = RMNET_DEFAULT_AGG_STATE; i < RMNET_MAX_AGG_STATE; i++) {
+		struct rmnet_aggregation_state *state = &port->agg_state[i];
+
+		hrtimer_cancel(&state->hrtimer);
+		cancel_work_sync(&state->agg_wq);
+	}
 
 	spin_lock_irqsave(&port->agg_lock, flags);
-	if (port->agg_state == -EINPROGRESS) {
-		if (port->agg_skb) {
-			kfree_skb(port->agg_skb);
-			port->agg_skb = NULL;
-			port->agg_count = 0;
-			memset(&port->agg_time, 0, sizeof(port->agg_time));
-		}
+	for (i = RMNET_DEFAULT_AGG_STATE; i < RMNET_MAX_AGG_STATE; i++) {
+		struct rmnet_aggregation_state *state = &port->agg_state[i];
+
+		if (state->agg_state == -EINPROGRESS) {
+			if (state->agg_skb) {
+				kfree_skb(state->agg_skb);
+				state->agg_skb = NULL;
+				state->agg_count = 0;
+				memset(&state->agg_time, 0,
+				       sizeof(state->agg_time));
+			}
 
-		port->agg_state = 0;
+			state->agg_state = 0;
+		}
 	}
 
 	rmnet_free_agg_pages(port);
@@ -1599,25 +1632,32 @@ void rmnet_map_tx_qmap_cmd(struct sk_buff *qmap_skb)
 	struct rmnet_port *port;
 	struct sk_buff *agg_skb;
 	unsigned long flags;
+	unsigned int i;
 
 	port = rmnet_get_port(qmap_skb->dev);
 
-	if (port && (port->data_format & RMNET_EGRESS_FORMAT_AGGREGATION)) {
+	if (port && (port->data_format & RMNET_EGRESS_FORMAT_AGGREGATION))
+		goto send;
+
+	for (i = RMNET_DEFAULT_AGG_STATE; i < RMNET_MAX_AGG_STATE; i++) {
+		struct rmnet_aggregation_state *state = &port->agg_state[i];
+
 		spin_lock_irqsave(&port->agg_lock, flags);
-		if (port->agg_skb) {
-			agg_skb = port->agg_skb;
-			port->agg_skb = 0;
-			port->agg_count = 0;
-			memset(&port->agg_time, 0, sizeof(port->agg_time));
-			port->agg_state = 0;
+		if (state->agg_skb) {
+			agg_skb = state->agg_skb;
+			state->agg_skb = NULL;
+			state->agg_count = 0;
+			memset(&state->agg_time, 0, sizeof(state->agg_time));
+			state->agg_state = 0;
 			spin_unlock_irqrestore(&port->agg_lock, flags);
-			hrtimer_cancel(&port->hrtimer);
-			dev_queue_xmit(agg_skb);
+			hrtimer_cancel(&state->hrtimer);
+			state->send_agg_skb(agg_skb);
 		} else {
 			spin_unlock_irqrestore(&port->agg_lock, flags);
 		}
 	}
 
+send:
 	dev_queue_xmit(qmap_skb);
 }
 EXPORT_SYMBOL(rmnet_map_tx_qmap_cmd);

core/rmnet_vnd.c

@@ -26,6 +26,7 @@
 #include "rmnet_map.h"
 #include "rmnet_vnd.h"
 #include "rmnet_genl.h"
+#include "rmnet_ll.h"
 
 #include "qmi_rmnet.h"
 #include "rmnet_qmi.h"
@@ -74,6 +75,7 @@ static netdev_tx_t rmnet_vnd_start_xmit(struct sk_buff *skb,
 	u32 mark;
 	unsigned int len;
 	rmnet_perf_tether_egress_hook_t rmnet_perf_tether_egress;
+	bool low_latency;
 
 	priv = netdev_priv(dev);
 	if (priv->real_dev) {
@@ -86,7 +88,8 @@ static netdev_tx_t rmnet_vnd_start_xmit(struct sk_buff *skb,
 		if (rmnet_perf_tether_egress) {
 			rmnet_perf_tether_egress(skb);
 		}
-		rmnet_egress_handler(skb);
+		low_latency = qmi_rmnet_flow_is_low_latency(dev, ip_type, mark);
+		rmnet_egress_handler(skb, low_latency);
 		qmi_rmnet_burst_fc_check(dev, ip_type, mark, len);
 		qmi_rmnet_work_maybe_restart(rmnet_get_rmnet_port(dev));
 	} else {
@@ -288,15 +291,35 @@ static const char rmnet_port_gstrings_stats[][ETH_GSTRING_LEN] = {
 	"UL agg alloc",
 };
 
+static const char rmnet_ll_gstrings_stats[][ETH_GSTRING_LEN] = {
+	"LL TX queues",
+	"LL TX queue errors",
+	"LL TX completions",
+	"LL TX completion errors",
+	"LL RX queues",
+	"LL RX queue errors",
+	"LL RX status errors",
+	"LL RX empty transfers",
+	"LL RX OOM errors",
+	"LL RX packets",
+	"LL RX temp buffer allocations",
+};
+
 static void rmnet_get_strings(struct net_device *dev, u32 stringset, u8 *buf)
 {
+	size_t off = 0;
+
 	switch (stringset) {
 	case ETH_SS_STATS:
 		memcpy(buf, &rmnet_gstrings_stats,
 		       sizeof(rmnet_gstrings_stats));
-		memcpy(buf + sizeof(rmnet_gstrings_stats),
+		off += sizeof(rmnet_gstrings_stats);
+		memcpy(buf + off,
 		       &rmnet_port_gstrings_stats,
 		       sizeof(rmnet_port_gstrings_stats));
+		off += sizeof(rmnet_port_gstrings_stats);
+		memcpy(buf + off, &rmnet_ll_gstrings_stats,
+		       sizeof(rmnet_ll_gstrings_stats));
 		break;
 	}
 }
@@ -306,7 +329,8 @@ static int rmnet_get_sset_count(struct net_device *dev, int sset)
 	switch (sset) {
 	case ETH_SS_STATS:
 		return ARRAY_SIZE(rmnet_gstrings_stats) +
-		       ARRAY_SIZE(rmnet_port_gstrings_stats);
+		       ARRAY_SIZE(rmnet_port_gstrings_stats) +
+		       ARRAY_SIZE(rmnet_ll_gstrings_stats);
 	default:
 		return -EOPNOTSUPP;
 	}
@@ -318,7 +342,9 @@ static void rmnet_get_ethtool_stats(struct net_device *dev,
 	struct rmnet_priv *priv = netdev_priv(dev);
 	struct rmnet_priv_stats *st = &priv->stats;
 	struct rmnet_port_priv_stats *stp;
+	struct rmnet_ll_stats *llp;
 	struct rmnet_port *port;
+	size_t off = 0;
 
 	port = rmnet_get_port(priv->real_dev);
 
@@ -326,10 +352,15 @@ static void rmnet_get_ethtool_stats(struct net_device *dev,
 		return;
 
 	stp = &port->stats;
+	llp = rmnet_ll_get_stats();
 
 	memcpy(data, st, ARRAY_SIZE(rmnet_gstrings_stats) * sizeof(u64));
-	memcpy(data + ARRAY_SIZE(rmnet_gstrings_stats), stp,
+	off += ARRAY_SIZE(rmnet_gstrings_stats);
+	memcpy(data + off, stp,
 	       ARRAY_SIZE(rmnet_port_gstrings_stats) * sizeof(u64));
+	off += ARRAY_SIZE(rmnet_port_gstrings_stats);
+	memcpy(data + off, llp,
+	       ARRAY_SIZE(rmnet_ll_gstrings_stats) * sizeof(u64));
 }
 
 static int rmnet_stats_reset(struct net_device *dev)