dpaa_eth: Use build_skb() on S/G path

Until now skbs were allocated on Rx (or taken from a preallocated
pool of skbs) and the frame headers copied into the skb linear part.

Eliminate the need to copy data by building the skb around the data
buffer. For S/G frames, use the first S/G entry to build the skb
linear part and add the other entries as fragments.

A lot of code is removed due to this change, including identifying
total header size while looking at the parse results, header copying,
seeding/extracting skbs to/from the skb list.

Change-Id: I6e461a2d3dced2abcaed7fa6eaf2606ba1bb6e04
Signed-off-by: Ioana Radulescu <ruxandra.radulescu@freescale.com>
Reviewed-on: http://git.am.freescale.net:8181/2412
Reviewed-by: Fleming Andrew-AFLEMING <AFLEMING@freescale.com>
Tested-by: Fleming Andrew-AFLEMING <AFLEMING@freescale.com>

3 files changed, 76 insertions, 224 deletions

diff --git a/drivers/net/ethernet/freescale/dpa/dpaa_eth.c b/drivers/net/ethernet/freescale/dpa/dpaa_eth.c
index cffef9e..4080d3d 100644
--- a/drivers/net/ethernet/freescale/dpa/dpaa_eth.c
+++ b/drivers/net/ethernet/freescale/dpa/dpaa_eth.c
@@ -413,15 +413,6 @@ static void dpaa_eth_refill_bpools(struct dpa_priv_s *priv,
 	while (count < CONFIG_FSL_DPAA_ETH_MAX_BUF_COUNT)
 		count += _dpa_bp_add_8_pages(dpa_bp);
 	*countptr = count;
-
-	/* Add skbs to the percpu skb list, reuse var count */
-	count = percpu_priv->skb_count;
-	if (unlikely(count < DEFAULT_SKB_COUNT / 4)) {
-		int skb_size = priv->tx_headroom + dpa_get_rx_extra_headroom() +
-				DPA_COPIED_HEADERS_SIZE;
-		dpa_list_add_skbs(percpu_priv, DEFAULT_SKB_COUNT - count,
-				  skb_size);
-	}
 #endif
 }
 
@@ -1403,10 +1394,8 @@ static void _dpa_tx_error(struct net_device		*net_dev,
  *		headers' length.
  */
 void __hot _dpa_process_parse_results(const t_FmPrsResult *parse_results,
-	const struct qm_fd *fd,
-	struct sk_buff *skb,
-	int *use_gro,
-	unsigned int *hdr_size __maybe_unused)
+				      const struct qm_fd *fd,
+				      struct sk_buff *skb, int *use_gro)
 {
 	if (fd->status & FM_FD_STAT_L4CV) {
 		/*
@@ -1428,13 +1417,6 @@ void __hot _dpa_process_parse_results(const t_FmPrsResult *parse_results,
 		if (!fm_l4_frame_is_tcp(parse_results))
 			*use_gro = 0;
 
-#ifdef CONFIG_FSL_DPAA_ETH_SG_SUPPORT
-		/*
-		 * If the L4 HXS Parser has successfully run, we can reduce the
-		 * number of bytes we'll memcopy into skb->data.
-		 */
-		*hdr_size = parse_results->nxthdr_off;
-#endif
 		return;
 	}
 
@@ -1444,18 +1426,6 @@ void __hot _dpa_process_parse_results(const t_FmPrsResult *parse_results,
 	 * checksum zero or an L4 proto other than TCP/UDP
 	 */
 	skb->ip_summed = CHECKSUM_NONE;
-#ifdef CONFIG_FSL_DPAA_ETH_SG_SUPPORT
-	/*
-	 * Even if checksum was not verified, it's still possible L4 parser
-	 * has run, in which case we know the headers size.
-	 * Otherwise we fall back to a safe default.
-	 */
-	if (fm_l4_hxs_has_run(parse_results))
-		*hdr_size = parse_results->nxthdr_off;
-	else
-		*hdr_size = min((ssize_t)DPA_COPIED_HEADERS_SIZE,
-				dpa_fd_length(fd));
-#endif
 
 	/* Bypass GRO for unknown traffic or if no PCDs are applied */
 	*use_gro = 0;
@@ -1475,7 +1445,6 @@ void __hot _dpa_rx(struct net_device *net_dev,
 	u32 fd_status = fd->status;
 	unsigned int skb_len;
 	t_FmPrsResult *parse_result;
-	unsigned int hdr_size_unused;
 	int use_gro = net_dev->features & NETIF_F_GRO;
 
 	skbh = (struct sk_buff **)phys_to_virt(addr);
@@ -1531,8 +1500,7 @@ void __hot _dpa_rx(struct net_device *net_dev,
 
 	/* Validate the skb csum and figure out whether GRO is appropriate */
 	parse_result = (t_FmPrsResult *)((u8 *)skbh + DPA_RX_PRIV_DATA_SIZE);
-	_dpa_process_parse_results(parse_result, fd, skb, &use_gro,
-					 &hdr_size_unused);
+	_dpa_process_parse_results(parse_result, fd, skb, &use_gro);
 
 #ifdef CONFIG_FSL_DPAA_TS
 	if (priv->ts_rx_en)
@@ -3869,17 +3837,6 @@ static int dpa_private_netdev_init(struct device_node *dpa_node,
 		percpu_priv = per_cpu_ptr(priv->percpu_priv, i);
 		percpu_priv->net_dev = net_dev;
 
-#ifdef CONFIG_FSL_DPAA_ETH_SG_SUPPORT
-		/* init the percpu list and add some skbs */
-		skb_queue_head_init(&percpu_priv->skb_list);
-
-		/* Skbs must accomodate the headroom and enough space
-		 * for the frame headers.
-		 */
-		dpa_list_add_skbs(percpu_priv, DEFAULT_SKB_COUNT,
-				  priv->tx_headroom + DPA_COPIED_HEADERS_SIZE +
-				  dpa_get_rx_extra_headroom());
-#endif
 		netif_napi_add(net_dev, &percpu_priv->napi, dpaa_eth_poll,
 			       DPA_NAPI_WEIGHT);
 	}
diff --git a/drivers/net/ethernet/freescale/dpa/dpaa_eth.h b/drivers/net/ethernet/freescale/dpa/dpaa_eth.h
index 22d6546..fbea46b 100644
--- a/drivers/net/ethernet/freescale/dpa/dpaa_eth.h
+++ b/drivers/net/ethernet/freescale/dpa/dpaa_eth.h
@@ -74,10 +74,12 @@
 
 
 #ifdef CONFIG_FSL_DPAA_ETH_SG_SUPPORT
-/* We may want this value configurable. Must be <= PAGE_SIZE
- * A lower value may help with recycling rates, at least on forwarding
+/* We may want this value configurable. Must be <= PAGE_SIZE minus a reserved
+ * area where skb shared info can reside
+ * A lower value may help with recycling rates, at least on forwarding.
  */
-#define dpa_bp_size(buffer_layout)	PAGE_SIZE
+#define DPA_SKB_TAILROOM	SKB_DATA_ALIGN(sizeof(struct skb_shared_info))
+#define dpa_bp_size(buffer_layout)	(PAGE_SIZE - DPA_SKB_TAILROOM)
 #else
 
 /* Default buffer size is based on L2 MAX_FRM value, minus the FCS which
@@ -165,16 +167,6 @@ void fsl_dpaa_eth_set_hooks(struct dpaa_eth_hooks_s *hooks);
 
 #define DPA_SGT_MAX_ENTRIES 16 /* maximum number of entries in SG Table */
 
-#ifdef CONFIG_FSL_DPAA_ETH_SG_SUPPORT
-#define DEFAULT_SKB_COUNT 64 /* maximum number of SKBs in each percpu list */
-/*
- * Default amount data to be copied from the beginning of a frame into the
- * linear part of the skb, in case we aren't using the hardware parser.
- */
-#define DPA_COPIED_HEADERS_SIZE 128
-
-#endif /* CONFIG_FSL_DPAA_ETH_SG_SUPPORT */
-
 /*
  * Largest value that the FQD's OAL field can hold.
  * This is DPAA-1.x specific.
@@ -329,12 +321,6 @@ struct dpa_percpu_priv_s {
 	int *dpa_bp_count;
 	struct dpa_bp *dpa_bp;
 	struct napi_struct napi;
-#ifdef CONFIG_FSL_DPAA_ETH_SG_SUPPORT
-	/* a list of preallocated SKBs for this CPU */
-	struct sk_buff_head skb_list;
-	/* current number of skbs in the CPU's list */
-	int skb_count;
-#endif
 	u64 in_interrupt;
 	u64 tx_returned;
 	u64 tx_confirm;
@@ -428,17 +414,13 @@ struct sk_buff *_dpa_cleanup_tx_fd(const struct dpa_priv_s *priv,
 				   const struct qm_fd *fd);
 
 void __hot _dpa_process_parse_results(const t_FmPrsResult *parse_results,
-				     const struct qm_fd *fd,
-				     struct sk_buff *skb,
-				     int *use_gro,
-				     unsigned int *hdr_size __maybe_unused);
+				      const struct qm_fd *fd,
+				      struct sk_buff *skb,
+				      int *use_gro);
 
 #ifdef CONFIG_FSL_DPAA_ETH_SG_SUPPORT
 void dpa_bp_add_8_pages(const struct dpa_bp *dpa_bp, int cpu_id);
 int _dpa_bp_add_8_pages(const struct dpa_bp *dpa_bp);
-
-void dpa_list_add_skbs(struct dpa_percpu_priv_s *cpu_priv, int count,
-		int skb_size);
 #endif
 
 /*
diff --git a/drivers/net/ethernet/freescale/dpa/dpaa_eth_sg.c b/drivers/net/ethernet/freescale/dpa/dpaa_eth_sg.c
index 8369374..a624e9a 100644
--- a/drivers/net/ethernet/freescale/dpa/dpaa_eth_sg.c
+++ b/drivers/net/ethernet/freescale/dpa/dpaa_eth_sg.c
@@ -149,52 +149,6 @@ void dpa_bp_add_8_pages(const struct dpa_bp *dpa_bp, int cpu)
 	*count_ptr += _dpa_bp_add_8_pages(dpa_bp);
 }
 
-void dpa_list_add_skb(struct dpa_percpu_priv_s *cpu_priv,
-		      struct sk_buff *new_skb)
-{
-	struct sk_buff_head *list_ptr;
-
-	if (cpu_priv->skb_count > DEFAULT_SKB_COUNT) {
-		dev_kfree_skb(new_skb);
-		return;
-	}
-
-	list_ptr = &cpu_priv->skb_list;
-	skb_queue_head(list_ptr, new_skb);
-
-	cpu_priv->skb_count += 1;
-}
-
-static struct sk_buff *dpa_list_get_skb(struct dpa_percpu_priv_s *cpu_priv)
-{
-	struct sk_buff_head *list_ptr;
-	struct sk_buff *new_skb;
-
-	list_ptr = &cpu_priv->skb_list;
-
-	new_skb = skb_dequeue(list_ptr);
-	if (new_skb)
-		cpu_priv->skb_count -= 1;
-
-	return new_skb;
-}
-
-void dpa_list_add_skbs(struct dpa_percpu_priv_s *cpu_priv, int count, int size)
-{
-	struct sk_buff *new_skb;
-	int i;
-
-	for (i = 0; i < count; i++) {
-		new_skb = dev_alloc_skb(size);
-		if (unlikely(!new_skb)) {
-			pr_err("dev_alloc_skb() failed\n");
-			break;
-		}
-
-		dpa_list_add_skb(cpu_priv, new_skb);
-	}
-}
-
 void dpa_make_private_pool(struct dpa_bp *dpa_bp)
 {
 	int i;
@@ -307,23 +261,18 @@ struct sk_buff *_dpa_cleanup_tx_fd(const struct dpa_priv_s *priv,
 }
 
 /*
- * Move the first DPA_COPIED_HEADERS_SIZE bytes to the skb linear buffer to
- * provide the networking stack the headers it requires in the linear buffer.
- *
- * If the entire frame fits in the skb linear buffer, the page holding the
- * received data is recycled as it is no longer required.
+ * Build a linear skb around the received buffer.
+ * We are guaranteed there is enough room at the end of the data buffer to
+ * accomodate the shared info area of the skb.
  */
-static void __hot contig_fd_to_skb(const struct dpa_priv_s *priv,
-	const struct qm_fd *fd, struct sk_buff *skb, int *use_gro)
+static struct sk_buff *__hot contig_fd_to_skb(const struct dpa_priv_s *priv,
+	const struct qm_fd *fd, int *use_gro)
 {
-	unsigned int copy_size = DPA_COPIED_HEADERS_SIZE;
 	dma_addr_t addr = qm_fd_addr(fd);
 	void *vaddr;
-	struct page *page;
-	int frag_offset, page_offset;
 	struct dpa_bp *dpa_bp = priv->dpa_bp;
-	unsigned char *tailptr;
 	const t_FmPrsResult *parse_results;
+	struct sk_buff *skb = NULL;
 
 	vaddr = phys_to_virt(addr);
 
@@ -333,53 +282,38 @@ static void __hot contig_fd_to_skb(const struct dpa_priv_s *priv,
 		dpa_ptp_store_rxstamp(priv, skb, vaddr);
 #endif
 
-	/* Peek at the parse results for csum validation and headers size */
+	/* Build the skb and adjust data and tail pointers */
+	skb = build_skb(vaddr, dpa_bp->size + DPA_SKB_TAILROOM);
+	if (unlikely(!skb))
+		return NULL;
+
+	/* Make sure forwarded skbs will have enough space on Tx,
+	 * if extra headers are added.
+	 */
+	skb_reserve(skb, priv->tx_headroom + dpa_get_rx_extra_headroom());
+	skb_put(skb, dpa_fd_length(fd));
+
+	/* Peek at the parse results for csum validation */
 	parse_results = (const t_FmPrsResult *)(vaddr + DPA_RX_PRIV_DATA_SIZE);
-	_dpa_process_parse_results(parse_results, fd, skb, use_gro, &copy_size);
+	_dpa_process_parse_results(parse_results, fd, skb, use_gro);
 
 #ifdef CONFIG_FSL_DPAA_TS
 	if (priv->ts_rx_en)
 		dpa_get_ts(priv, RX, skb_hwtstamps(skb), vaddr);
 #endif /* CONFIG_FSL_DPAA_TS */
 
-	tailptr = skb_put(skb, copy_size);
-
-	/* Copy (at least) the headers in the linear portion */
-	memcpy(tailptr, vaddr + dpa_fd_offset(fd), copy_size);
-
-	/*
-	 * If frame is longer than the amount we copy in the linear
-	 * buffer, add the page as fragment,
-	 * otherwise recycle the page
-	 */
-	page = pfn_to_page(addr >> PAGE_SHIFT);
-
-	if (copy_size < dpa_fd_length(fd)) {
-		/* add the page as a fragment in the skb */
-		page_offset = (unsigned long)vaddr & (PAGE_SIZE - 1);
-		frag_offset = page_offset + dpa_fd_offset(fd) + copy_size;
-		skb_add_rx_frag(skb, 0, page, frag_offset,
-		                dpa_fd_length(fd) - copy_size,
-		                /* TODO kernel 3.8 fixup; we might want
-		                 * to better account for the truesize */
-				dpa_fd_length(fd) - copy_size);
-	} else {
-		/* recycle the page */
-		dpa_bp_add_page(dpa_bp, (unsigned long)vaddr);
-	}
+	return skb;
 }
 
 
 /*
- * Move the first bytes of the frame to the skb linear buffer to
- * provide the networking stack the headers it requires in the linear buffer,
- * and add the rest of the frame as skb fragments.
+ * Build an skb with the data of the first S/G entry in the linear portion and
+ * the rest of the frame as skb fragments.
  *
  * The page holding the S/G Table is recycled here.
  */
-static void __hot sg_fd_to_skb(const struct dpa_priv_s *priv,
-			       const struct qm_fd *fd, struct sk_buff *skb,
-			       int *use_gro)
+static struct sk_buff *__hot sg_fd_to_skb(const struct dpa_priv_s *priv,
+			       const struct qm_fd *fd, int *use_gro)
 {
 	const struct qm_sg_entry *sgt;
 	dma_addr_t addr = qm_fd_addr(fd);
@@ -390,31 +324,21 @@ static void __hot sg_fd_to_skb(const struct dpa_priv_s *priv,
 	int frag_offset, frag_len;
 	int page_offset;
 	int i;
-	unsigned int copy_size = DPA_COPIED_HEADERS_SIZE;
 	const t_FmPrsResult *parse_results;
+	struct sk_buff *skb = NULL;
 
 	vaddr = phys_to_virt(addr);
 #ifdef CONFIG_FSL_DPAA_1588
 	if (priv->tsu && priv->tsu->valid && priv->tsu->hwts_rx_en_ioctl)
 		dpa_ptp_store_rxstamp(priv, skb, vaddr);
 #endif
-	/*
-	 * In the case of a SG frame, FMan stores the Internal Context
-	 * in the buffer containing the sgt.
-	 */
-	parse_results = (const t_FmPrsResult *)(vaddr + DPA_RX_PRIV_DATA_SIZE);
-	/* Inspect the parse results before anything else. */
-	_dpa_process_parse_results(parse_results, fd, skb, use_gro, &copy_size);
 
 #ifdef CONFIG_FSL_DPAA_TS
 	if (priv->ts_rx_en)
 		dpa_get_ts(priv, RX, skb_hwtstamps(skb), vaddr);
 #endif /* CONFIG_FSL_DPAA_TS */
 
-	/*
-	 * Iterate through the SGT entries and add the data buffers as
-	 * skb fragments
-	 */
+	/* Iterate through the SGT entries and add data buffers to the skb */
 	sgt = vaddr + dpa_fd_offset(fd);
 	for (i = 0; i < DPA_SGT_MAX_ENTRIES; i++) {
 		/* Extension bit is not supported */
@@ -427,39 +351,48 @@ static void __hot sg_fd_to_skb(const struct dpa_priv_s *priv,
 		sg_vaddr = phys_to_virt(sg_addr);
 
 		dpa_bp_removed_one_page(dpa_bp, sg_addr);
-		page = pfn_to_page(sg_addr >> PAGE_SHIFT);
-
-		/*
-		 * Padding at the beginning of the page
-		 * (offset in page from where BMan buffer begins)
-		 */
-		page_offset = (unsigned long)sg_vaddr & (PAGE_SIZE - 1);
 
 		if (i == 0) {
-			/* This is the first fragment */
-			/* Move the network headers in the skb linear portion */
-			memcpy(skb_put(skb, copy_size),
-				sg_vaddr + sgt[i].offset,
-				copy_size);
-
-			/* Adjust offset/length for the remaining data */
-			frag_offset = sgt[i].offset + page_offset + copy_size;
-			frag_len = sgt[i].length - copy_size;
+			/* This is the first S/G entry, so build the skb
+			 * around its data buffer
+			 */
+			skb = build_skb(sg_vaddr,
+					dpa_bp->size + DPA_SKB_TAILROOM);
+			if (unlikely(!skb))
+				return NULL;
+
+			/* In the case of a SG frame, FMan stores the Internal
+			 * Context in the buffer containing the sgt.
+			 * Inspect the parse results before anything else.
+			 */
+			parse_results = (const t_FmPrsResult *)(vaddr +
+						DPA_RX_PRIV_DATA_SIZE);
+			_dpa_process_parse_results(parse_results, fd, skb,
+						   use_gro);
+
+			/* Make sure forwarded skbs will have enough space
+			 * on Tx, if extra headers are added.
+			 */
+			skb_reserve(skb, priv->tx_headroom +
+				dpa_get_rx_extra_headroom());
+			skb_put(skb, sgt[i].length);
 		} else {
 			/*
-			 * Not the first fragment; all data from buferr will
-			 * be added in an skb fragment
+			 * Not the first S/G entry; all data from buffer will
+			 * be added in an skb fragment; fragment index is offset
+			 * by one since first S/G entry was incorporated in the
+			 * linear part of the skb.
 			 */
+			page = pfn_to_page(sg_addr >> PAGE_SHIFT);
+			page_offset = (unsigned long)sg_vaddr & (PAGE_SIZE - 1);
 			frag_offset = sgt[i].offset + page_offset;
 			frag_len = sgt[i].length;
+			/* TODO kernel 3.8 fixup; we might want to account for
+			 * the true-truesize.
+			 */
+			skb_add_rx_frag(skb, i - 1, page, frag_offset, frag_len,
+					frag_len);
 		}
-		/*
-		 * Add data buffer to the skb
-		 *
-		 * TODO kernel 3.8 fixup; we might want to account for
-		 * the true-truesize.
-		 */
-		skb_add_rx_frag(skb, i, page, frag_offset, frag_len, frag_len);
 
 		if (sgt[i].final)
 			break;
@@ -469,6 +402,7 @@ static void __hot sg_fd_to_skb(const struct dpa_priv_s *priv,
 	dpa_bp = dpa_bpid2pool(fd->bpid);
 	BUG_ON(IS_ERR(dpa_bp));
 	dpa_bp_add_page(dpa_bp, (unsigned long)vaddr);
+	return skb;
 }
 
 void __hot _dpa_rx(struct net_device *net_dev,
@@ -495,41 +429,20 @@ void __hot _dpa_rx(struct net_device *net_dev,
 	}
 
 	dpa_bp = dpa_bpid2pool(fd->bpid);
-	skb = dpa_list_get_skb(percpu_priv);
-
-	if (unlikely(skb == NULL)) {
-		/* List is empty, so allocate a new skb */
-		skb = dev_alloc_skb(priv->tx_headroom +
-			dpa_get_rx_extra_headroom() + DPA_COPIED_HEADERS_SIZE);
-		if (unlikely(skb == NULL)) {
-			if (netif_msg_rx_err(priv) && net_ratelimit())
-				netdev_err(net_dev,
-						"Could not alloc skb\n");
-			percpu_stats->rx_dropped++;
-			goto _release_frame;
-		}
-	}
-
-	/* TODO We might want to do some prefetches here (skb, shinfo, data) */
-
-	/*
-	 * Make sure forwarded skbs will have enough space on Tx,
-	 * if extra headers are added.
-	 */
-	skb_reserve(skb, priv->tx_headroom + dpa_get_rx_extra_headroom());
-
 	dpa_bp_removed_one_page(dpa_bp, addr);
 
 	/* prefetch the first 64 bytes of the frame or the SGT start */
 	prefetch(phys_to_virt(addr) + dpa_fd_offset(fd));
 
 	if (likely(fd->format == qm_fd_contig))
-		contig_fd_to_skb(priv, fd, skb, &use_gro);
+		skb = contig_fd_to_skb(priv, fd, &use_gro);
 	else if (fd->format == qm_fd_sg)
-		sg_fd_to_skb(priv, fd, skb, &use_gro);
+		skb = sg_fd_to_skb(priv, fd, &use_gro);
 	else
 		/* The only FD types that we may receive are contig and S/G */
 		BUG();
+	if (unlikely(!skb))
+		goto _release_frame;
 
 	skb->protocol = eth_type_trans(skb, net_dev);