diff options
author | Ioana Radulescu <ruxandra.radulescu@freescale.com> | 2013-04-30 19:48:16 (GMT) |
---|---|---|
committer | Fleming Andrew-AFLEMING <AFLEMING@freescale.com> | 2013-05-17 19:52:16 (GMT) |
commit | ad1bdc90739189c25881f12e21940f84398b8ef0 (patch) | |
tree | d7953ca1e8426f43f24510b6fbc381ab33a64d3a | |
parent | 6a2b97552c5c9b612a9ef37fb2f181005b1919aa (diff) | |
download | linux-fsl-qoriq-ad1bdc90739189c25881f12e21940f84398b8ef0.tar.xz |
dpaa_eth: Use build_skb() on S/G path
Until now skbs were allocated on Rx (or taken from a preallocated
pool of skbs) and the frame headers copied into the skb linear part.
Eliminate the need to copy data by building the skb around the data
buffer. For S/G frames, use the first S/G entry to build the skb
linear part and add the other entries as fragments.
A lot of code is removed due to this change, including identifying
total header size while looking at the parse results, header copying,
seeding/extracting skbs to/from the skb list.
Change-Id: I6e461a2d3dced2abcaed7fa6eaf2606ba1bb6e04
Signed-off-by: Ioana Radulescu <ruxandra.radulescu@freescale.com>
Reviewed-on: http://git.am.freescale.net:8181/2412
Reviewed-by: Fleming Andrew-AFLEMING <AFLEMING@freescale.com>
Tested-by: Fleming Andrew-AFLEMING <AFLEMING@freescale.com>
-rw-r--r-- | drivers/net/ethernet/freescale/dpa/dpaa_eth.c | 49 | ||||
-rw-r--r-- | drivers/net/ethernet/freescale/dpa/dpaa_eth.h | 34 | ||||
-rw-r--r-- | drivers/net/ethernet/freescale/dpa/dpaa_eth_sg.c | 217 |
3 files changed, 76 insertions, 224 deletions
diff --git a/drivers/net/ethernet/freescale/dpa/dpaa_eth.c b/drivers/net/ethernet/freescale/dpa/dpaa_eth.c index cffef9e..4080d3d 100644 --- a/drivers/net/ethernet/freescale/dpa/dpaa_eth.c +++ b/drivers/net/ethernet/freescale/dpa/dpaa_eth.c @@ -413,15 +413,6 @@ static void dpaa_eth_refill_bpools(struct dpa_priv_s *priv, while (count < CONFIG_FSL_DPAA_ETH_MAX_BUF_COUNT) count += _dpa_bp_add_8_pages(dpa_bp); *countptr = count; - - /* Add skbs to the percpu skb list, reuse var count */ - count = percpu_priv->skb_count; - if (unlikely(count < DEFAULT_SKB_COUNT / 4)) { - int skb_size = priv->tx_headroom + dpa_get_rx_extra_headroom() + - DPA_COPIED_HEADERS_SIZE; - dpa_list_add_skbs(percpu_priv, DEFAULT_SKB_COUNT - count, - skb_size); - } #endif } @@ -1403,10 +1394,8 @@ static void _dpa_tx_error(struct net_device *net_dev, * headers' length. */ void __hot _dpa_process_parse_results(const t_FmPrsResult *parse_results, - const struct qm_fd *fd, - struct sk_buff *skb, - int *use_gro, - unsigned int *hdr_size __maybe_unused) + const struct qm_fd *fd, + struct sk_buff *skb, int *use_gro) { if (fd->status & FM_FD_STAT_L4CV) { /* @@ -1428,13 +1417,6 @@ void __hot _dpa_process_parse_results(const t_FmPrsResult *parse_results, if (!fm_l4_frame_is_tcp(parse_results)) *use_gro = 0; -#ifdef CONFIG_FSL_DPAA_ETH_SG_SUPPORT - /* - * If the L4 HXS Parser has successfully run, we can reduce the - * number of bytes we'll memcopy into skb->data. - */ - *hdr_size = parse_results->nxthdr_off; -#endif return; } @@ -1444,18 +1426,6 @@ void __hot _dpa_process_parse_results(const t_FmPrsResult *parse_results, * checksum zero or an L4 proto other than TCP/UDP */ skb->ip_summed = CHECKSUM_NONE; -#ifdef CONFIG_FSL_DPAA_ETH_SG_SUPPORT - /* - * Even if checksum was not verified, it's still possible L4 parser - * has run, in which case we know the headers size. - * Otherwise we fall back to a safe default. - */ - if (fm_l4_hxs_has_run(parse_results)) - *hdr_size = parse_results->nxthdr_off; - else - *hdr_size = min((ssize_t)DPA_COPIED_HEADERS_SIZE, - dpa_fd_length(fd)); -#endif /* Bypass GRO for unknown traffic or if no PCDs are applied */ *use_gro = 0; @@ -1475,7 +1445,6 @@ void __hot _dpa_rx(struct net_device *net_dev, u32 fd_status = fd->status; unsigned int skb_len; t_FmPrsResult *parse_result; - unsigned int hdr_size_unused; int use_gro = net_dev->features & NETIF_F_GRO; skbh = (struct sk_buff **)phys_to_virt(addr); @@ -1531,8 +1500,7 @@ void __hot _dpa_rx(struct net_device *net_dev, /* Validate the skb csum and figure out whether GRO is appropriate */ parse_result = (t_FmPrsResult *)((u8 *)skbh + DPA_RX_PRIV_DATA_SIZE); - _dpa_process_parse_results(parse_result, fd, skb, &use_gro, - &hdr_size_unused); + _dpa_process_parse_results(parse_result, fd, skb, &use_gro); #ifdef CONFIG_FSL_DPAA_TS if (priv->ts_rx_en) @@ -3869,17 +3837,6 @@ static int dpa_private_netdev_init(struct device_node *dpa_node, percpu_priv = per_cpu_ptr(priv->percpu_priv, i); percpu_priv->net_dev = net_dev; -#ifdef CONFIG_FSL_DPAA_ETH_SG_SUPPORT - /* init the percpu list and add some skbs */ - skb_queue_head_init(&percpu_priv->skb_list); - - /* Skbs must accomodate the headroom and enough space - * for the frame headers. - */ - dpa_list_add_skbs(percpu_priv, DEFAULT_SKB_COUNT, - priv->tx_headroom + DPA_COPIED_HEADERS_SIZE + - dpa_get_rx_extra_headroom()); -#endif netif_napi_add(net_dev, &percpu_priv->napi, dpaa_eth_poll, DPA_NAPI_WEIGHT); } diff --git a/drivers/net/ethernet/freescale/dpa/dpaa_eth.h b/drivers/net/ethernet/freescale/dpa/dpaa_eth.h index 22d6546..fbea46b 100644 --- a/drivers/net/ethernet/freescale/dpa/dpaa_eth.h +++ b/drivers/net/ethernet/freescale/dpa/dpaa_eth.h @@ -74,10 +74,12 @@ #ifdef CONFIG_FSL_DPAA_ETH_SG_SUPPORT -/* We may want this value configurable. Must be <= PAGE_SIZE - * A lower value may help with recycling rates, at least on forwarding +/* We may want this value configurable. Must be <= PAGE_SIZE minus a reserved + * area where skb shared info can reside + * A lower value may help with recycling rates, at least on forwarding. */ -#define dpa_bp_size(buffer_layout) PAGE_SIZE +#define DPA_SKB_TAILROOM SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +#define dpa_bp_size(buffer_layout) (PAGE_SIZE - DPA_SKB_TAILROOM) #else /* Default buffer size is based on L2 MAX_FRM value, minus the FCS which @@ -165,16 +167,6 @@ void fsl_dpaa_eth_set_hooks(struct dpaa_eth_hooks_s *hooks); #define DPA_SGT_MAX_ENTRIES 16 /* maximum number of entries in SG Table */ -#ifdef CONFIG_FSL_DPAA_ETH_SG_SUPPORT -#define DEFAULT_SKB_COUNT 64 /* maximum number of SKBs in each percpu list */ -/* - * Default amount data to be copied from the beginning of a frame into the - * linear part of the skb, in case we aren't using the hardware parser. - */ -#define DPA_COPIED_HEADERS_SIZE 128 - -#endif /* CONFIG_FSL_DPAA_ETH_SG_SUPPORT */ - /* * Largest value that the FQD's OAL field can hold. * This is DPAA-1.x specific. @@ -329,12 +321,6 @@ struct dpa_percpu_priv_s { int *dpa_bp_count; struct dpa_bp *dpa_bp; struct napi_struct napi; -#ifdef CONFIG_FSL_DPAA_ETH_SG_SUPPORT - /* a list of preallocated SKBs for this CPU */ - struct sk_buff_head skb_list; - /* current number of skbs in the CPU's list */ - int skb_count; -#endif u64 in_interrupt; u64 tx_returned; u64 tx_confirm; @@ -428,17 +414,13 @@ struct sk_buff *_dpa_cleanup_tx_fd(const struct dpa_priv_s *priv, const struct qm_fd *fd); void __hot _dpa_process_parse_results(const t_FmPrsResult *parse_results, - const struct qm_fd *fd, - struct sk_buff *skb, - int *use_gro, - unsigned int *hdr_size __maybe_unused); + const struct qm_fd *fd, + struct sk_buff *skb, + int *use_gro); #ifdef CONFIG_FSL_DPAA_ETH_SG_SUPPORT void dpa_bp_add_8_pages(const struct dpa_bp *dpa_bp, int cpu_id); int _dpa_bp_add_8_pages(const struct dpa_bp *dpa_bp); - -void dpa_list_add_skbs(struct dpa_percpu_priv_s *cpu_priv, int count, - int skb_size); #endif /* diff --git a/drivers/net/ethernet/freescale/dpa/dpaa_eth_sg.c b/drivers/net/ethernet/freescale/dpa/dpaa_eth_sg.c index 8369374..a624e9a 100644 --- a/drivers/net/ethernet/freescale/dpa/dpaa_eth_sg.c +++ b/drivers/net/ethernet/freescale/dpa/dpaa_eth_sg.c @@ -149,52 +149,6 @@ void dpa_bp_add_8_pages(const struct dpa_bp *dpa_bp, int cpu) *count_ptr += _dpa_bp_add_8_pages(dpa_bp); } -void dpa_list_add_skb(struct dpa_percpu_priv_s *cpu_priv, - struct sk_buff *new_skb) -{ - struct sk_buff_head *list_ptr; - - if (cpu_priv->skb_count > DEFAULT_SKB_COUNT) { - dev_kfree_skb(new_skb); - return; - } - - list_ptr = &cpu_priv->skb_list; - skb_queue_head(list_ptr, new_skb); - - cpu_priv->skb_count += 1; -} - -static struct sk_buff *dpa_list_get_skb(struct dpa_percpu_priv_s *cpu_priv) -{ - struct sk_buff_head *list_ptr; - struct sk_buff *new_skb; - - list_ptr = &cpu_priv->skb_list; - - new_skb = skb_dequeue(list_ptr); - if (new_skb) - cpu_priv->skb_count -= 1; - - return new_skb; -} - -void dpa_list_add_skbs(struct dpa_percpu_priv_s *cpu_priv, int count, int size) -{ - struct sk_buff *new_skb; - int i; - - for (i = 0; i < count; i++) { - new_skb = dev_alloc_skb(size); - if (unlikely(!new_skb)) { - pr_err("dev_alloc_skb() failed\n"); - break; - } - - dpa_list_add_skb(cpu_priv, new_skb); - } -} - void dpa_make_private_pool(struct dpa_bp *dpa_bp) { int i; @@ -307,23 +261,18 @@ struct sk_buff *_dpa_cleanup_tx_fd(const struct dpa_priv_s *priv, } /* - * Move the first DPA_COPIED_HEADERS_SIZE bytes to the skb linear buffer to - * provide the networking stack the headers it requires in the linear buffer. - * - * If the entire frame fits in the skb linear buffer, the page holding the - * received data is recycled as it is no longer required. + * Build a linear skb around the received buffer. + * We are guaranteed there is enough room at the end of the data buffer to + * accomodate the shared info area of the skb. */ -static void __hot contig_fd_to_skb(const struct dpa_priv_s *priv, - const struct qm_fd *fd, struct sk_buff *skb, int *use_gro) +static struct sk_buff *__hot contig_fd_to_skb(const struct dpa_priv_s *priv, + const struct qm_fd *fd, int *use_gro) { - unsigned int copy_size = DPA_COPIED_HEADERS_SIZE; dma_addr_t addr = qm_fd_addr(fd); void *vaddr; - struct page *page; - int frag_offset, page_offset; struct dpa_bp *dpa_bp = priv->dpa_bp; - unsigned char *tailptr; const t_FmPrsResult *parse_results; + struct sk_buff *skb = NULL; vaddr = phys_to_virt(addr); @@ -333,53 +282,38 @@ static void __hot contig_fd_to_skb(const struct dpa_priv_s *priv, dpa_ptp_store_rxstamp(priv, skb, vaddr); #endif - /* Peek at the parse results for csum validation and headers size */ + /* Build the skb and adjust data and tail pointers */ + skb = build_skb(vaddr, dpa_bp->size + DPA_SKB_TAILROOM); + if (unlikely(!skb)) + return NULL; + + /* Make sure forwarded skbs will have enough space on Tx, + * if extra headers are added. + */ + skb_reserve(skb, priv->tx_headroom + dpa_get_rx_extra_headroom()); + skb_put(skb, dpa_fd_length(fd)); + + /* Peek at the parse results for csum validation */ parse_results = (const t_FmPrsResult *)(vaddr + DPA_RX_PRIV_DATA_SIZE); - _dpa_process_parse_results(parse_results, fd, skb, use_gro, ©_size); + _dpa_process_parse_results(parse_results, fd, skb, use_gro); #ifdef CONFIG_FSL_DPAA_TS if (priv->ts_rx_en) dpa_get_ts(priv, RX, skb_hwtstamps(skb), vaddr); #endif /* CONFIG_FSL_DPAA_TS */ - tailptr = skb_put(skb, copy_size); - - /* Copy (at least) the headers in the linear portion */ - memcpy(tailptr, vaddr + dpa_fd_offset(fd), copy_size); - - /* - * If frame is longer than the amount we copy in the linear - * buffer, add the page as fragment, - * otherwise recycle the page - */ - page = pfn_to_page(addr >> PAGE_SHIFT); - - if (copy_size < dpa_fd_length(fd)) { - /* add the page as a fragment in the skb */ - page_offset = (unsigned long)vaddr & (PAGE_SIZE - 1); - frag_offset = page_offset + dpa_fd_offset(fd) + copy_size; - skb_add_rx_frag(skb, 0, page, frag_offset, - dpa_fd_length(fd) - copy_size, - /* TODO kernel 3.8 fixup; we might want - * to better account for the truesize */ - dpa_fd_length(fd) - copy_size); - } else { - /* recycle the page */ - dpa_bp_add_page(dpa_bp, (unsigned long)vaddr); - } + return skb; } /* - * Move the first bytes of the frame to the skb linear buffer to - * provide the networking stack the headers it requires in the linear buffer, - * and add the rest of the frame as skb fragments. + * Build an skb with the data of the first S/G entry in the linear portion and + * the rest of the frame as skb fragments. * * The page holding the S/G Table is recycled here. */ -static void __hot sg_fd_to_skb(const struct dpa_priv_s *priv, - const struct qm_fd *fd, struct sk_buff *skb, - int *use_gro) +static struct sk_buff *__hot sg_fd_to_skb(const struct dpa_priv_s *priv, + const struct qm_fd *fd, int *use_gro) { const struct qm_sg_entry *sgt; dma_addr_t addr = qm_fd_addr(fd); @@ -390,31 +324,21 @@ static void __hot sg_fd_to_skb(const struct dpa_priv_s *priv, int frag_offset, frag_len; int page_offset; int i; - unsigned int copy_size = DPA_COPIED_HEADERS_SIZE; const t_FmPrsResult *parse_results; + struct sk_buff *skb = NULL; vaddr = phys_to_virt(addr); #ifdef CONFIG_FSL_DPAA_1588 if (priv->tsu && priv->tsu->valid && priv->tsu->hwts_rx_en_ioctl) dpa_ptp_store_rxstamp(priv, skb, vaddr); #endif - /* - * In the case of a SG frame, FMan stores the Internal Context - * in the buffer containing the sgt. - */ - parse_results = (const t_FmPrsResult *)(vaddr + DPA_RX_PRIV_DATA_SIZE); - /* Inspect the parse results before anything else. */ - _dpa_process_parse_results(parse_results, fd, skb, use_gro, ©_size); #ifdef CONFIG_FSL_DPAA_TS if (priv->ts_rx_en) dpa_get_ts(priv, RX, skb_hwtstamps(skb), vaddr); #endif /* CONFIG_FSL_DPAA_TS */ - /* - * Iterate through the SGT entries and add the data buffers as - * skb fragments - */ + /* Iterate through the SGT entries and add data buffers to the skb */ sgt = vaddr + dpa_fd_offset(fd); for (i = 0; i < DPA_SGT_MAX_ENTRIES; i++) { /* Extension bit is not supported */ @@ -427,39 +351,48 @@ static void __hot sg_fd_to_skb(const struct dpa_priv_s *priv, sg_vaddr = phys_to_virt(sg_addr); dpa_bp_removed_one_page(dpa_bp, sg_addr); - page = pfn_to_page(sg_addr >> PAGE_SHIFT); - - /* - * Padding at the beginning of the page - * (offset in page from where BMan buffer begins) - */ - page_offset = (unsigned long)sg_vaddr & (PAGE_SIZE - 1); if (i == 0) { - /* This is the first fragment */ - /* Move the network headers in the skb linear portion */ - memcpy(skb_put(skb, copy_size), - sg_vaddr + sgt[i].offset, - copy_size); - - /* Adjust offset/length for the remaining data */ - frag_offset = sgt[i].offset + page_offset + copy_size; - frag_len = sgt[i].length - copy_size; + /* This is the first S/G entry, so build the skb + * around its data buffer + */ + skb = build_skb(sg_vaddr, + dpa_bp->size + DPA_SKB_TAILROOM); + if (unlikely(!skb)) + return NULL; + + /* In the case of a SG frame, FMan stores the Internal + * Context in the buffer containing the sgt. + * Inspect the parse results before anything else. + */ + parse_results = (const t_FmPrsResult *)(vaddr + + DPA_RX_PRIV_DATA_SIZE); + _dpa_process_parse_results(parse_results, fd, skb, + use_gro); + + /* Make sure forwarded skbs will have enough space + * on Tx, if extra headers are added. + */ + skb_reserve(skb, priv->tx_headroom + + dpa_get_rx_extra_headroom()); + skb_put(skb, sgt[i].length); } else { /* - * Not the first fragment; all data from buferr will - * be added in an skb fragment + * Not the first S/G entry; all data from buffer will + * be added in an skb fragment; fragment index is offset + * by one since first S/G entry was incorporated in the + * linear part of the skb. */ + page = pfn_to_page(sg_addr >> PAGE_SHIFT); + page_offset = (unsigned long)sg_vaddr & (PAGE_SIZE - 1); frag_offset = sgt[i].offset + page_offset; frag_len = sgt[i].length; + /* TODO kernel 3.8 fixup; we might want to account for + * the true-truesize. + */ + skb_add_rx_frag(skb, i - 1, page, frag_offset, frag_len, + frag_len); } - /* - * Add data buffer to the skb - * - * TODO kernel 3.8 fixup; we might want to account for - * the true-truesize. - */ - skb_add_rx_frag(skb, i, page, frag_offset, frag_len, frag_len); if (sgt[i].final) break; @@ -469,6 +402,7 @@ static void __hot sg_fd_to_skb(const struct dpa_priv_s *priv, dpa_bp = dpa_bpid2pool(fd->bpid); BUG_ON(IS_ERR(dpa_bp)); dpa_bp_add_page(dpa_bp, (unsigned long)vaddr); + return skb; } void __hot _dpa_rx(struct net_device *net_dev, @@ -495,41 +429,20 @@ void __hot _dpa_rx(struct net_device *net_dev, } dpa_bp = dpa_bpid2pool(fd->bpid); - skb = dpa_list_get_skb(percpu_priv); - - if (unlikely(skb == NULL)) { - /* List is empty, so allocate a new skb */ - skb = dev_alloc_skb(priv->tx_headroom + - dpa_get_rx_extra_headroom() + DPA_COPIED_HEADERS_SIZE); - if (unlikely(skb == NULL)) { - if (netif_msg_rx_err(priv) && net_ratelimit()) - netdev_err(net_dev, - "Could not alloc skb\n"); - percpu_stats->rx_dropped++; - goto _release_frame; - } - } - - /* TODO We might want to do some prefetches here (skb, shinfo, data) */ - - /* - * Make sure forwarded skbs will have enough space on Tx, - * if extra headers are added. - */ - skb_reserve(skb, priv->tx_headroom + dpa_get_rx_extra_headroom()); - dpa_bp_removed_one_page(dpa_bp, addr); /* prefetch the first 64 bytes of the frame or the SGT start */ prefetch(phys_to_virt(addr) + dpa_fd_offset(fd)); if (likely(fd->format == qm_fd_contig)) - contig_fd_to_skb(priv, fd, skb, &use_gro); + skb = contig_fd_to_skb(priv, fd, &use_gro); else if (fd->format == qm_fd_sg) - sg_fd_to_skb(priv, fd, skb, &use_gro); + skb = sg_fd_to_skb(priv, fd, &use_gro); else /* The only FD types that we may receive are contig and S/G */ BUG(); + if (unlikely(!skb)) + goto _release_frame; skb->protocol = eth_type_trans(skb, net_dev); |