diff options
Diffstat (limited to 'net/core/dev.c')
-rw-r--r-- | net/core/dev.c | 238 |
1 files changed, 70 insertions, 168 deletions
diff --git a/net/core/dev.c b/net/core/dev.c index 382df6c..b715a55 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -170,25 +170,6 @@ static DEFINE_SPINLOCK(ptype_lock); static struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly; static struct list_head ptype_all __read_mostly; /* Taps */ -#ifdef CONFIG_NET_DMA -struct net_dma { - struct dma_client client; - spinlock_t lock; - cpumask_t channel_mask; - struct dma_chan **channels; -}; - -static enum dma_state_client -netdev_dma_event(struct dma_client *client, struct dma_chan *chan, - enum dma_state state); - -static struct net_dma net_dma = { - .client = { - .event_callback = netdev_dma_event, - }, -}; -#endif - /* * The @dev_base_head list is protected by @dev_base_lock and the rtnl * semaphore. @@ -1107,6 +1088,11 @@ int dev_open(struct net_device *dev) dev->flags |= IFF_UP; /* + * Enable NET_DMA + */ + dmaengine_get(); + + /* * Initialize multicasting status */ dev_set_rx_mode(dev); @@ -1183,6 +1169,11 @@ int dev_close(struct net_device *dev) */ call_netdevice_notifiers(NETDEV_DOWN, dev); + /* + * Shutdown NET_DMA + */ + dmaengine_put(); + return 0; } @@ -2387,7 +2378,7 @@ void napi_gro_flush(struct napi_struct *napi) } EXPORT_SYMBOL(napi_gro_flush); -static int __napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb) +int dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb) { struct sk_buff **pp = NULL; struct packet_type *ptype; @@ -2417,11 +2408,14 @@ static int __napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb) for (p = napi->gro_list; p; p = p->next) { count++; - NAPI_GRO_CB(p)->same_flow = - p->mac_len == mac_len && - !memcmp(skb_mac_header(p), skb_mac_header(skb), - mac_len); - NAPI_GRO_CB(p)->flush = 0; + + if (!NAPI_GRO_CB(p)->same_flow) + continue; + + if (p->mac_len != mac_len || + memcmp(skb_mac_header(p), skb_mac_header(skb), + mac_len)) + NAPI_GRO_CB(p)->same_flow = 0; } pp = ptype->gro_receive(&napi->gro_list, skb); @@ -2463,6 +2457,19 @@ ok: normal: return -1; } +EXPORT_SYMBOL(dev_gro_receive); + +static int __napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb) +{ + struct sk_buff *p; + + for (p = napi->gro_list; p; p = p->next) { + NAPI_GRO_CB(p)->same_flow = 1; + NAPI_GRO_CB(p)->flush = 0; + } + + return dev_gro_receive(napi, skb); +} int napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb) { @@ -2479,11 +2486,26 @@ int napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb) } EXPORT_SYMBOL(napi_gro_receive); -int napi_gro_frags(struct napi_struct *napi, struct napi_gro_fraginfo *info) +void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb) +{ + skb_shinfo(skb)->nr_frags = 0; + + skb->len -= skb->data_len; + skb->truesize -= skb->data_len; + skb->data_len = 0; + + __skb_pull(skb, skb_headlen(skb)); + skb_reserve(skb, NET_IP_ALIGN - skb_headroom(skb)); + + napi->skb = skb; +} +EXPORT_SYMBOL(napi_reuse_skb); + +struct sk_buff *napi_fraginfo_skb(struct napi_struct *napi, + struct napi_gro_fraginfo *info) { struct net_device *dev = napi->dev; struct sk_buff *skb = napi->skb; - int err = NET_RX_DROP; napi->skb = NULL; @@ -2503,16 +2525,31 @@ int napi_gro_frags(struct napi_struct *napi, struct napi_gro_fraginfo *info) skb->len += info->len; skb->truesize += info->len; - if (!pskb_may_pull(skb, ETH_HLEN)) - goto reuse; - - err = NET_RX_SUCCESS; + if (!pskb_may_pull(skb, ETH_HLEN)) { + napi_reuse_skb(napi, skb); + goto out; + } skb->protocol = eth_type_trans(skb, dev); skb->ip_summed = info->ip_summed; skb->csum = info->csum; +out: + return skb; +} +EXPORT_SYMBOL(napi_fraginfo_skb); + +int napi_gro_frags(struct napi_struct *napi, struct napi_gro_fraginfo *info) +{ + struct sk_buff *skb = napi_fraginfo_skb(napi, info); + int err = NET_RX_DROP; + + if (!skb) + goto out; + + err = NET_RX_SUCCESS; + switch (__napi_gro_receive(napi, skb)) { case -1: return netif_receive_skb(skb); @@ -2521,17 +2558,7 @@ int napi_gro_frags(struct napi_struct *napi, struct napi_gro_fraginfo *info) goto out; } -reuse: - skb_shinfo(skb)->nr_frags = 0; - - skb->len -= skb->data_len; - skb->truesize -= skb->data_len; - skb->data_len = 0; - - __skb_pull(skb, skb_headlen(skb)); - skb_reserve(skb, NET_IP_ALIGN - skb_headroom(skb)); - - napi->skb = skb; + napi_reuse_skb(napi, skb); out: return err; @@ -2718,14 +2745,7 @@ out: * There may not be any more sk_buffs coming right now, so push * any pending DMA copies to hardware */ - if (!cpus_empty(net_dma.channel_mask)) { - int chan_idx; - for_each_cpu_mask_nr(chan_idx, net_dma.channel_mask) { - struct dma_chan *chan = net_dma.channels[chan_idx]; - if (chan) - dma_async_memcpy_issue_pending(chan); - } - } + dma_issue_pending_all(); #endif return; @@ -4916,122 +4936,6 @@ static int dev_cpu_callback(struct notifier_block *nfb, return NOTIFY_OK; } -#ifdef CONFIG_NET_DMA -/** - * net_dma_rebalance - try to maintain one DMA channel per CPU - * @net_dma: DMA client and associated data (lock, channels, channel_mask) - * - * This is called when the number of channels allocated to the net_dma client - * changes. The net_dma client tries to have one DMA channel per CPU. - */ - -static void net_dma_rebalance(struct net_dma *net_dma) -{ - unsigned int cpu, i, n, chan_idx; - struct dma_chan *chan; - - if (cpus_empty(net_dma->channel_mask)) { - for_each_online_cpu(cpu) - rcu_assign_pointer(per_cpu(softnet_data, cpu).net_dma, NULL); - return; - } - - i = 0; - cpu = first_cpu(cpu_online_map); - - for_each_cpu_mask_nr(chan_idx, net_dma->channel_mask) { - chan = net_dma->channels[chan_idx]; - - n = ((num_online_cpus() / cpus_weight(net_dma->channel_mask)) - + (i < (num_online_cpus() % - cpus_weight(net_dma->channel_mask)) ? 1 : 0)); - - while(n) { - per_cpu(softnet_data, cpu).net_dma = chan; - cpu = next_cpu(cpu, cpu_online_map); - n--; - } - i++; - } -} - -/** - * netdev_dma_event - event callback for the net_dma_client - * @client: should always be net_dma_client - * @chan: DMA channel for the event - * @state: DMA state to be handled - */ -static enum dma_state_client -netdev_dma_event(struct dma_client *client, struct dma_chan *chan, - enum dma_state state) -{ - int i, found = 0, pos = -1; - struct net_dma *net_dma = - container_of(client, struct net_dma, client); - enum dma_state_client ack = DMA_DUP; /* default: take no action */ - - spin_lock(&net_dma->lock); - switch (state) { - case DMA_RESOURCE_AVAILABLE: - for (i = 0; i < nr_cpu_ids; i++) - if (net_dma->channels[i] == chan) { - found = 1; - break; - } else if (net_dma->channels[i] == NULL && pos < 0) - pos = i; - - if (!found && pos >= 0) { - ack = DMA_ACK; - net_dma->channels[pos] = chan; - cpu_set(pos, net_dma->channel_mask); - net_dma_rebalance(net_dma); - } - break; - case DMA_RESOURCE_REMOVED: - for (i = 0; i < nr_cpu_ids; i++) - if (net_dma->channels[i] == chan) { - found = 1; - pos = i; - break; - } - - if (found) { - ack = DMA_ACK; - cpu_clear(pos, net_dma->channel_mask); - net_dma->channels[i] = NULL; - net_dma_rebalance(net_dma); - } - break; - default: - break; - } - spin_unlock(&net_dma->lock); - - return ack; -} - -/** - * netdev_dma_register - register the networking subsystem as a DMA client - */ -static int __init netdev_dma_register(void) -{ - net_dma.channels = kzalloc(nr_cpu_ids * sizeof(struct net_dma), - GFP_KERNEL); - if (unlikely(!net_dma.channels)) { - printk(KERN_NOTICE - "netdev_dma: no memory for net_dma.channels\n"); - return -ENOMEM; - } - spin_lock_init(&net_dma.lock); - dma_cap_set(DMA_MEMCPY, net_dma.client.cap_mask); - dma_async_client_register(&net_dma.client); - dma_async_client_chan_request(&net_dma.client); - return 0; -} - -#else -static int __init netdev_dma_register(void) { return -ENODEV; } -#endif /* CONFIG_NET_DMA */ /** * netdev_increment_features - increment feature set by one @@ -5251,8 +5155,6 @@ static int __init net_dev_init(void) if (register_pernet_device(&default_device_ops)) goto out; - netdev_dma_register(); - open_softirq(NET_TX_SOFTIRQ, net_tx_action); open_softirq(NET_RX_SOFTIRQ, net_rx_action); |