dpaa_eth: Use one NAPI structure per portal

We currently use one NAPI instance per CPU which is enough when
having a single portal affined to each CPU.
In CPU Hotplug scenario, when CPUs become offline, portals affined
to them are migrated to the boot CPU therefore CPU0 will receive
interrupts not only from its affined portal but also from the
portal(s) that have migrated to it. In this case, a single NAPI
structure is not sufficient, therefore we introduce a list of
NAPI structures per CPU per portal such that when an interrupt
is received from a specific portal, we retrieve and schedule the
NAPI corresponding to that CPU and that portal.

Signed-off-by: Cristian Sovaiala <cristian.sovaiala@freescale.com>
Change-Id: I837e007a4256aba31bb1df9f893a20a1b0160a32
Reviewed-on: http://git.am.freescale.net:8181/5628
Tested-by: Review Code-CDREVIEW <CDREVIEW@freescale.com>
Reviewed-by: Bucur Madalin-Cristian-B32716 <madalin.bucur@freescale.com>
Reviewed-by: Ruxandra Ioana Radulescu <ruxandra.radulescu@freescale.com>
Reviewed-by: Bogdan Hamciuc <bogdan.hamciuc@freescale.com>
Reviewed-by: Rivera Jose-B46482 <German.Rivera@freescale.com>

5 files changed, 105 insertions, 29 deletions

diff --git a/drivers/net/ethernet/freescale/dpa/dpaa_eth.c b/drivers/net/ethernet/freescale/dpa/dpaa_eth.c
index b68d8d1..dbb9112 100644
--- a/drivers/net/ethernet/freescale/dpa/dpaa_eth.c
+++ b/drivers/net/ethernet/freescale/dpa/dpaa_eth.c
@@ -116,6 +116,8 @@ int dpa_free_pcd_fqids(struct device *, uint32_t) __attribute__((weak));
 
 uint8_t dpa_priv_common_bpid;
 
+extern u16 qman_portal_max;
+
 /* A set of callbacks for hooking into the fastpath at different points. */
 struct dpaa_eth_hooks_s dpaa_eth_hooks;
 /*
@@ -275,15 +277,15 @@ void __hot _dpa_process_parse_results(const fm_prs_result_t *parse_results,
 
 static int dpaa_eth_poll(struct napi_struct *napi, int budget)
 {
-	struct dpa_percpu_priv_s *percpu_priv =
-			container_of(napi, struct dpa_percpu_priv_s, napi);
+	struct dpa_napi_portal *np =
+			container_of(napi, struct dpa_napi_portal, napi);
 
-	int cleaned = qman_p_poll_dqrr(percpu_priv->p, budget);
+	int cleaned = qman_p_poll_dqrr(np->p, budget);
 
 	if (cleaned < budget) {
 		int tmp;
 		napi_complete(napi);
-		tmp = qman_p_irqsource_add(percpu_priv->p, QM_PIRQ_DQRI);
+		tmp = qman_p_irqsource_add(np->p, QM_PIRQ_DQRI);
 		BUG_ON(tmp);
 	}
 
@@ -339,9 +341,8 @@ priv_rx_error_dqrr(struct qman_portal		*portal,
 	priv = netdev_priv(net_dev);
 
 	percpu_priv = __this_cpu_ptr(priv->percpu_priv);
-	percpu_priv->p = portal;
 
-	if (dpaa_eth_napi_schedule(percpu_priv))
+	if (dpaa_eth_napi_schedule(percpu_priv, portal))
 		return qman_cb_dqrr_stop;
 
 	if (unlikely(dpaa_eth_refill_bpools(priv->dpa_bp)))
@@ -374,9 +375,8 @@ priv_rx_default_dqrr(struct qman_portal		*portal,
 
 	/* IRQ handler, non-migratable; safe to use __this_cpu_ptr here */
 	percpu_priv = __this_cpu_ptr(priv->percpu_priv);
-	percpu_priv->p = portal;
 
-	if (unlikely(dpaa_eth_napi_schedule(percpu_priv)))
+	if (unlikely(dpaa_eth_napi_schedule(percpu_priv, portal)))
 		return qman_cb_dqrr_stop;
 
 	/* Vale of plenty: make sure we didn't run out of buffers */
@@ -388,7 +388,7 @@ priv_rx_default_dqrr(struct qman_portal		*portal,
 		 */
 		dpa_fd_release(net_dev, &dq->fd);
 	else
-		_dpa_rx(net_dev, priv, percpu_priv, &dq->fd, fq->fqid);
+		_dpa_rx(net_dev, portal, priv, percpu_priv, &dq->fd, fq->fqid);
 
 	return qman_cb_dqrr_consume;
 }
@@ -406,9 +406,8 @@ priv_tx_conf_error_dqrr(struct qman_portal		*portal,
 	priv = netdev_priv(net_dev);
 
 	percpu_priv = __this_cpu_ptr(priv->percpu_priv);
-	percpu_priv->p = portal;
 
-	if (dpaa_eth_napi_schedule(percpu_priv))
+	if (dpaa_eth_napi_schedule(percpu_priv, portal))
 		return qman_cb_dqrr_stop;
 
 	_dpa_tx_error(net_dev, priv, percpu_priv, &dq->fd, fq->fqid);
@@ -433,9 +432,8 @@ priv_tx_conf_default_dqrr(struct qman_portal		*portal,
 
 	/* Non-migratable context, safe to use __this_cpu_ptr */
 	percpu_priv = __this_cpu_ptr(priv->percpu_priv);
-	percpu_priv->p = portal;
 
-	if (dpaa_eth_napi_schedule(percpu_priv))
+	if (dpaa_eth_napi_schedule(percpu_priv, portal))
 		return qman_cb_dqrr_stop;
 
 	_dpa_tx_conf(net_dev, priv, percpu_priv, &dq->fd, fq->fqid);
@@ -486,22 +484,26 @@ static const dpa_fq_cbs_t private_fq_cbs = {
 static void dpaa_eth_napi_enable(struct dpa_priv_s *priv)
 {
 	struct dpa_percpu_priv_s *percpu_priv;
-	int i;
+	int i, j;
 
 	for_each_possible_cpu(i) {
 		percpu_priv = per_cpu_ptr(priv->percpu_priv, i);
-		napi_enable(&percpu_priv->napi);
+
+		for (j = 0; j < qman_portal_max; j++)
+			napi_enable(&percpu_priv->np[j].napi);
 	}
 }
 
 static void dpaa_eth_napi_disable(struct dpa_priv_s *priv)
 {
 	struct dpa_percpu_priv_s *percpu_priv;
-	int i;
+	int i, j;
 
 	for_each_possible_cpu(i) {
 		percpu_priv = per_cpu_ptr(priv->percpu_priv, i);
-		napi_disable(&percpu_priv->napi);
+
+		for (j = 0; j < qman_portal_max; j++)
+			napi_disable(&percpu_priv->np[j].napi);
 	}
 }
 
@@ -580,6 +582,50 @@ static const struct net_device_ops dpa_private_ops = {
 #endif
 };
 
+static int dpa_private_napi_add(struct net_device *net_dev)
+{
+	struct dpa_priv_s *priv = netdev_priv(net_dev);
+	struct dpa_percpu_priv_s *percpu_priv;
+	int i, cpu;
+
+	for_each_possible_cpu(cpu) {
+		percpu_priv = per_cpu_ptr(priv->percpu_priv, cpu);
+
+		percpu_priv->np = devm_kzalloc(net_dev->dev.parent,
+			qman_portal_max * sizeof(struct dpa_napi_portal),
+			GFP_KERNEL);
+
+		if (unlikely(percpu_priv->np == NULL)) {
+			dev_err(net_dev->dev.parent, "devm_kzalloc() failed\n");
+			return -ENOMEM;
+		}
+
+		for (i = 0; i < qman_portal_max; i++)
+			netif_napi_add(net_dev, &percpu_priv->np[i].napi,
+					dpaa_eth_poll, DPA_NAPI_WEIGHT);
+	}
+
+	return 0;
+}
+
+void dpa_private_napi_del(struct net_device *net_dev)
+{
+	struct dpa_priv_s *priv = netdev_priv(net_dev);
+	struct dpa_percpu_priv_s *percpu_priv;
+	int i, cpu;
+
+	for_each_possible_cpu(cpu) {
+		percpu_priv = per_cpu_ptr(priv->percpu_priv, cpu);
+
+		if (percpu_priv->np) {
+			for (i = 0; i < qman_portal_max; i++)
+				netif_napi_del(&percpu_priv->np[i].napi);
+
+			devm_kfree(net_dev->dev.parent, percpu_priv->np);
+		}
+	}
+}
+
 static int dpa_private_netdev_init(struct device_node *dpa_node,
 				struct net_device *net_dev)
 {
@@ -594,13 +640,9 @@ static int dpa_private_netdev_init(struct device_node *dpa_node,
 	for_each_possible_cpu(i) {
 		percpu_priv = per_cpu_ptr(priv->percpu_priv, i);
 		percpu_priv->net_dev = net_dev;
-
-		netif_napi_add(net_dev, &percpu_priv->napi, dpaa_eth_poll,
-			       DPA_NAPI_WEIGHT);
 	}
 
 	net_dev->netdev_ops = &dpa_private_ops;
-
 	mac_addr = priv->mac_dev->addr;
 
 	net_dev->mem_start = priv->mac_dev->res->start;
@@ -834,6 +876,12 @@ dpaa_eth_priv_probe(struct platform_device *_of_dev)
 		memset(percpu_priv, 0, sizeof(*percpu_priv));
 	}
 
+	/* Initialize NAPI */
+	err = dpa_private_napi_add(net_dev);
+
+	if (err < 0)
+		goto napi_add_failed;
+
 	err = dpa_private_netdev_init(dpa_node, net_dev);
 
 	if (err < 0)
@@ -845,9 +893,12 @@ dpaa_eth_priv_probe(struct platform_device *_of_dev)
 
 	return 0;
 
+napi_add_failed:
 netdev_init_failed:
-	if (net_dev)
+	if (net_dev) {
+		dpa_private_napi_del(net_dev);
 		free_percpu(priv->percpu_priv);
+	}
 alloc_percpu_failed:
 fq_alloc_failed:
 	if (net_dev) {
diff --git a/drivers/net/ethernet/freescale/dpa/dpaa_eth.h b/drivers/net/ethernet/freescale/dpa/dpaa_eth.h
index 97f3672..dea880a 100644
--- a/drivers/net/ethernet/freescale/dpa/dpaa_eth.h
+++ b/drivers/net/ethernet/freescale/dpa/dpaa_eth.h
@@ -380,10 +380,14 @@ struct dpa_ern_cnt {
 	u64 orp_zero;		/* ORP disabled */
 };
 
-struct dpa_percpu_priv_s {
-	struct net_device *net_dev;
+struct dpa_napi_portal {
 	struct napi_struct napi;
 	struct qman_portal *p;
+};
+
+struct dpa_percpu_priv_s {
+	struct net_device *net_dev;
+	struct dpa_napi_portal *np;
 	u64 in_interrupt;
 	u64 tx_returned;
 	u64 tx_confirm;
@@ -467,6 +471,7 @@ struct fm_port_fqs {
 int dpa_bp_priv_seed(struct dpa_bp *dpa_bp);
 int dpaa_eth_refill_bpools(struct dpa_bp *dpa_bp);
 void __hot _dpa_rx(struct net_device *net_dev,
+		struct qman_portal *portal,
 		const struct dpa_priv_s *priv,
 		struct dpa_percpu_priv_s *percpu_priv,
 		const struct qm_fd *fd,
@@ -497,7 +502,8 @@ int _dpa_bp_add_8_bufs(const struct dpa_bp *dpa_bp);
 int dpa_enable_tx_csum(struct dpa_priv_s *priv,
 	struct sk_buff *skb, struct qm_fd *fd, char *parse_results);
 
-static inline int dpaa_eth_napi_schedule(struct dpa_percpu_priv_s *percpu_priv)
+static inline int dpaa_eth_napi_schedule(struct dpa_percpu_priv_s *percpu_priv,
+			struct qman_portal *portal)
 {
 	/* In case of threaded ISR for RT enable kernel,
 	 * in_irq() does not return appropriate value, so use
@@ -505,9 +511,15 @@ static inline int dpaa_eth_napi_schedule(struct dpa_percpu_priv_s *percpu_priv)
 	 */
 	if (unlikely(in_irq() || !in_serving_softirq())) {
 		/* Disable QMan IRQ and invoke NAPI */
-		int ret = qman_p_irqsource_remove(percpu_priv->p, QM_PIRQ_DQRI);
+		int ret = qman_p_irqsource_remove(portal, QM_PIRQ_DQRI);
 		if (likely(!ret)) {
-			napi_schedule(&percpu_priv->napi);
+			const struct qman_portal_config *pc =
+					qman_p_get_portal_config(portal);
+			struct dpa_napi_portal *np =
+					&percpu_priv->np[pc->index];
+
+			np->p = portal;
+			napi_schedule(&np->napi);
 			percpu_priv->in_interrupt++;
 			return 1;
 		}
@@ -571,6 +583,8 @@ int fm_mac_dump_regs(struct mac_device *h_dev, char *buf, int n);
 void dpaa_eth_sysfs_remove(struct device *dev);
 void dpaa_eth_sysfs_init(struct device *dev);
 
+void dpa_private_napi_del(struct net_device *net_dev);
+
 /* Equivalent to a memset(0), but works faster */
 static inline void clear_fd(struct qm_fd *fd)
 {
diff --git a/drivers/net/ethernet/freescale/dpa/dpaa_eth_common.c b/drivers/net/ethernet/freescale/dpa/dpaa_eth_common.c
index d4ded36..49d91b0 100644
--- a/drivers/net/ethernet/freescale/dpa/dpaa_eth_common.c
+++ b/drivers/net/ethernet/freescale/dpa/dpaa_eth_common.c
@@ -471,6 +471,7 @@ int __cold dpa_remove(struct platform_device *of_dev)
 
 	err = dpa_fq_free(dev, &priv->dpa_fq_list);
 
+	dpa_private_napi_del(net_dev);
 	free_percpu(priv->percpu_priv);
 
 	dpa_bp_free(priv, priv->dpa_bp);
diff --git a/drivers/net/ethernet/freescale/dpa/dpaa_eth_non_sg.c b/drivers/net/ethernet/freescale/dpa/dpaa_eth_non_sg.c
index f1f9fce..d608881 100644
--- a/drivers/net/ethernet/freescale/dpa/dpaa_eth_non_sg.c
+++ b/drivers/net/ethernet/freescale/dpa/dpaa_eth_non_sg.c
@@ -294,6 +294,7 @@ static int dpa_process_one(struct dpa_percpu_priv_s *percpu_priv,
 }
 
 void __hot _dpa_rx(struct net_device *net_dev,
+		struct qman_portal *portal,
 		const struct dpa_priv_s *priv,
 		struct dpa_percpu_priv_s *percpu_priv,
 		const struct qm_fd *fd,
@@ -370,8 +371,12 @@ void __hot _dpa_rx(struct net_device *net_dev,
 
 	if (use_gro) {
 		gro_result_t gro_result;
+		const struct qman_portal_config *pc =
+					qman_p_get_portal_config(portal);
+		struct dpa_napi_portal *np = &percpu_priv->np[pc->index];
 
-		gro_result = napi_gro_receive(&percpu_priv->napi, skb);
+		np->p = portal;
+		gro_result = napi_gro_receive(&np->napi, skb);
 		/* If frame is dropped by the stack, rx_dropped counter is
 		 * incremented automatically, so no need for us to update it
 		 */
diff --git a/drivers/net/ethernet/freescale/dpa/dpaa_eth_sg.c b/drivers/net/ethernet/freescale/dpa/dpaa_eth_sg.c
index 38580c9..eade30a 100644
--- a/drivers/net/ethernet/freescale/dpa/dpaa_eth_sg.c
+++ b/drivers/net/ethernet/freescale/dpa/dpaa_eth_sg.c
@@ -505,6 +505,7 @@ static struct sk_buff *__hot sg_fd_to_skb(const struct dpa_priv_s *priv,
 }
 
 void __hot _dpa_rx(struct net_device *net_dev,
+		struct qman_portal *portal,
 		const struct dpa_priv_s *priv,
 		struct dpa_percpu_priv_s *percpu_priv,
 		const struct qm_fd *fd,
@@ -573,8 +574,12 @@ void __hot _dpa_rx(struct net_device *net_dev,
 
 	if (use_gro) {
 		gro_result_t gro_result;
+		const struct qman_portal_config *pc =
+					qman_p_get_portal_config(portal);
+		struct dpa_napi_portal *np = &percpu_priv->np[pc->index];
 
-		gro_result = napi_gro_receive(&percpu_priv->napi, skb);
+		np->p = portal;
+		gro_result = napi_gro_receive(&np->napi, skb);
 		/* If frame is dropped by the stack, rx_dropped counter is
 		 * incremented automatically, so no need for us to update it
 		 */