1 files changed, 637 insertions, 0 deletions

diff --git a/drivers/staging/fsl_ppfe/pfe_hif_lib.c b/drivers/staging/fsl_ppfe/pfe_hif_lib.c
new file mode 100644
index 0000000..eeab4ff
--- /dev/null
+++ b/drivers/staging/fsl_ppfe/pfe_hif_lib.c
@@ -0,0 +1,637 @@
+/*
+ * Copyright 2015-2016 Freescale Semiconductor, Inc.
+ * Copyright 2017 NXP
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/version.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/interrupt.h>
+#include <linux/workqueue.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmapool.h>
+#include <linux/sched.h>
+#include <linux/skbuff.h>
+#include <linux/moduleparam.h>
+#include <linux/cpu.h>
+
+#include "pfe_mod.h"
+#include "pfe_hif.h"
+#include "pfe_hif_lib.h"
+
+unsigned int lro_mode;
+unsigned int page_mode;
+unsigned int tx_qos = 1;
+module_param(tx_qos, uint, 0444);
+MODULE_PARM_DESC(tx_qos, "0: disable ,\n"
+			 "1: enable (default), guarantee no packet drop at TMU level\n");
+unsigned int pfe_pkt_size;
+unsigned int pfe_pkt_headroom;
+unsigned int emac_txq_cnt;
+
+/*
+ * @pfe_hal_lib.c.
+ * Common functions used by HIF client drivers
+ */
+
+/*HIF shared memory Global variable */
+struct hif_shm ghif_shm;
+
+/* Cleanup the HIF shared memory, release HIF rx_buffer_pool.
+ * This function should be called after pfe_hif_exit
+ *
+ * @param[in] hif_shm		Shared memory address location in DDR
+ */
+static void pfe_hif_shm_clean(struct hif_shm *hif_shm)
+{
+	int i;
+	void *pkt;
+
+	for (i = 0; i < hif_shm->rx_buf_pool_cnt; i++) {
+		pkt = hif_shm->rx_buf_pool[i];
+		if (pkt) {
+			hif_shm->rx_buf_pool[i] = NULL;
+			pkt -= pfe_pkt_headroom;
+
+			if (page_mode)
+				put_page(virt_to_page(pkt));
+			else
+				kfree(pkt);
+		}
+	}
+}
+
+/* Initialize shared memory used between HIF driver and clients,
+ * allocate rx_buffer_pool required for HIF Rx descriptors.
+ * This function should be called before initializing HIF driver.
+ *
+ * @param[in] hif_shm		Shared memory address location in DDR
+ * @rerurn			0 - on succes, <0 on fail to initialize
+ */
+static int pfe_hif_shm_init(struct hif_shm *hif_shm)
+{
+	int i;
+	void *pkt;
+
+	memset(hif_shm, 0, sizeof(struct hif_shm));
+	hif_shm->rx_buf_pool_cnt = HIF_RX_DESC_NT;
+
+	for (i = 0; i < hif_shm->rx_buf_pool_cnt; i++) {
+		if (page_mode) {
+			pkt = (void *)__get_free_page(GFP_KERNEL |
+				GFP_DMA_PFE);
+		} else {
+			pkt = kmalloc(PFE_BUF_SIZE, GFP_KERNEL | GFP_DMA_PFE);
+		}
+
+		if (pkt)
+			hif_shm->rx_buf_pool[i] = pkt + pfe_pkt_headroom;
+		else
+			goto err0;
+	}
+
+	return 0;
+
+err0:
+	pr_err("%s Low memory\n", __func__);
+	pfe_hif_shm_clean(hif_shm);
+	return -ENOMEM;
+}
+
+/*This function sends indication to HIF driver
+ *
+ * @param[in] hif	hif context
+ */
+static void hif_lib_indicate_hif(struct pfe_hif *hif, int req, int data1, int
+					data2)
+{
+	hif_process_client_req(hif, req, data1, data2);
+}
+
+void hif_lib_indicate_client(int client_id, int event_type, int qno)
+{
+	struct hif_client_s *client = pfe->hif_client[client_id];
+
+	if (!client || (event_type >= HIF_EVENT_MAX) || (qno >=
+		HIF_CLIENT_QUEUES_MAX))
+		return;
+
+	if (!test_and_set_bit(qno, &client->queue_mask[event_type]))
+		client->event_handler(client->priv, event_type, qno);
+}
+
+/*This function releases Rx queue descriptors memory and pre-filled buffers
+ *
+ * @param[in] client	hif_client context
+ */
+static void hif_lib_client_release_rx_buffers(struct hif_client_s *client)
+{
+	struct rx_queue_desc *desc;
+	int qno, ii;
+	void *buf;
+
+	for (qno = 0; qno < client->rx_qn; qno++) {
+		desc = client->rx_q[qno].base;
+
+		for (ii = 0; ii < client->rx_q[qno].size; ii++) {
+			buf = (void *)desc->data;
+			if (buf) {
+				buf -= pfe_pkt_headroom;
+
+				if (page_mode)
+					free_page((unsigned long)buf);
+				else
+					kfree(buf);
+
+				desc->ctrl = 0;
+			}
+
+			desc++;
+		}
+	}
+
+	kfree(client->rx_qbase);
+}
+
+/*This function allocates memory for the rxq descriptors and pre-fill rx queues
+ * with buffers.
+ * @param[in] client	client context
+ * @param[in] q_size	size of the rxQ, all queues are of same size
+ */
+static int hif_lib_client_init_rx_buffers(struct hif_client_s *client, int
+						q_size)
+{
+	struct rx_queue_desc *desc;
+	struct hif_client_rx_queue *queue;
+	int ii, qno;
+
+	/*Allocate memory for the client queues */
+	client->rx_qbase = kzalloc(client->rx_qn * q_size * sizeof(struct
+				rx_queue_desc), GFP_KERNEL);
+	if (!client->rx_qbase)
+		goto err;
+
+	for (qno = 0; qno < client->rx_qn; qno++) {
+		queue = &client->rx_q[qno];
+
+		queue->base = client->rx_qbase + qno * q_size * sizeof(struct
+				rx_queue_desc);
+		queue->size = q_size;
+		queue->read_idx = 0;
+		queue->write_idx = 0;
+
+		pr_debug("rx queue: %d, base: %p, size: %d\n", qno,
+			 queue->base, queue->size);
+	}
+
+	for (qno = 0; qno < client->rx_qn; qno++) {
+		queue = &client->rx_q[qno];
+		desc = queue->base;
+
+		for (ii = 0; ii < queue->size; ii++) {
+			desc->ctrl = CL_DESC_BUF_LEN(pfe_pkt_size) |
+					CL_DESC_OWN;
+			desc++;
+		}
+	}
+
+	return 0;
+
+err:
+	return 1;
+}
+
+
+static void hif_lib_client_cleanup_tx_queue(struct hif_client_tx_queue *queue)
+{
+	pr_debug("%s\n", __func__);
+
+	/*
+	 * Check if there are any pending packets. Client must flush the tx
+	 * queues before unregistering, by calling by calling
+	 * hif_lib_tx_get_next_complete()
+	 *
+	 * Hif no longer calls since we are no longer registered
+	 */
+	if (queue->tx_pending)
+		pr_err("%s: pending transmit packets\n", __func__);
+}
+
+static void hif_lib_client_release_tx_buffers(struct hif_client_s *client)
+{
+	int qno;
+
+	pr_debug("%s\n", __func__);
+
+	for (qno = 0; qno < client->tx_qn; qno++)
+		hif_lib_client_cleanup_tx_queue(&client->tx_q[qno]);
+
+	kfree(client->tx_qbase);
+}
+
+static int hif_lib_client_init_tx_buffers(struct hif_client_s *client, int
+						q_size)
+{
+	struct hif_client_tx_queue *queue;
+	int qno;
+
+	client->tx_qbase = kzalloc(client->tx_qn * q_size * sizeof(struct
+					tx_queue_desc), GFP_KERNEL);
+	if (!client->tx_qbase)
+		return 1;
+
+	for (qno = 0; qno < client->tx_qn; qno++) {
+		queue = &client->tx_q[qno];
+
+		queue->base = client->tx_qbase + qno * q_size * sizeof(struct
+				tx_queue_desc);
+		queue->size = q_size;
+		queue->read_idx = 0;
+		queue->write_idx = 0;
+		queue->tx_pending = 0;
+		queue->nocpy_flag = 0;
+		queue->prev_tmu_tx_pkts = 0;
+		queue->done_tmu_tx_pkts = 0;
+
+		pr_debug("tx queue: %d, base: %p, size: %d\n", qno,
+			 queue->base, queue->size);
+	}
+
+	return 0;
+}
+
+static int hif_lib_event_dummy(void *priv, int event_type, int qno)
+{
+	return 0;
+}
+
+int hif_lib_client_register(struct hif_client_s *client)
+{
+	struct hif_shm *hif_shm;
+	struct hif_client_shm *client_shm;
+	int err, i;
+	/* int loop_cnt = 0; */
+
+	pr_debug("%s\n", __func__);
+
+	/*Allocate memory before spin_lock*/
+	if (hif_lib_client_init_rx_buffers(client, client->rx_qsize)) {
+		err = -ENOMEM;
+		goto err_rx;
+	}
+
+	if (hif_lib_client_init_tx_buffers(client, client->tx_qsize)) {
+		err = -ENOMEM;
+		goto err_tx;
+	}
+
+	spin_lock_bh(&pfe->hif.lock);
+	if (!(client->pfe) || (client->id >= HIF_CLIENTS_MAX) ||
+	    (pfe->hif_client[client->id])) {
+		err = -EINVAL;
+		goto err;
+	}
+
+	hif_shm = client->pfe->hif.shm;
+
+	if (!client->event_handler)
+		client->event_handler = hif_lib_event_dummy;
+
+	/*Initialize client specific shared memory */
+	client_shm = (struct hif_client_shm *)&hif_shm->client[client->id];
+	client_shm->rx_qbase = (unsigned long int)client->rx_qbase;
+	client_shm->rx_qsize = client->rx_qsize;
+	client_shm->tx_qbase = (unsigned long int)client->tx_qbase;
+	client_shm->tx_qsize = client->tx_qsize;
+	client_shm->ctrl = (client->tx_qn << CLIENT_CTRL_TX_Q_CNT_OFST) |
+				(client->rx_qn << CLIENT_CTRL_RX_Q_CNT_OFST);
+	/* spin_lock_init(&client->rx_lock); */
+
+	for (i = 0; i < HIF_EVENT_MAX; i++) {
+		client->queue_mask[i] = 0;  /*
+					     * By default all events are
+					     * unmasked
+					     */
+	}
+
+	/*Indicate to HIF driver*/
+	hif_lib_indicate_hif(&pfe->hif, REQUEST_CL_REGISTER, client->id, 0);
+
+	pr_debug("%s: client: %p, client_id: %d, tx_qsize: %d, rx_qsize: %d\n",
+		 __func__, client, client->id, client->tx_qsize,
+		 client->rx_qsize);
+
+	client->cpu_id = -1;
+
+	pfe->hif_client[client->id] = client;
+	spin_unlock_bh(&pfe->hif.lock);
+
+	return 0;
+
+err:
+	spin_unlock_bh(&pfe->hif.lock);
+	hif_lib_client_release_tx_buffers(client);
+
+err_tx:
+	hif_lib_client_release_rx_buffers(client);
+
+err_rx:
+	return err;
+}
+
+int hif_lib_client_unregister(struct hif_client_s *client)
+{
+	struct pfe *pfe = client->pfe;
+	u32 client_id = client->id;
+
+	pr_info(
+		"%s : client: %p, client_id: %d, txQ_depth: %d, rxQ_depth: %d\n"
+		, __func__, client, client->id, client->tx_qsize,
+		client->rx_qsize);
+
+	spin_lock_bh(&pfe->hif.lock);
+	hif_lib_indicate_hif(&pfe->hif, REQUEST_CL_UNREGISTER, client->id, 0);
+
+	hif_lib_client_release_tx_buffers(client);
+	hif_lib_client_release_rx_buffers(client);
+	pfe->hif_client[client_id] = NULL;
+	spin_unlock_bh(&pfe->hif.lock);
+
+	return 0;
+}
+
+int hif_lib_event_handler_start(struct hif_client_s *client, int event,
+				int qno)
+{
+	struct hif_client_rx_queue *queue = &client->rx_q[qno];
+	struct rx_queue_desc *desc = queue->base + queue->read_idx;
+
+	if ((event >= HIF_EVENT_MAX) || (qno >= HIF_CLIENT_QUEUES_MAX)) {
+		pr_debug("%s: Unsupported event : %d  queue number : %d\n",
+			 __func__, event, qno);
+		return -1;
+	}
+
+	test_and_clear_bit(qno, &client->queue_mask[event]);
+
+	switch (event) {
+	case EVENT_RX_PKT_IND:
+		if (!(desc->ctrl & CL_DESC_OWN))
+			hif_lib_indicate_client(client->id,
+						EVENT_RX_PKT_IND, qno);
+		break;
+
+	case EVENT_HIGH_RX_WM:
+	case EVENT_TXDONE_IND:
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+/*
+ * This function gets one packet from the specified client queue
+ * It also refill the rx buffer
+ */
+void *hif_lib_receive_pkt(struct hif_client_s *client, int qno, int *len, int
+				*ofst, unsigned int *rx_ctrl,
+				unsigned int *desc_ctrl, void **priv_data)
+{
+	struct hif_client_rx_queue *queue = &client->rx_q[qno];
+	struct rx_queue_desc *desc;
+	void *pkt = NULL;
+
+	/*
+	 * Following lock is to protect rx queue access from,
+	 * hif_lib_event_handler_start.
+	 * In general below lock is not required, because hif_lib_xmit_pkt and
+	 * hif_lib_event_handler_start are called from napi poll and which is
+	 * not re-entrant. But if some client use in different way this lock is
+	 * required.
+	 */
+	/*spin_lock_irqsave(&client->rx_lock, flags); */
+	desc = queue->base + queue->read_idx;
+	if (!(desc->ctrl & CL_DESC_OWN)) {
+		pkt = desc->data - pfe_pkt_headroom;
+
+		*rx_ctrl = desc->client_ctrl;
+		*desc_ctrl = desc->ctrl;
+
+		if (desc->ctrl & CL_DESC_FIRST) {
+			u16 size = *rx_ctrl >> HIF_CTRL_RX_OFFSET_OFST;
+
+			if (size) {
+				*len = CL_DESC_BUF_LEN(desc->ctrl) -
+						PFE_PKT_HEADER_SZ - size;
+				*ofst = pfe_pkt_headroom + PFE_PKT_HEADER_SZ
+								+ size;
+				*priv_data = desc->data + PFE_PKT_HEADER_SZ;
+			} else {
+				*len = CL_DESC_BUF_LEN(desc->ctrl) -
+						PFE_PKT_HEADER_SZ;
+				*ofst = pfe_pkt_headroom + PFE_PKT_HEADER_SZ;
+				*priv_data = NULL;
+			}
+
+		} else {
+			*len = CL_DESC_BUF_LEN(desc->ctrl);
+			*ofst = pfe_pkt_headroom;
+		}
+
+		/*
+		 * Needed so we don't free a buffer/page
+		 * twice on module_exit
+		 */
+		desc->data = NULL;
+
+		/*
+		 * Ensure everything else is written to DDR before
+		 * writing bd->ctrl
+		 */
+		smp_wmb();
+
+		desc->ctrl = CL_DESC_BUF_LEN(pfe_pkt_size) | CL_DESC_OWN;
+		queue->read_idx = (queue->read_idx + 1) & (queue->size - 1);
+	}
+
+	/*spin_unlock_irqrestore(&client->rx_lock, flags); */
+	return pkt;
+}
+
+static inline void hif_hdr_write(struct hif_hdr *pkt_hdr, unsigned int
+					client_id, unsigned int qno,
+					u32 client_ctrl)
+{
+	/* Optimize the write since the destinaton may be non-cacheable */
+	if (!((unsigned long)pkt_hdr & 0x3)) {
+		((u32 *)pkt_hdr)[0] = (client_ctrl << 16) | (qno << 8) |
+					client_id;
+	} else {
+		((u16 *)pkt_hdr)[0] = (qno << 8) | (client_id & 0xFF);
+		((u16 *)pkt_hdr)[1] = (client_ctrl & 0xFFFF);
+	}
+}
+
+/*This function puts the given packet in the specific client queue */
+void __hif_lib_xmit_pkt(struct hif_client_s *client, unsigned int qno, void
+				*data, unsigned int len, u32 client_ctrl,
+				unsigned int flags, void *client_data)
+{
+	struct hif_client_tx_queue *queue = &client->tx_q[qno];
+	struct tx_queue_desc *desc = queue->base + queue->write_idx;
+
+	/* First buffer */
+	if (flags & HIF_FIRST_BUFFER) {
+		data -= sizeof(struct hif_hdr);
+		len += sizeof(struct hif_hdr);
+
+		hif_hdr_write(data, client->id, qno, client_ctrl);
+	}
+
+	desc->data = client_data;
+	desc->ctrl = CL_DESC_OWN | CL_DESC_FLAGS(flags);
+
+	__hif_xmit_pkt(&pfe->hif, client->id, qno, data, len, flags);
+
+	queue->write_idx = (queue->write_idx + 1) & (queue->size - 1);
+	queue->tx_pending++;
+	queue->jiffies_last_packet = jiffies;
+}
+
+void *hif_lib_tx_get_next_complete(struct hif_client_s *client, int qno,
+				   unsigned int *flags, int count)
+{
+	struct hif_client_tx_queue *queue = &client->tx_q[qno];
+	struct tx_queue_desc *desc = queue->base + queue->read_idx;
+
+	pr_debug("%s: qno : %d rd_indx: %d pending:%d\n", __func__, qno,
+		 queue->read_idx, queue->tx_pending);
+
+	if (!queue->tx_pending)
+		return NULL;
+
+	if (queue->nocpy_flag && !queue->done_tmu_tx_pkts) {
+		u32 tmu_tx_pkts = be32_to_cpu(pe_dmem_read(TMU0_ID +
+			client->id, TMU_DM_TX_TRANS, 4));
+
+		if (queue->prev_tmu_tx_pkts > tmu_tx_pkts)
+			queue->done_tmu_tx_pkts = UINT_MAX -
+				queue->prev_tmu_tx_pkts + tmu_tx_pkts;
+		else
+			queue->done_tmu_tx_pkts = tmu_tx_pkts -
+						queue->prev_tmu_tx_pkts;
+
+		queue->prev_tmu_tx_pkts  = tmu_tx_pkts;
+
+		if (!queue->done_tmu_tx_pkts)
+			return NULL;
+	}
+
+	if (desc->ctrl & CL_DESC_OWN)
+		return NULL;
+
+	queue->read_idx = (queue->read_idx + 1) & (queue->size - 1);
+	queue->tx_pending--;
+
+	*flags = CL_DESC_GET_FLAGS(desc->ctrl);
+
+	if (queue->done_tmu_tx_pkts && (*flags & HIF_LAST_BUFFER))
+		queue->done_tmu_tx_pkts--;
+
+	return desc->data;
+}
+
+static void hif_lib_tmu_credit_init(struct pfe *pfe)
+{
+	int i, q;
+
+	for (i = 0; i < NUM_GEMAC_SUPPORT; i++)
+		for (q = 0; q < emac_txq_cnt; q++) {
+			pfe->tmu_credit.tx_credit_max[i][q] = (q == 0) ?
+					DEFAULT_Q0_QDEPTH : DEFAULT_MAX_QDEPTH;
+			pfe->tmu_credit.tx_credit[i][q] =
+					pfe->tmu_credit.tx_credit_max[i][q];
+		}
+}
+
+/* __hif_lib_update_credit
+ *
+ * @param[in] client	hif client context
+ * @param[in] queue	queue number in match with TMU
+ */
+void __hif_lib_update_credit(struct hif_client_s *client, unsigned int queue)
+{
+	unsigned int tmu_tx_packets, tmp;
+
+	if (tx_qos) {
+		tmu_tx_packets = be32_to_cpu(pe_dmem_read(TMU0_ID +
+			client->id, (TMU_DM_TX_TRANS + (queue * 4)), 4));
+
+		/* tx_packets counter overflowed */
+		if (tmu_tx_packets >
+		    pfe->tmu_credit.tx_packets[client->id][queue]) {
+			tmp = UINT_MAX - tmu_tx_packets +
+			pfe->tmu_credit.tx_packets[client->id][queue];
+
+			pfe->tmu_credit.tx_credit[client->id][queue] =
+			pfe->tmu_credit.tx_credit_max[client->id][queue] - tmp;
+		} else {
+		/* TMU tx <= pfe_eth tx, normal case or both OF since
+		 * last time
+		 */
+			pfe->tmu_credit.tx_credit[client->id][queue] =
+			pfe->tmu_credit.tx_credit_max[client->id][queue] -
+			(pfe->tmu_credit.tx_packets[client->id][queue] -
+			tmu_tx_packets);
+		}
+	}
+}
+
+int pfe_hif_lib_init(struct pfe *pfe)
+{
+	int rc;
+
+	pr_info("%s\n", __func__);
+
+	if (lro_mode) {
+		page_mode = 1;
+		pfe_pkt_size = min(PAGE_SIZE, MAX_PFE_PKT_SIZE);
+		pfe_pkt_headroom = 0;
+	} else {
+		page_mode = 0;
+		pfe_pkt_size = PFE_PKT_SIZE;
+		pfe_pkt_headroom = PFE_PKT_HEADROOM;
+	}
+
+	if (tx_qos)
+		emac_txq_cnt = EMAC_TXQ_CNT / 2;
+	else
+		emac_txq_cnt = EMAC_TXQ_CNT;
+
+	hif_lib_tmu_credit_init(pfe);
+	pfe->hif.shm = &ghif_shm;
+	rc = pfe_hif_shm_init(pfe->hif.shm);
+
+	return rc;
+}
+
+void pfe_hif_lib_exit(struct pfe *pfe)
+{
+	pr_info("%s\n", __func__);
+
+	pfe_hif_shm_clean(pfe->hif.shm);
+}