path: root/drivers/staging/fsl_qbman/qman_high.c

/* Copyright 2008-2012 Freescale Semiconductor, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *     * Neither the name of Freescale Semiconductor nor the
 *       names of its contributors may be used to endorse or promote products
 *       derived from this software without specific prior written permission.
 *
 *
 * ALTERNATIVELY, this software may be distributed under the terms of the
 * GNU General Public License ("GPL") as published by the Free Software
 * Foundation, either version 2 of that License or (at your option) any
 * later version.
 *
 * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "qman_low.h"

/* Compilation constants */
#define DQRR_MAXFILL	15
#define EQCR_ITHRESH	4	/* if EQCR congests, interrupt threshold */
#define IRQNAME		"QMan portal %d"
#define MAX_IRQNAME	16	/* big enough for "QMan portal %d" */

/* Divide 'n' by 'd', rounding down if 'r' is negative, rounding up if it's
 * positive, and rounding to the closest value if it's zero. NB, this macro
 * implicitly upgrades parameters to unsigned 64-bit, so feed it with types
 * that are compatible with this. NB, these arguments should not be expressions
 * unless it is safe for them to be evaluated multiple times. Eg. do not pass
 * in "some_value++" as a parameter to the macro! */
#define ROUNDING(n, d, r) \
	(((r) < 0) ? div64_u64((n), (d)) : \
	(((r) > 0) ? div64_u64(((n) + (d) - 1), (d)) : \
	div64_u64(((n) + ((d) / 2)), (d))))

/* Lock/unlock frame queues, subject to the "LOCKED" flag. This is about
 * inter-processor locking only. Note, FQLOCK() is always called either under a
 * local_irq_save() or from interrupt context - hence there's no need for irq
 * protection (and indeed, attempting to nest irq-protection doesn't work, as
 * the "irq en/disable" machinery isn't recursive...). */
#define FQLOCK(fq) \
	do { \
		struct qman_fq *__fq478 = (fq); \
		if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \
			spin_lock(&__fq478->fqlock); \
	} while (0)
#define FQUNLOCK(fq) \
	do { \
		struct qman_fq *__fq478 = (fq); \
		if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \
			spin_unlock(&__fq478->fqlock); \
	} while (0)

static inline void fq_set(struct qman_fq *fq, u32 mask)
{
	set_bits(mask, &fq->flags);
}
static inline void fq_clear(struct qman_fq *fq, u32 mask)
{
	clear_bits(mask, &fq->flags);
}
static inline int fq_isset(struct qman_fq *fq, u32 mask)
{
	return fq->flags & mask;
}
static inline int fq_isclear(struct qman_fq *fq, u32 mask)
{
	return !(fq->flags & mask);
}

struct qman_portal {
	struct qm_portal p;
	unsigned long bits; /* PORTAL_BITS_*** - dynamic, strictly internal */
	unsigned long irq_sources;
	u32 use_eqcr_ci_stashing;
	u32 slowpoll;	/* only used when interrupts are off */
	struct qman_fq *vdqcr_owned; /* only 1 volatile dequeue at a time */
#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
	struct qman_fq *eqci_owned; /* only 1 enqueue WAIT_SYNC at a time */
#endif
#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
	raw_spinlock_t sharing_lock; /* only used if is_shared */
	int is_shared;
	struct qman_portal *sharing_redirect;
#endif
	u32 sdqcr;
	int dqrr_disable_ref;
	/* A portal-specific handler for DCP ERNs. If this is NULL, the global
	 * handler is called instead. */
	qman_cb_dc_ern cb_dc_ern;
	/* When the cpu-affine portal is activated, this is non-NULL */
	const struct qm_portal_config *config;
	/* This is needed for providing a non-NULL device to dma_map_***() */
	struct platform_device *pdev;
	struct dpa_rbtree retire_table;
	char irqname[MAX_IRQNAME];
	/* 2-element array. cgrs[0] is mask, cgrs[1] is snapshot. */
	struct qman_cgrs *cgrs;
	/* linked-list of CSCN handlers. */
	struct list_head cgr_cbs;
	/* list lock */
	spinlock_t cgr_lock;
	/* 2-element array. ccgrs[0] is mask, ccgrs[1] is snapshot. */
	struct qman_ccgrs *ccgrs[QMAN_CEETM_MAX];
	/* 256-element array, each is a linked-list of CCSCN handlers. */
	struct list_head ccgr_cbs[QMAN_CEETM_MAX];
	/* list lock */
	spinlock_t ccgr_lock;
	/* track if memory was allocated by the driver */
	u8 alloced;
	/* power management data */
	u32 save_isdr;
};

#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
#define PORTAL_IRQ_LOCK(p, irqflags) \
	do { \
		if ((p)->is_shared) \
			raw_spin_lock_irqsave(&(p)->sharing_lock, irqflags); \
		else \
			local_irq_save(irqflags); \
	} while (0)
#define PORTAL_IRQ_UNLOCK(p, irqflags) \
	do { \
		if ((p)->is_shared) \
			raw_spin_unlock_irqrestore(&(p)->sharing_lock, \
						   irqflags); \
		else \
			local_irq_restore(irqflags); \
	} while (0)
#else
#define PORTAL_IRQ_LOCK(p, irqflags) local_irq_save(irqflags)
#define PORTAL_IRQ_UNLOCK(p, irqflags) local_irq_restore(irqflags)
#endif

/* Global handler for DCP ERNs. Used when the portal receiving the message does
 * not have a portal-specific handler. */
static qman_cb_dc_ern cb_dc_ern;

static cpumask_t affine_mask;
static DEFINE_SPINLOCK(affine_mask_lock);
static u16 affine_channels[NR_CPUS];
static DEFINE_PER_CPU(struct qman_portal, qman_affine_portal);
void *affine_portals[NR_CPUS];

/* "raw" gets the cpu-local struct whether it's a redirect or not. */
static inline struct qman_portal *get_raw_affine_portal(void)
{
	return &get_cpu_var(qman_affine_portal);
}
/* For ops that can redirect, this obtains the portal to use */
#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
static inline struct qman_portal *get_affine_portal(void)
{
	struct qman_portal *p = get_raw_affine_portal();
	if (p->sharing_redirect)
		return p->sharing_redirect;
	return p;
}
#else
#define get_affine_portal() get_raw_affine_portal()
#endif
/* For every "get", there must be a "put" */
static inline void put_affine_portal(void)
{
	put_cpu_var(qman_affine_portal);
}
/* Exception: poll functions assume the caller is cpu-affine and in no risk of
 * re-entrance, which are the two reasons we usually use the get/put_cpu_var()
 * semantic - ie. to disable pre-emption. Some use-cases expect the execution
 * context to remain as non-atomic during poll-triggered callbacks as it was
 * when the poll API was first called (eg. NAPI), so we go out of our way in
 * this case to not disable pre-emption. */
static inline struct qman_portal *get_poll_portal(void)
{
	return &__get_cpu_var(qman_affine_portal);
}
#define put_poll_portal()

/* This gives a FQID->FQ lookup to cover the fact that we can't directly demux
 * retirement notifications (the fact they are sometimes h/w-consumed means that
 * contextB isn't always a s/w demux - and as we can't know which case it is
 * when looking at the notification, we have to use the slow lookup for all of
 * them). NB, it's possible to have multiple FQ objects refer to the same FQID
 * (though at most one of them should be the consumer), so this table isn't for
 * all FQs - FQs are added when retirement commands are issued, and removed when
 * they complete, which also massively reduces the size of this table. */
IMPLEMENT_DPA_RBTREE(fqtree, struct qman_fq, node, fqid);

/* This is what everything can wait on, even if it migrates to a different cpu
 * to the one whose affine portal it is waiting on. */
static DECLARE_WAIT_QUEUE_HEAD(affine_queue);

static inline int table_push_fq(struct qman_portal *p, struct qman_fq *fq)
{
	int ret = fqtree_push(&p->retire_table, fq);
	if (ret)
		pr_err("ERROR: double FQ-retirement %d\n", fq->fqid);
	return ret;
}

static inline void table_del_fq(struct qman_portal *p, struct qman_fq *fq)
{
	fqtree_del(&p->retire_table, fq);
}

static inline struct qman_fq *table_find_fq(struct qman_portal *p, u32 fqid)
{
	return fqtree_find(&p->retire_table, fqid);
}

#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
static void **qman_fq_lookup_table;
static size_t qman_fq_lookup_table_size;

int qman_setup_fq_lookup_table(size_t num_entries)
{
	num_entries++;
	/* Allocate 1 more entry since the first entry is not used */
	qman_fq_lookup_table = vzalloc((num_entries * sizeof(void *)));
	if (!qman_fq_lookup_table) {
		pr_err("QMan: Could not allocate fq lookup table\n");
		return -ENOMEM;
	}
	qman_fq_lookup_table_size = num_entries;
	pr_info("QMan: Allocated lookup table at %p, entry count %lu\n",
			qman_fq_lookup_table,
			(unsigned long)qman_fq_lookup_table_size);
	return 0;
}

/* global structure that maintains fq object mapping */
static DEFINE_SPINLOCK(fq_hash_table_lock);

static int find_empty_fq_table_entry(u32 *entry, struct qman_fq *fq)
{
	u32 i;

	spin_lock(&fq_hash_table_lock);
	/* Can't use index zero because this has special meaning
	 * in context_b field. */
	for (i = 1; i < qman_fq_lookup_table_size; i++) {
		if (qman_fq_lookup_table[i] == NULL) {
			*entry = i;
			qman_fq_lookup_table[i] = fq;
			spin_unlock(&fq_hash_table_lock);
			return 0;
		}
	}
	spin_unlock(&fq_hash_table_lock);
	return -ENOMEM;
}

static void clear_fq_table_entry(u32 entry)
{
	spin_lock(&fq_hash_table_lock);
	BUG_ON(entry >= qman_fq_lookup_table_size);
	qman_fq_lookup_table[entry] = NULL;
	spin_unlock(&fq_hash_table_lock);
}

static inline struct qman_fq *get_fq_table_entry(u32 entry)
{
	BUG_ON(entry >= qman_fq_lookup_table_size);
	return qman_fq_lookup_table[entry];
}
#endif

/* In the case that slow- and fast-path handling are both done by qman_poll()
 * (ie. because there is no interrupt handling), we ought to balance how often
 * we do the fast-path poll versus the slow-path poll. We'll use two decrementer
 * sources, so we call the fast poll 'n' times before calling the slow poll
 * once. The idle decrementer constant is used when the last slow-poll detected
 * no work to do, and the busy decrementer constant when the last slow-poll had
 * work to do. */
#define SLOW_POLL_IDLE   1000
#define SLOW_POLL_BUSY   10
static u32 __poll_portal_slow(struct qman_portal *p, u32 is);
static inline unsigned int __poll_portal_fast(struct qman_portal *p,
					unsigned int poll_limit);

/* Portal interrupt handler */
static irqreturn_t portal_isr(__always_unused int irq, void *ptr)
{
	struct qman_portal *p = ptr;
	/*
	 * The CSCI/CCSCI source is cleared inside __poll_portal_slow(), because
	 * it could race against a Query Congestion State command also given
	 * as part of the handling of this interrupt source. We mustn't
	 * clear it a second time in this top-level function.
	 */
	u32 clear = QM_DQAVAIL_MASK | (p->irq_sources &
		~(QM_PIRQ_CSCI | QM_PIRQ_CCSCI));
	u32 is = qm_isr_status_read(&p->p) & p->irq_sources;
	/* DQRR-handling if it's interrupt-driven */
	if (is & QM_PIRQ_DQRI)
		__poll_portal_fast(p, CONFIG_FSL_QMAN_POLL_LIMIT);
	/* Handling of anything else that's interrupt-driven */
	clear |= __poll_portal_slow(p, is);
	qm_isr_status_clear(&p->p, clear);
	return IRQ_HANDLED;
}

/* This inner version is used privately by qman_create_affine_portal(), as well
 * as by the exported qman_stop_dequeues(). */
static inline void qman_stop_dequeues_ex(struct qman_portal *p)
{
	unsigned long irqflags __maybe_unused;
	PORTAL_IRQ_LOCK(p, irqflags);
	if (!(p->dqrr_disable_ref++))
		qm_dqrr_set_maxfill(&p->p, 0);
	PORTAL_IRQ_UNLOCK(p, irqflags);
}

static int drain_mr(struct qm_portal *p)
{
	const struct qm_mr_entry *msg;
loop:
	msg = qm_mr_current(p);
	if (!msg) {
		/* if MR was full and h/w had other FQRNI entries to produce, we
		 * need to allow it time to produce those entries once the
		 * existing entries are consumed. A worst-case situation
		 * (fully-loaded system) means h/w sequencers may have to do 3-4
		 * other things before servicing the portal's MR pump, each of
		 * which (if slow) may take ~50 qman cycles (which is ~200
		 * processor cycles). So rounding up and then multiplying this
		 * worst-case estimate by a factor of 10, just to be
		 * ultra-paranoid, goes as high as 10,000 cycles. NB, we consume
		 * one entry at a time, so h/w has an opportunity to produce new
		 * entries well before the ring has been fully consumed, so
		 * we're being *really* paranoid here. */
		u64 now, then = mfatb();
		do {
			now = mfatb();
		} while ((then + 10000) > now);
		msg = qm_mr_current(p);
		if (!msg)
			return 0;
	}
	qm_mr_next(p);
	qm_mr_cci_consume(p, 1);
	goto loop;
}

#ifdef CONFIG_SUSPEND
static int _qman_portal_suspend_noirq(struct device *dev)
{
	struct qman_portal *p = (struct qman_portal *)dev->platform_data;
#ifdef CONFIG_PM_DEBUG
	struct platform_device *pdev = to_platform_device(dev);
#endif

	p->save_isdr = qm_isr_disable_read(&p->p);
	qm_isr_disable_write(&p->p, 0xffffffff);
	qm_isr_status_clear(&p->p, 0xffffffff);
#ifdef CONFIG_PM_DEBUG
	pr_info("Suspend for %s\n", pdev->name);
#endif
	return 0;
}

static int _qman_portal_resume_noirq(struct device *dev)
{
	struct qman_portal *p = (struct qman_portal *)dev->platform_data;

	/* restore isdr */
	qm_isr_disable_write(&p->p, p->save_isdr);
	return 0;
}
#else
#define _qman_portal_suspend_noirq NULL
#define _qman_portal_resume_noirq NULL
#endif

struct dev_pm_domain qman_portal_device_pm_domain = {
	.ops = {
		USE_PLATFORM_PM_SLEEP_OPS
		.suspend_noirq = _qman_portal_suspend_noirq,
		.resume_noirq = _qman_portal_resume_noirq,
	}
};

struct qman_portal *qman_create_portal(
			struct qman_portal *portal,
			const struct qm_portal_config *config,
			const struct qman_cgrs *cgrs)
{
	struct qm_portal *__p;
	char buf[16];
	int ret;
	u32 isdr;

	if (!portal) {
		portal = kmalloc(sizeof(*portal), GFP_KERNEL);
		if (!portal)
			return portal;
		portal->alloced = 1;
	} else
		portal->alloced = 0;

	__p = &portal->p;

	portal->use_eqcr_ci_stashing = ((qman_ip_rev >= QMAN_REV30) ?
								1 : 0);

	/* prep the low-level portal struct with the mapped addresses from the
	 * config, everything that follows depends on it and "config" is more
	 * for (de)reference... */
	__p->addr.addr_ce = config->addr_virt[DPA_PORTAL_CE];
	__p->addr.addr_ci = config->addr_virt[DPA_PORTAL_CI];
	/*
	 * If CI-stashing is used, the current defaults use a threshold of 3,
	 * and stash with high-than-DQRR priority.
	 */
	if (qm_eqcr_init(__p, qm_eqcr_pvb,
			portal->use_eqcr_ci_stashing ? 3 : 0, 1)) {
		pr_err("Qman EQCR initialisation failed\n");
		goto fail_eqcr;
	}
	if (qm_dqrr_init(__p, config, qm_dqrr_dpush, qm_dqrr_pvb,
			qm_dqrr_cdc, DQRR_MAXFILL)) {
		pr_err("Qman DQRR initialisation failed\n");
		goto fail_dqrr;
	}
	if (qm_mr_init(__p, qm_mr_pvb, qm_mr_cci)) {
		pr_err("Qman MR initialisation failed\n");
		goto fail_mr;
	}
	if (qm_mc_init(__p)) {
		pr_err("Qman MC initialisation failed\n");
		goto fail_mc;
	}
	if (qm_isr_init(__p)) {
		pr_err("Qman ISR initialisation failed\n");
		goto fail_isr;
	}
	/* static interrupt-gating controls */
	qm_dqrr_set_ithresh(__p, CONFIG_FSL_QMAN_PIRQ_DQRR_ITHRESH);
	qm_mr_set_ithresh(__p, CONFIG_FSL_QMAN_PIRQ_MR_ITHRESH);
	qm_isr_set_iperiod(__p, CONFIG_FSL_QMAN_PIRQ_IPERIOD);
	portal->cgrs = kmalloc(2 * sizeof(*cgrs), GFP_KERNEL);
	if (!portal->cgrs)
		goto fail_cgrs;
	/* initial snapshot is no-depletion */
	qman_cgrs_init(&portal->cgrs[1]);
	if (cgrs)
		portal->cgrs[0] = *cgrs;
	else
		/* if the given mask is NULL, assume all CGRs can be seen */
		qman_cgrs_fill(&portal->cgrs[0]);
	INIT_LIST_HEAD(&portal->cgr_cbs);
	spin_lock_init(&portal->cgr_lock);
	if (num_ceetms) {
		for (ret = 0; ret < num_ceetms; ret++) {
			portal->ccgrs[ret] = kmalloc(2 *
				sizeof(struct qman_ccgrs), GFP_KERNEL);
			if (!portal->ccgrs[ret])
				goto fail_ccgrs;
			qman_ccgrs_init(&portal->ccgrs[ret][1]);
			qman_ccgrs_fill(&portal->ccgrs[ret][0]);
			INIT_LIST_HEAD(&portal->ccgr_cbs[ret]);
		}
	}
	spin_lock_init(&portal->ccgr_lock);
	portal->bits = 0;
	portal->slowpoll = 0;
#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
	portal->eqci_owned = NULL;
#endif
#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
	raw_spin_lock_init(&portal->sharing_lock);
	portal->is_shared = config->public_cfg.is_shared;
	portal->sharing_redirect = NULL;
#endif
	portal->sdqcr = QM_SDQCR_SOURCE_CHANNELS | QM_SDQCR_COUNT_UPTO3 |
			QM_SDQCR_DEDICATED_PRECEDENCE | QM_SDQCR_TYPE_PRIO_QOS |
			QM_SDQCR_TOKEN_SET(0xab) | QM_SDQCR_CHANNELS_DEDICATED;
	portal->dqrr_disable_ref = 0;
	portal->cb_dc_ern = NULL;
	sprintf(buf, "qportal-%d", config->public_cfg.channel);
	portal->pdev = platform_device_alloc(buf, -1);
	if (!portal->pdev)
		goto fail_devalloc;
	if (dma_set_mask(&portal->pdev->dev, DMA_BIT_MASK(40)))
		goto fail_devadd;
	portal->pdev->dev.pm_domain = &qman_portal_device_pm_domain;
	portal->pdev->dev.platform_data = portal;
	ret = platform_device_add(portal->pdev);
	if (ret)
		goto fail_devadd;
	dpa_rbtree_init(&portal->retire_table);
	isdr = 0xffffffff;
	qm_isr_disable_write(__p, isdr);
	portal->irq_sources = 0;
	qm_isr_enable_write(__p, portal->irq_sources);
	qm_isr_status_clear(__p, 0xffffffff);
	snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, config->public_cfg.cpu);
	if (request_irq(config->public_cfg.irq, portal_isr, 0, portal->irqname,
				portal)) {
		pr_err("request_irq() failed\n");
		goto fail_irq;
	}
	if ((config->public_cfg.cpu != -1) &&
			irq_can_set_affinity(config->public_cfg.irq) &&
			irq_set_affinity(config->public_cfg.irq,
				cpumask_of(config->public_cfg.cpu))) {
		pr_err("irq_set_affinity() failed\n");
		goto fail_affinity;
	}

	/* Need EQCR to be empty before continuing */
	isdr ^= QM_PIRQ_EQCI;
	qm_isr_disable_write(__p, isdr);
	ret = qm_eqcr_get_fill(__p);
	if (ret) {
		pr_err("Qman EQCR unclean\n");
		goto fail_eqcr_empty;
	}
	isdr ^= (QM_PIRQ_DQRI | QM_PIRQ_MRI);
	qm_isr_disable_write(__p, isdr);
	while (qm_dqrr_current(__p) != NULL)
		qm_dqrr_cdc_consume_n(__p, 0xffff);
	drain_mr(__p);
	/* Success */
	portal->config = config;
	qm_isr_disable_write(__p, 0);
	qm_isr_uninhibit(__p);
	/* Write a sane SDQCR */
	qm_dqrr_sdqcr_set(__p, portal->sdqcr);
	return portal;
fail_eqcr_empty:
fail_affinity:
	free_irq(config->public_cfg.irq, portal);
fail_irq:
	platform_device_del(portal->pdev);
fail_devadd:
	platform_device_put(portal->pdev);
fail_devalloc:
	if (num_ceetms)
		for (ret = 0; ret < num_ceetms; ret++)
			kfree(portal->ccgrs[ret]);
fail_ccgrs:
	kfree(portal->cgrs);
fail_cgrs:
	qm_isr_finish(__p);
fail_isr:
	qm_mc_finish(__p);
fail_mc:
	qm_mr_finish(__p);
fail_mr:
	qm_dqrr_finish(__p);
fail_dqrr:
	qm_eqcr_finish(__p);
fail_eqcr:
	if (portal->alloced)
		kfree(portal);
	return NULL;
}

struct qman_portal *qman_create_affine_portal(
			const struct qm_portal_config *config,
			const struct qman_cgrs *cgrs)
{
	struct qman_portal *res;
	struct qman_portal *portal;

	portal = &per_cpu(qman_affine_portal, config->public_cfg.cpu);
	res = qman_create_portal(portal, config, cgrs);
	if (res) {
		spin_lock(&affine_mask_lock);
		cpumask_set_cpu(config->public_cfg.cpu, &affine_mask);
		affine_channels[config->public_cfg.cpu] =
			config->public_cfg.channel;
		affine_portals[config->public_cfg.cpu] = portal;
		spin_unlock(&affine_mask_lock);
	}
	return res;
}

/* These checks are BUG_ON()s because the driver is already supposed to avoid
 * these cases. */
struct qman_portal *qman_create_affine_slave(struct qman_portal *redirect,
								int cpu)
{
#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
	struct qman_portal *p;
	p = &per_cpu(qman_affine_portal, cpu);
	/* Check that we don't already have our own portal */
	BUG_ON(p->config);
	/* Check that we aren't already slaving to another portal */
	BUG_ON(p->is_shared);
	/* Check that 'redirect' is prepared to have us */
	BUG_ON(!redirect->config->public_cfg.is_shared);
	/* These are the only elements to initialise when redirecting */
	p->irq_sources = 0;
	p->sharing_redirect = redirect;
	affine_portals[cpu] = p;
	return p;
#else
	BUG();
	return NULL;
#endif
}

void qman_destroy_portal(struct qman_portal *qm)
{
	const struct qm_portal_config *pcfg;
	int i;

	/* Stop dequeues on the portal */
	qm_dqrr_sdqcr_set(&qm->p, 0);

	/* NB we do this to "quiesce" EQCR. If we add enqueue-completions or
	 * something related to QM_PIRQ_EQCI, this may need fixing.
	 * Also, due to the prefetching model used for CI updates in the enqueue
	 * path, this update will only invalidate the CI cacheline *after*
	 * working on it, so we need to call this twice to ensure a full update
	 * irrespective of where the enqueue processing was at when the teardown
	 * began. */
	qm_eqcr_cce_update(&qm->p);
	qm_eqcr_cce_update(&qm->p);
	pcfg = qm->config;

	free_irq(pcfg->public_cfg.irq, qm);

	kfree(qm->cgrs);
	if (num_ceetms)
		for (i = 0; i < num_ceetms; i++)
			kfree(qm->ccgrs[i]);
	qm_isr_finish(&qm->p);
	qm_mc_finish(&qm->p);
	qm_mr_finish(&qm->p);
	qm_dqrr_finish(&qm->p);
	qm_eqcr_finish(&qm->p);

	platform_device_del(qm->pdev);
	platform_device_put(qm->pdev);

	qm->config = NULL;
	if (qm->alloced)
		kfree(qm);
}

const struct qm_portal_config *qman_destroy_affine_portal(void)
{
	/* We don't want to redirect if we're a slave, use "raw" */
	struct qman_portal *qm = get_raw_affine_portal();
	const struct qm_portal_config *pcfg;
	int cpu;
#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
	if (qm->sharing_redirect) {
		qm->sharing_redirect = NULL;
		put_affine_portal();
		return NULL;
	}
	qm->is_shared = 0;
#endif
	pcfg = qm->config;
	cpu = pcfg->public_cfg.cpu;

	qman_destroy_portal(qm);

	spin_lock(&affine_mask_lock);
	cpumask_clear_cpu(cpu, &affine_mask);
	spin_unlock(&affine_mask_lock);
	put_affine_portal();
	return pcfg;
}

const struct qman_portal_config *qman_p_get_portal_config(struct qman_portal *p)
{
	return &p->config->public_cfg;
}
EXPORT_SYMBOL(qman_p_get_portal_config);

const struct qman_portal_config *qman_get_portal_config(void)
{
	struct qman_portal *p = get_affine_portal();
	const struct qman_portal_config *ret = qman_p_get_portal_config(p);
	put_affine_portal();
	return ret;
}
EXPORT_SYMBOL(qman_get_portal_config);

/* Inline helper to reduce nesting in __poll_portal_slow() */
static inline void fq_state_change(struct qman_portal *p, struct qman_fq *fq,
				const struct qm_mr_entry *msg, u8 verb)
{
	FQLOCK(fq);
	switch (verb) {
	case QM_MR_VERB_FQRL:
		DPA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_ORL));
		fq_clear(fq, QMAN_FQ_STATE_ORL);
		table_del_fq(p, fq);
		break;
	case QM_MR_VERB_FQRN:
		DPA_ASSERT((fq->state == qman_fq_state_parked) ||
			(fq->state == qman_fq_state_sched));
		DPA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_CHANGING));
		fq_clear(fq, QMAN_FQ_STATE_CHANGING);
		if (msg->fq.fqs & QM_MR_FQS_NOTEMPTY)
			fq_set(fq, QMAN_FQ_STATE_NE);
		if (msg->fq.fqs & QM_MR_FQS_ORLPRESENT)
			fq_set(fq, QMAN_FQ_STATE_ORL);
		else
			table_del_fq(p, fq);
		fq->state = qman_fq_state_retired;
		break;
	case QM_MR_VERB_FQPN:
		DPA_ASSERT(fq->state == qman_fq_state_sched);
		DPA_ASSERT(fq_isclear(fq, QMAN_FQ_STATE_CHANGING));
		fq->state = qman_fq_state_parked;
	}
	FQUNLOCK(fq);
}

static u32 __poll_portal_slow(struct qman_portal *p, u32 is)
{
	const struct qm_mr_entry *msg;

	if (is & QM_PIRQ_CSCI) {
		struct qman_cgrs rr, c;
		struct qm_mc_result *mcr;
		struct qman_cgr *cgr;
		unsigned long irqflags __maybe_unused;

		spin_lock_irqsave(&p->cgr_lock, irqflags);
		/*
		 * The CSCI bit must be cleared _before_ issuing the
		 * Query Congestion State command, to ensure that a long
		 * CGR State Change callback cannot miss an intervening
		 * state change.
		 */
		qm_isr_status_clear(&p->p, QM_PIRQ_CSCI);
		qm_mc_start(&p->p);
		qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION);
		while (!(mcr = qm_mc_result(&p->p)))
			cpu_relax();
		/* mask out the ones I'm not interested in */
		qman_cgrs_and(&rr, (const struct qman_cgrs *)
			&mcr->querycongestion.state, &p->cgrs[0]);
		/* check previous snapshot for delta, enter/exit congestion */
		qman_cgrs_xor(&c, &rr, &p->cgrs[1]);
		/* update snapshot */
		qman_cgrs_cp(&p->cgrs[1], &rr);
		/* Invoke callback */
		list_for_each_entry(cgr, &p->cgr_cbs, node)
			if (cgr->cb && qman_cgrs_get(&c, cgr->cgrid))
				cgr->cb(p, cgr, qman_cgrs_get(&rr, cgr->cgrid));
		spin_unlock_irqrestore(&p->cgr_lock, irqflags);
	}
	if (is & QM_PIRQ_CCSCI) {
		struct qman_ccgrs rr, c, congestion_result;
		struct qm_mc_result *mcr;
		struct qm_mc_command *mcc;
		struct qm_ceetm_ccg *ccg;
		unsigned long irqflags __maybe_unused;
		int i, j;

		spin_lock_irqsave(&p->ccgr_lock, irqflags);
		/*
		 * The CCSCI bit must be cleared _before_ issuing the
		 * Query Congestion State command, to ensure that a long
		 * CCGR State Change callback cannot miss an intervening
		 * state change.
		 */
		qm_isr_status_clear(&p->p, QM_PIRQ_CCSCI);

		for (i = 0; i < num_ceetms; i++) {
			for (j = 0; j < 2; j++) {
				mcc = qm_mc_start(&p->p);
				mcc->ccgr_query.ccgrid =
					CEETM_QUERY_CONGESTION_STATE | j;
				mcc->ccgr_query.dcpid = i;
				qm_mc_commit(&p->p, QM_CEETM_VERB_CCGR_QUERY);
				while (!(mcr = qm_mc_result(&p->p)))
					cpu_relax();
				congestion_result.q[j] =
					mcr->ccgr_query.congestion_state.state;
			}
			/* mask out the ones I'm not interested in */
			qman_ccgrs_and(&rr, &congestion_result,
							&p->ccgrs[i][0]);
			/*
			 * check previous snapshot for delta, enter/exit
			 * congestion.
			 */
			qman_ccgrs_xor(&c, &rr, &p->ccgrs[i][1]);
			/* update snapshot */
			qman_ccgrs_cp(&p->ccgrs[i][1], &rr);
			/* Invoke callback */
			list_for_each_entry(ccg, &p->ccgr_cbs[i], cb_node)
				if (ccg->cb && qman_ccgrs_get(&c,
					(ccg->parent->idx << 4) | ccg->idx))
					ccg->cb(ccg, ccg->cb_ctx,
						qman_ccgrs_get(&rr,
							(ccg->parent->idx << 4)
							| ccg->idx));
		}
		spin_unlock_irqrestore(&p->ccgr_lock, irqflags);
	}

#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
	if (is & QM_PIRQ_EQCI) {
		unsigned long irqflags;
		PORTAL_IRQ_LOCK(p, irqflags);
		p->eqci_owned = NULL;
		PORTAL_IRQ_UNLOCK(p, irqflags);
		wake_up(&affine_queue);
	}
#endif

	if (is & QM_PIRQ_EQRI) {
		unsigned long irqflags __maybe_unused;
		PORTAL_IRQ_LOCK(p, irqflags);
		qm_eqcr_cce_update(&p->p);
		qm_eqcr_set_ithresh(&p->p, 0);
		PORTAL_IRQ_UNLOCK(p, irqflags);
		wake_up(&affine_queue);
	}

	if (is & QM_PIRQ_MRI) {
		struct qman_fq *fq;
		u8 verb, num = 0;
mr_loop:
		qm_mr_pvb_update(&p->p);
		msg = qm_mr_current(&p->p);
		if (!msg)
			goto mr_done;
		verb = msg->verb & QM_MR_VERB_TYPE_MASK;
		/* The message is a software ERN iff the 0x20 bit is set */
		if (verb & 0x20) {
			switch (verb) {
			case QM_MR_VERB_FQRNI:
				/* nada, we drop FQRNIs on the floor */
				break;
			case QM_MR_VERB_FQRN:
			case QM_MR_VERB_FQRL:
				/* Lookup in the retirement table */
				fq = table_find_fq(p, msg->fq.fqid);
				BUG_ON(!fq);
				fq_state_change(p, fq, msg, verb);
				if (fq->cb.fqs)
					fq->cb.fqs(p, fq, msg);
				break;
			case QM_MR_VERB_FQPN:
				/* Parked */
#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
				fq = get_fq_table_entry(msg->fq.contextB);
#else
				fq = (void *)(uintptr_t)msg->fq.contextB;
#endif
				fq_state_change(p, fq, msg, verb);
				if (fq->cb.fqs)
					fq->cb.fqs(p, fq, msg);
				break;
			case QM_MR_VERB_DC_ERN:
				/* DCP ERN */
				if (p->cb_dc_ern)
					p->cb_dc_ern(p, msg);
				else if (cb_dc_ern)
					cb_dc_ern(p, msg);
				else {
					static int warn_once;
					if (!warn_once) {
						pr_crit("Leaking DCP ERNs!\n");
						warn_once = 1;
					}
				}
				break;
			default:
				pr_crit("Invalid MR verb 0x%02x\n", verb);
			}
		} else {
			/* Its a software ERN */
#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
			fq = get_fq_table_entry(msg->ern.tag);
#else
			fq = (void *)(uintptr_t)msg->ern.tag;
#endif
			fq->cb.ern(p, fq, msg);
		}
		num++;
		qm_mr_next(&p->p);
		goto mr_loop;
mr_done:
		qm_mr_cci_consume(&p->p, num);
	}
	/*
	 * QM_PIRQ_CSCI/CCSCI has already been cleared, as part of its specific
	 * processing. If that interrupt source has meanwhile been re-asserted,
	 * we mustn't clear it here (or in the top-level interrupt handler).
	 */
	return is & (QM_PIRQ_EQCI | QM_PIRQ_EQRI | QM_PIRQ_MRI);
}

/* remove some slowish-path stuff from the "fast path" and make sure it isn't
 * inlined. */
static noinline void clear_vdqcr(struct qman_portal *p, struct qman_fq *fq)
{
	p->vdqcr_owned = NULL;
	FQLOCK(fq);
	fq_clear(fq, QMAN_FQ_STATE_VDQCR);
	FQUNLOCK(fq);
	wake_up(&affine_queue);
}

/* Look: no locks, no irq_save()s, no preempt_disable()s! :-) The only states
 * that would conflict with other things if they ran at the same time on the
 * same cpu are;
 *
 *   (i) setting/clearing vdqcr_owned, and
 *  (ii) clearing the NE (Not Empty) flag.
 *
 * Both are safe. Because;
 *
 *   (i) this clearing can only occur after qman_volatile_dequeue() has set the
 *       vdqcr_owned field (which it does before setting VDQCR), and
 *       qman_volatile_dequeue() blocks interrupts and preemption while this is
 *       done so that we can't interfere.
 *  (ii) the NE flag is only cleared after qman_retire_fq() has set it, and as
 *       with (i) that API prevents us from interfering until it's safe.
 *
 * The good thing is that qman_volatile_dequeue() and qman_retire_fq() run far
 * less frequently (ie. per-FQ) than __poll_portal_fast() does, so the nett
 * advantage comes from this function not having to "lock" anything at all.
 *
 * Note also that the callbacks are invoked at points which are safe against the
 * above potential conflicts, but that this function itself is not re-entrant
 * (this is because the function tracks one end of each FIFO in the portal and
 * we do *not* want to lock that). So the consequence is that it is safe for
 * user callbacks to call into any Qman API *except* qman_poll() (as that's the
 * sole API that could be invoking the callback through this function).
 */
static inline unsigned int __poll_portal_fast(struct qman_portal *p,
					unsigned int poll_limit)
{
	const struct qm_dqrr_entry *dq;
	struct qman_fq *fq;
	enum qman_cb_dqrr_result res;
	unsigned int limit = 0;

loop:
	qm_dqrr_pvb_update(&p->p);
	dq = qm_dqrr_current(&p->p);
	if (!dq)
		goto done;
	if (dq->stat & QM_DQRR_STAT_UNSCHEDULED) {
		/* VDQCR: don't trust contextB as the FQ may have been
		 * configured for h/w consumption and we're draining it
		 * post-retirement. */
		fq = p->vdqcr_owned;
		/* We only set QMAN_FQ_STATE_NE when retiring, so we only need
		 * to check for clearing it when doing volatile dequeues. It's
		 * one less thing to check in the critical path (SDQCR). */
		if (dq->stat & QM_DQRR_STAT_FQ_EMPTY)
			fq_clear(fq, QMAN_FQ_STATE_NE);
		/* this is duplicated from the SDQCR code, but we have stuff to
		 * do before *and* after this callback, and we don't want
		 * multiple if()s in the critical path (SDQCR). */
		res = fq->cb.dqrr(p, fq, dq);
		if (res == qman_cb_dqrr_stop)
			goto done;
		/* Check for VDQCR completion */
		if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED)
			clear_vdqcr(p, fq);
	} else {
		/* SDQCR: contextB points to the FQ */
#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
		fq = get_fq_table_entry(dq->contextB);
#else
		fq = (void *)(uintptr_t)dq->contextB;
#endif
		/* Now let the callback do its stuff */
		res = fq->cb.dqrr(p, fq, dq);
		/* The callback can request that we exit without consuming this
		 * entry nor advancing; */
		if (res == qman_cb_dqrr_stop)
			goto done;
	}
	/* Interpret 'dq' from a driver perspective. */
	/* Parking isn't possible unless HELDACTIVE was set. NB,
	 * FORCEELIGIBLE implies HELDACTIVE, so we only need to
	 * check for HELDACTIVE to cover both. */
	DPA_ASSERT((dq->stat & QM_DQRR_STAT_FQ_HELDACTIVE) ||
		(res != qman_cb_dqrr_park));
	/* Defer just means "skip it, I'll consume it myself later on" */
	if (res != qman_cb_dqrr_defer)
		qm_dqrr_cdc_consume_1ptr(&p->p, dq, (res == qman_cb_dqrr_park));
	/* Move forward */
	qm_dqrr_next(&p->p);
	/* Entry processed and consumed, increment our counter. The callback can
	 * request that we exit after consuming the entry, and we also exit if
	 * we reach our processing limit, so loop back only if neither of these
	 * conditions is met. */
	if ((++limit < poll_limit) && (res != qman_cb_dqrr_consume_stop))
		goto loop;
done:
	return limit;
}

u32 qman_irqsource_get(void)
{
	/* "irqsource" and "poll" APIs mustn't redirect when sharing, they
	 * should shut the user out if they are not the primary CPU hosting the
	 * portal. That's why we use the "raw" interface. */
	struct qman_portal *p = get_raw_affine_portal();
	u32 ret = p->irq_sources & QM_PIRQ_VISIBLE;
	put_affine_portal();
	return ret;
}
EXPORT_SYMBOL(qman_irqsource_get);

int qman_p_irqsource_add(struct qman_portal *p, u32 bits __maybe_unused)
{
	__maybe_unused unsigned long irqflags;
#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
	if (p->sharing_redirect)
		return -EINVAL;
	else
#endif
	{
		PORTAL_IRQ_LOCK(p, irqflags);
		set_bits(bits & QM_PIRQ_VISIBLE, &p->irq_sources);
		qm_isr_enable_write(&p->p, p->irq_sources);
		PORTAL_IRQ_UNLOCK(p, irqflags);
	}
	return 0;
}
EXPORT_SYMBOL(qman_p_irqsource_add);

int qman_irqsource_add(u32 bits __maybe_unused)
{
	struct qman_portal *p = get_raw_affine_portal();
	int ret;
	ret = qman_p_irqsource_add(p, bits);
	put_affine_portal();
	return ret;
}
EXPORT_SYMBOL(qman_irqsource_add);

int qman_p_irqsource_remove(struct qman_portal *p, u32 bits)
{
	__maybe_unused unsigned long irqflags;
	u32 ier;
#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
	if (p->sharing_redirect) {
		put_affine_portal();
		return -EINVAL;
	}
#endif
	/* Our interrupt handler only processes+clears status register bits that
	 * are in p->irq_sources. As we're trimming that mask, if one of them
	 * were to assert in the status register just before we remove it from
	 * the enable register, there would be an interrupt-storm when we
	 * release the IRQ lock. So we wait for the enable register update to
	 * take effect in h/w (by reading it back) and then clear all other bits
	 * in the status register. Ie. we clear them from ISR once it's certain
	 * IER won't allow them to reassert. */
	PORTAL_IRQ_LOCK(p, irqflags);
	bits &= QM_PIRQ_VISIBLE;
	clear_bits(bits, &p->irq_sources);
	qm_isr_enable_write(&p->p, p->irq_sources);
	ier = qm_isr_enable_read(&p->p);
	/* Using "~ier" (rather than "bits" or "~p->irq_sources") creates a
	 * data-dependency, ie. to protect against re-ordering. */
	qm_isr_status_clear(&p->p, ~ier);
	PORTAL_IRQ_UNLOCK(p, irqflags);
	return 0;
}
EXPORT_SYMBOL(qman_p_irqsource_remove);

int qman_irqsource_remove(u32 bits)
{
	struct qman_portal *p = get_raw_affine_portal();
	int ret;
	ret = qman_p_irqsource_remove(p, bits);
	put_affine_portal();
	return ret;
}
EXPORT_SYMBOL(qman_irqsource_remove);

const cpumask_t *qman_affine_cpus(void)
{
	return &affine_mask;
}
EXPORT_SYMBOL(qman_affine_cpus);

u16 qman_affine_channel(int cpu)
{
	if (cpu < 0) {
		struct qman_portal *portal = get_raw_affine_portal();
#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
		BUG_ON(portal->sharing_redirect);
#endif
		cpu = portal->config->public_cfg.cpu;
		put_affine_portal();
	}
	BUG_ON(!cpumask_test_cpu(cpu, &affine_mask));
	return affine_channels[cpu];
}
EXPORT_SYMBOL(qman_affine_channel);

void *qman_get_affine_portal(int cpu)
{
	return affine_portals[cpu];
}
EXPORT_SYMBOL(qman_get_affine_portal);

int qman_p_poll_dqrr(struct qman_portal *p, unsigned int limit)
{
	int ret;

#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
	if (unlikely(p->sharing_redirect))
		ret = -EINVAL;
	else
#endif
	{
		BUG_ON(p->irq_sources & QM_PIRQ_DQRI);
		ret = __poll_portal_fast(p, limit);
	}
	return ret;
}
EXPORT_SYMBOL(qman_p_poll_dqrr);

int qman_poll_dqrr(unsigned int limit)
{
	struct qman_portal *p = get_poll_portal();
	int ret;
	ret = qman_p_poll_dqrr(p, limit);
	put_poll_portal();
	return ret;
}
EXPORT_SYMBOL(qman_poll_dqrr);

u32 qman_p_poll_slow(struct qman_portal *p)
{
	u32 ret;
#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
	if (unlikely(p->sharing_redirect))
		ret = (u32)-1;
	else
#endif
	{
		u32 is = qm_isr_status_read(&p->p) & ~p->irq_sources;
		ret = __poll_portal_slow(p, is);
		qm_isr_status_clear(&p->p, ret);
	}
	return ret;
}
EXPORT_SYMBOL(qman_p_poll_slow);

u32 qman_poll_slow(void)
{
	struct qman_portal *p = get_poll_portal();
	u32 ret;
	ret = qman_p_poll_slow(p);
	put_poll_portal();
	return ret;
}
EXPORT_SYMBOL(qman_poll_slow);

/* Legacy wrapper */
void qman_p_poll(struct qman_portal *p)
{
#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
	if (unlikely(p->sharing_redirect))
		return;
#endif
	if ((~p->irq_sources) & QM_PIRQ_SLOW) {
		if (!(p->slowpoll--)) {
			u32 is = qm_isr_status_read(&p->p) & ~p->irq_sources;
			u32 active = __poll_portal_slow(p, is);
			if (active) {
				qm_isr_status_clear(&p->p, active);
				p->slowpoll = SLOW_POLL_BUSY;
			} else
				p->slowpoll = SLOW_POLL_IDLE;
		}
	}
	if ((~p->irq_sources) & QM_PIRQ_DQRI)
		__poll_portal_fast(p, CONFIG_FSL_QMAN_POLL_LIMIT);
}
EXPORT_SYMBOL(qman_p_poll);

void qman_poll(void)
{
	struct qman_portal *p = get_poll_portal();
	qman_p_poll(p);
	put_poll_portal();
}
EXPORT_SYMBOL(qman_poll);

void qman_p_stop_dequeues(struct qman_portal *p)
{
	qman_stop_dequeues_ex(p);
}
EXPORT_SYMBOL(qman_p_stop_dequeues);

void qman_stop_dequeues(void)
{
	struct qman_portal *p = get_affine_portal();
	qman_p_stop_dequeues(p);
	put_affine_portal();
}
EXPORT_SYMBOL(qman_stop_dequeues);

void qman_p_start_dequeues(struct qman_portal *p)
{
	unsigned long irqflags __maybe_unused;
	PORTAL_IRQ_LOCK(p, irqflags);
	DPA_ASSERT(p->dqrr_disable_ref > 0);
	if (!(--p->dqrr_disable_ref))
		qm_dqrr_set_maxfill(&p->p, DQRR_MAXFILL);
	PORTAL_IRQ_UNLOCK(p, irqflags);
}
EXPORT_SYMBOL(qman_p_start_dequeues);

void qman_start_dequeues(void)
{
	struct qman_portal *p = get_affine_portal();
	qman_p_start_dequeues(p);
	put_affine_portal();
}
EXPORT_SYMBOL(qman_start_dequeues);

void qman_p_static_dequeue_add(struct qman_portal *p, u32 pools)
{
	unsigned long irqflags __maybe_unused;
	PORTAL_IRQ_LOCK(p, irqflags);
	pools &= p->config->public_cfg.pools;
	p->sdqcr |= pools;
	qm_dqrr_sdqcr_set(&p->p, p->sdqcr);
	PORTAL_IRQ_UNLOCK(p, irqflags);
}
EXPORT_SYMBOL(qman_p_static_dequeue_add);

void qman_static_dequeue_add(u32 pools)
{
	struct qman_portal *p = get_affine_portal();
	qman_p_static_dequeue_add(p, pools);
	put_affine_portal();
}
EXPORT_SYMBOL(qman_static_dequeue_add);

void qman_p_static_dequeue_del(struct qman_portal *p, u32 pools)
{
	unsigned long irqflags __maybe_unused;
	PORTAL_IRQ_LOCK(p, irqflags);
	pools &= p->config->public_cfg.pools;
	p->sdqcr &= ~pools;
	qm_dqrr_sdqcr_set(&p->p, p->sdqcr);
	PORTAL_IRQ_UNLOCK(p, irqflags);
}
EXPORT_SYMBOL(qman_p_static_dequeue_del);

void qman_static_dequeue_del(u32 pools)
{
	struct qman_portal *p = get_affine_portal();
	qman_p_static_dequeue_del(p, pools);
	put_affine_portal();
}
EXPORT_SYMBOL(qman_static_dequeue_del);

u32 qman_p_static_dequeue_get(struct qman_portal *p)
{
	return p->sdqcr;
}
EXPORT_SYMBOL(qman_p_static_dequeue_get);

u32 qman_static_dequeue_get(void)
{
	struct qman_portal *p = get_affine_portal();
	u32 ret = qman_p_static_dequeue_get(p);
	put_affine_portal();
	return ret;
}
EXPORT_SYMBOL(qman_static_dequeue_get);

void qman_p_dca(struct qman_portal *p, struct qm_dqrr_entry *dq,
						int park_request)
{
	qm_dqrr_cdc_consume_1ptr(&p->p, dq, park_request);
}
EXPORT_SYMBOL(qman_p_dca);

void qman_dca(struct qm_dqrr_entry *dq, int park_request)
{
	struct qman_portal *p = get_affine_portal();
	qman_p_dca(p, dq, park_request);
	put_affine_portal();
}
EXPORT_SYMBOL(qman_dca);

/*******************/
/* Frame queue API */
/*******************/

static const char *mcr_result_str(u8 result)
{
	switch (result) {
	case QM_MCR_RESULT_NULL:
		return "QM_MCR_RESULT_NULL";
	case QM_MCR_RESULT_OK:
		return "QM_MCR_RESULT_OK";
	case QM_MCR_RESULT_ERR_FQID:
		return "QM_MCR_RESULT_ERR_FQID";
	case QM_MCR_RESULT_ERR_FQSTATE:
		return "QM_MCR_RESULT_ERR_FQSTATE";
	case QM_MCR_RESULT_ERR_NOTEMPTY:
		return "QM_MCR_RESULT_ERR_NOTEMPTY";
	case QM_MCR_RESULT_PENDING:
		return "QM_MCR_RESULT_PENDING";
	case QM_MCR_RESULT_ERR_BADCOMMAND:
		return "QM_MCR_RESULT_ERR_BADCOMMAND";
	}
	return "<unknown MCR result>";
}

int qman_create_fq(u32 fqid, u32 flags, struct qman_fq *fq)
{
	struct qm_fqd fqd;
	struct qm_mcr_queryfq_np np;
	struct qm_mc_command *mcc;
	struct qm_mc_result *mcr;
	struct qman_portal *p;
	unsigned long irqflags __maybe_unused;

	if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID) {
		int ret = qman_alloc_fqid(&fqid);
		if (ret)
			return ret;
	}
	spin_lock_init(&fq->fqlock);
	fq->fqid = fqid;
	fq->flags = flags;
	fq->state = qman_fq_state_oos;
	fq->cgr_groupid = 0;
#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
	if (unlikely(find_empty_fq_table_entry(&fq->key, fq)))
		return -ENOMEM;
#endif
	if (!(flags & QMAN_FQ_FLAG_AS_IS) || (flags & QMAN_FQ_FLAG_NO_MODIFY))
		return 0;
	/* Everything else is AS_IS support */
	p = get_affine_portal();
	PORTAL_IRQ_LOCK(p, irqflags);
	mcc = qm_mc_start(&p->p);
	mcc->queryfq.fqid = fqid;
	qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ);
	while (!(mcr = qm_mc_result(&p->p)))
		cpu_relax();
	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ);
	if (mcr->result != QM_MCR_RESULT_OK) {
		pr_err("QUERYFQ failed: %s\n", mcr_result_str(mcr->result));
		goto err;
	}
	fqd = mcr->queryfq.fqd;
	mcc = qm_mc_start(&p->p);
	mcc->queryfq_np.fqid = fqid;
	qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
	while (!(mcr = qm_mc_result(&p->p)))
		cpu_relax();
	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ_NP);
	if (mcr->result != QM_MCR_RESULT_OK) {
		pr_err("QUERYFQ_NP failed: %s\n", mcr_result_str(mcr->result));
		goto err;
	}
	np = mcr->queryfq_np;
	/* Phew, have queryfq and queryfq_np results, stitch together
	 * the FQ object from those. */
	fq->cgr_groupid = fqd.cgid;
	switch (np.state & QM_MCR_NP_STATE_MASK) {
	case QM_MCR_NP_STATE_OOS:
		break;
	case QM_MCR_NP_STATE_RETIRED:
		fq->state = qman_fq_state_retired;
		if (np.frm_cnt)
			fq_set(fq, QMAN_FQ_STATE_NE);
		break;
	case QM_MCR_NP_STATE_TEN_SCHED:
	case QM_MCR_NP_STATE_TRU_SCHED:
	case QM_MCR_NP_STATE_ACTIVE:
		fq->state = qman_fq_state_sched;
		if (np.state & QM_MCR_NP_STATE_R)
			fq_set(fq, QMAN_FQ_STATE_CHANGING);
		break;
	case QM_MCR_NP_STATE_PARKED:
		fq->state = qman_fq_state_parked;
		break;
	default:
		DPA_ASSERT(NULL == "invalid FQ state");
	}
	if (fqd.fq_ctrl & QM_FQCTRL_CGE)
		fq->state |= QMAN_FQ_STATE_CGR_EN;
	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();
	return 0;
err:
	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();
	if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID)
		qman_release_fqid(fqid);
	return -EIO;
}
EXPORT_SYMBOL(qman_create_fq);

void qman_destroy_fq(struct qman_fq *fq, u32 flags __maybe_unused)
{

	/* We don't need to lock the FQ as it is a pre-condition that the FQ be
	 * quiesced. Instead, run some checks. */
	switch (fq->state) {
	case qman_fq_state_parked:
		DPA_ASSERT(flags & QMAN_FQ_DESTROY_PARKED);
	case qman_fq_state_oos:
		if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID))
			qman_release_fqid(fq->fqid);
#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
		clear_fq_table_entry(fq->key);
#endif
		return;
	default:
		break;
	}
	DPA_ASSERT(NULL == "qman_free_fq() on unquiesced FQ!");
}
EXPORT_SYMBOL(qman_destroy_fq);

u32 qman_fq_fqid(struct qman_fq *fq)
{
	return fq->fqid;
}
EXPORT_SYMBOL(qman_fq_fqid);

void qman_fq_state(struct qman_fq *fq, enum qman_fq_state *state, u32 *flags)
{
	if (state)
		*state = fq->state;
	if (flags)
		*flags = fq->flags;
}
EXPORT_SYMBOL(qman_fq_state);

int qman_init_fq(struct qman_fq *fq, u32 flags, struct qm_mcc_initfq *opts)
{
	struct qm_mc_command *mcc;
	struct qm_mc_result *mcr;
	struct qman_portal *p;
	unsigned long irqflags __maybe_unused;
	u8 res, myverb = (flags & QMAN_INITFQ_FLAG_SCHED) ?
		QM_MCC_VERB_INITFQ_SCHED : QM_MCC_VERB_INITFQ_PARKED;

	if ((fq->state != qman_fq_state_oos) &&
			(fq->state != qman_fq_state_parked))
		return -EINVAL;
#ifdef CONFIG_FSL_DPA_CHECKING
	if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
		return -EINVAL;
#endif
	if (opts && (opts->we_mask & QM_INITFQ_WE_OAC)) {
		/* And can't be set at the same time as TDTHRESH */
		if (opts->we_mask & QM_INITFQ_WE_TDTHRESH)
			return -EINVAL;
	}
	/* Issue an INITFQ_[PARKED|SCHED] management command */
	p = get_affine_portal();
	PORTAL_IRQ_LOCK(p, irqflags);
	FQLOCK(fq);
	if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
			((fq->state != qman_fq_state_oos) &&
				(fq->state != qman_fq_state_parked)))) {
		FQUNLOCK(fq);
		PORTAL_IRQ_UNLOCK(p, irqflags);
		put_affine_portal();
		return -EBUSY;
	}
	mcc = qm_mc_start(&p->p);
	if (opts)
		mcc->initfq = *opts;
	mcc->initfq.fqid = fq->fqid;
	mcc->initfq.count = 0;
	/* If the FQ does *not* have the TO_DCPORTAL flag, contextB is set as a
	 * demux pointer. Otherwise, the caller-provided value is allowed to
	 * stand, don't overwrite it. */
	if (fq_isclear(fq, QMAN_FQ_FLAG_TO_DCPORTAL)) {
		dma_addr_t phys_fq;
		mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTB;
#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
		mcc->initfq.fqd.context_b = fq->key;
#else
		mcc->initfq.fqd.context_b = (u32)(uintptr_t)fq;
#endif
		/* and the physical address - NB, if the user wasn't trying to
		 * set CONTEXTA, clear the stashing settings. */
		if (!(mcc->initfq.we_mask & QM_INITFQ_WE_CONTEXTA)) {
			mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTA;
			memset(&mcc->initfq.fqd.context_a, 0,
				sizeof(mcc->initfq.fqd.context_a));
		} else {
			phys_fq = dma_map_single(&p->pdev->dev, fq, sizeof(*fq),
						DMA_TO_DEVICE);
			qm_fqd_stashing_set64(&mcc->initfq.fqd, phys_fq);
		}
	}
	if (flags & QMAN_INITFQ_FLAG_LOCAL) {
		mcc->initfq.fqd.dest.channel = p->config->public_cfg.channel;
		if (!(mcc->initfq.we_mask & QM_INITFQ_WE_DESTWQ)) {
			mcc->initfq.we_mask |= QM_INITFQ_WE_DESTWQ;
			mcc->initfq.fqd.dest.wq = 4;
		}
	}
	qm_mc_commit(&p->p, myverb);
	while (!(mcr = qm_mc_result(&p->p)))
		cpu_relax();
	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb);
	res = mcr->result;
	if (res != QM_MCR_RESULT_OK) {
		FQUNLOCK(fq);
		PORTAL_IRQ_UNLOCK(p, irqflags);
		put_affine_portal();
		return -EIO;
	}
	if (opts) {
		if (opts->we_mask & QM_INITFQ_WE_FQCTRL) {
			if (opts->fqd.fq_ctrl & QM_FQCTRL_CGE)
				fq_set(fq, QMAN_FQ_STATE_CGR_EN);
			else
				fq_clear(fq, QMAN_FQ_STATE_CGR_EN);
		}
		if (opts->we_mask & QM_INITFQ_WE_CGID)
			fq->cgr_groupid = opts->fqd.cgid;
	}
	fq->state = (flags & QMAN_INITFQ_FLAG_SCHED) ?
			qman_fq_state_sched : qman_fq_state_parked;
	FQUNLOCK(fq);
	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();
	return 0;
}
EXPORT_SYMBOL(qman_init_fq);

int qman_schedule_fq(struct qman_fq *fq)
{
	struct qm_mc_command *mcc;
	struct qm_mc_result *mcr;
	struct qman_portal *p;
	unsigned long irqflags __maybe_unused;
	int ret = 0;
	u8 res;

	if (fq->state != qman_fq_state_parked)
		return -EINVAL;
#ifdef CONFIG_FSL_DPA_CHECKING
	if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
		return -EINVAL;
#endif
	/* Issue a ALTERFQ_SCHED management command */
	p = get_affine_portal();
	PORTAL_IRQ_LOCK(p, irqflags);
	FQLOCK(fq);
	if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
			(fq->state != qman_fq_state_parked))) {
		ret = -EBUSY;
		goto out;
	}
	mcc = qm_mc_start(&p->p);
	mcc->alterfq.fqid = fq->fqid;
	qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_SCHED);
	while (!(mcr = qm_mc_result(&p->p)))
		cpu_relax();
	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_SCHED);
	res = mcr->result;
	if (res != QM_MCR_RESULT_OK) {
		ret = -EIO;
		goto out;
	}
	fq->state = qman_fq_state_sched;
out:
	FQUNLOCK(fq);
	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();
	return ret;
}
EXPORT_SYMBOL(qman_schedule_fq);

int qman_retire_fq(struct qman_fq *fq, u32 *flags)
{
	struct qm_mc_command *mcc;
	struct qm_mc_result *mcr;
	struct qman_portal *p;
	unsigned long irqflags __maybe_unused;
	int rval;
	u8 res;

	if ((fq->state != qman_fq_state_parked) &&
			(fq->state != qman_fq_state_sched))
		return -EINVAL;
#ifdef CONFIG_FSL_DPA_CHECKING
	if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
		return -EINVAL;
#endif
	p = get_affine_portal();
	PORTAL_IRQ_LOCK(p, irqflags);
	FQLOCK(fq);
	if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
			(fq->state == qman_fq_state_retired) ||
				(fq->state == qman_fq_state_oos))) {
		rval = -EBUSY;
		goto out;
	}
	rval = table_push_fq(p, fq);
	if (rval)
		goto out;
	mcc = qm_mc_start(&p->p);
	mcc->alterfq.fqid = fq->fqid;
	qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_RETIRE);
	while (!(mcr = qm_mc_result(&p->p)))
		cpu_relax();
	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_RETIRE);
	res = mcr->result;
	/* "Elegant" would be to treat OK/PENDING the same way; set CHANGING,
	 * and defer the flags until FQRNI or FQRN (respectively) show up. But
	 * "Friendly" is to process OK immediately, and not set CHANGING. We do
	 * friendly, otherwise the caller doesn't necessarily have a fully
	 * "retired" FQ on return even if the retirement was immediate. However
	 * this does mean some code duplication between here and
	 * fq_state_change(). */
	if (likely(res == QM_MCR_RESULT_OK)) {
		rval = 0;
		/* Process 'fq' right away, we'll ignore FQRNI */
		if (mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY)
			fq_set(fq, QMAN_FQ_STATE_NE);
		if (mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT)
			fq_set(fq, QMAN_FQ_STATE_ORL);
		else
			table_del_fq(p, fq);
		if (flags)
			*flags = fq->flags;
		fq->state = qman_fq_state_retired;
		if (fq->cb.fqs) {
			/* Another issue with supporting "immediate" retirement
			 * is that we're forced to drop FQRNIs, because by the
			 * time they're seen it may already be "too late" (the
			 * fq may have been OOS'd and free()'d already). But if
			 * the upper layer wants a callback whether it's
			 * immediate or not, we have to fake a "MR" entry to
			 * look like an FQRNI... */
			struct qm_mr_entry msg;
			msg.verb = QM_MR_VERB_FQRNI;
			msg.fq.fqs = mcr->alterfq.fqs;
			msg.fq.fqid = fq->fqid;
#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
			msg.fq.contextB = fq->key;
#else
			msg.fq.contextB = (u32)(uintptr_t)fq;
#endif
			fq->cb.fqs(p, fq, &msg);
		}
	} else if (res == QM_MCR_RESULT_PENDING) {
		rval = 1;
		fq_set(fq, QMAN_FQ_STATE_CHANGING);
	} else {
		rval = -EIO;
		table_del_fq(p, fq);
	}
out:
	FQUNLOCK(fq);
	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();
	return rval;
}
EXPORT_SYMBOL(qman_retire_fq);

int qman_oos_fq(struct qman_fq *fq)
{
	struct qm_mc_command *mcc;
	struct qm_mc_result *mcr;
	struct qman_portal *p;
	unsigned long irqflags __maybe_unused;
	int ret = 0;
	u8 res;

	if (fq->state != qman_fq_state_retired)
		return -EINVAL;
#ifdef CONFIG_FSL_DPA_CHECKING
	if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
		return -EINVAL;
#endif
	p = get_affine_portal();
	PORTAL_IRQ_LOCK(p, irqflags);
	FQLOCK(fq);
	if (unlikely((fq_isset(fq, QMAN_FQ_STATE_BLOCKOOS)) ||
			(fq->state != qman_fq_state_retired))) {
		ret = -EBUSY;
		goto out;
	}
	mcc = qm_mc_start(&p->p);
	mcc->alterfq.fqid = fq->fqid;
	qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_OOS);
	while (!(mcr = qm_mc_result(&p->p)))
		cpu_relax();
	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_OOS);
	res = mcr->result;
	if (res != QM_MCR_RESULT_OK) {
		ret = -EIO;
		goto out;
	}
	fq->state = qman_fq_state_oos;
out:
	FQUNLOCK(fq);
	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();
	return ret;
}
EXPORT_SYMBOL(qman_oos_fq);

int qman_fq_flow_control(struct qman_fq *fq, int xon)
{
	struct qm_mc_command *mcc;
	struct qm_mc_result *mcr;
	struct qman_portal *p;
	unsigned long irqflags __maybe_unused;
	int ret = 0;
	u8 res;
	u8 myverb;

	if ((fq->state == qman_fq_state_oos) ||
		(fq->state == qman_fq_state_retired) ||
		(fq->state == qman_fq_state_parked))
		return -EINVAL;

#ifdef CONFIG_FSL_DPA_CHECKING
	if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
		return -EINVAL;
#endif
	/* Issue a ALTER_FQXON or ALTER_FQXOFF management command */
	p = get_affine_portal();
	PORTAL_IRQ_LOCK(p, irqflags);
	FQLOCK(fq);
	if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
			(fq->state == qman_fq_state_parked) ||
			(fq->state == qman_fq_state_oos) ||
			(fq->state == qman_fq_state_retired))) {
		ret = -EBUSY;
		goto out;
	}
	mcc = qm_mc_start(&p->p);
	mcc->alterfq.fqid = fq->fqid;
	mcc->alterfq.count = 0;
	myverb = xon ? QM_MCC_VERB_ALTER_FQXON : QM_MCC_VERB_ALTER_FQXOFF;

	qm_mc_commit(&p->p, myverb);
	while (!(mcr = qm_mc_result(&p->p)))
		cpu_relax();
	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb);

	res = mcr->result;
	if (res != QM_MCR_RESULT_OK) {
		ret = -EIO;
		goto out;
	}
out:
	FQUNLOCK(fq);
	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();
	return ret;
}
EXPORT_SYMBOL(qman_fq_flow_control);

int qman_query_fq(struct qman_fq *fq, struct qm_fqd *fqd)
{
	struct qm_mc_command *mcc;
	struct qm_mc_result *mcr;
	struct qman_portal *p = get_affine_portal();
	unsigned long irqflags __maybe_unused;
	u8 res;

	PORTAL_IRQ_LOCK(p, irqflags);
	mcc = qm_mc_start(&p->p);
	mcc->queryfq.fqid = fq->fqid;
	qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ);
	while (!(mcr = qm_mc_result(&p->p)))
		cpu_relax();
	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ);
	res = mcr->result;
	if (res == QM_MCR_RESULT_OK)
		*fqd = mcr->queryfq.fqd;
	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();
	if (res != QM_MCR_RESULT_OK)
		return -EIO;
	return 0;
}
EXPORT_SYMBOL(qman_query_fq);

int qman_query_fq_np(struct qman_fq *fq, struct qm_mcr_queryfq_np *np)
{
	struct qm_mc_command *mcc;
	struct qm_mc_result *mcr;
	struct qman_portal *p = get_affine_portal();
	unsigned long irqflags __maybe_unused;
	u8 res;

	PORTAL_IRQ_LOCK(p, irqflags);
	mcc = qm_mc_start(&p->p);
	mcc->queryfq.fqid = fq->fqid;
	qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
	while (!(mcr = qm_mc_result(&p->p)))
		cpu_relax();
	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP);
	res = mcr->result;
	if (res == QM_MCR_RESULT_OK)
		*np = mcr->queryfq_np;
	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();
	if (res == QM_MCR_RESULT_ERR_FQID)
		return -ERANGE;
	else if (res != QM_MCR_RESULT_OK)
		return -EIO;
	return 0;
}
EXPORT_SYMBOL(qman_query_fq_np);

int qman_query_wq(u8 query_dedicated, struct qm_mcr_querywq *wq)
{
	struct qm_mc_command *mcc;
	struct qm_mc_result *mcr;
	struct qman_portal *p = get_affine_portal();
	unsigned long irqflags __maybe_unused;
	u8 res, myverb;

	PORTAL_IRQ_LOCK(p, irqflags);
	myverb = (query_dedicated) ? QM_MCR_VERB_QUERYWQ_DEDICATED :
				 QM_MCR_VERB_QUERYWQ;
	mcc = qm_mc_start(&p->p);
	mcc->querywq.channel.id = wq->channel.id;
	qm_mc_commit(&p->p, myverb);
	while (!(mcr = qm_mc_result(&p->p)))
		cpu_relax();
	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb);
	res = mcr->result;
	if (res == QM_MCR_RESULT_OK)
		*wq = mcr->querywq;
	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();
	if (res != QM_MCR_RESULT_OK) {
		pr_err("QUERYWQ failed: %s\n", mcr_result_str(res));
		return -EIO;
	}
	return 0;
}
EXPORT_SYMBOL(qman_query_wq);

int qman_testwrite_cgr(struct qman_cgr *cgr, u64 i_bcnt,
			struct qm_mcr_cgrtestwrite *result)
{
	struct qm_mc_command *mcc;
	struct qm_mc_result *mcr;
	struct qman_portal *p = get_affine_portal();
	unsigned long irqflags __maybe_unused;
	u8 res;

	PORTAL_IRQ_LOCK(p, irqflags);
	mcc = qm_mc_start(&p->p);
	mcc->cgrtestwrite.cgid = cgr->cgrid;
	mcc->cgrtestwrite.i_bcnt_hi = (u8)(i_bcnt >> 32);
	mcc->cgrtestwrite.i_bcnt_lo = (u32)i_bcnt;
	qm_mc_commit(&p->p, QM_MCC_VERB_CGRTESTWRITE);
	while (!(mcr = qm_mc_result(&p->p)))
		cpu_relax();
	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_CGRTESTWRITE);
	res = mcr->result;
	if (res == QM_MCR_RESULT_OK)
		*result = mcr->cgrtestwrite;
	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();
	if (res != QM_MCR_RESULT_OK) {
		pr_err("CGR TEST WRITE failed: %s\n", mcr_result_str(res));
		return -EIO;
	}
	return 0;
}
EXPORT_SYMBOL(qman_testwrite_cgr);

int qman_query_cgr(struct qman_cgr *cgr, struct qm_mcr_querycgr *cgrd)
{
	struct qm_mc_command *mcc;
	struct qm_mc_result *mcr;
	struct qman_portal *p = get_affine_portal();
	unsigned long irqflags __maybe_unused;
	u8 res;

	PORTAL_IRQ_LOCK(p, irqflags);
	mcc = qm_mc_start(&p->p);
	mcc->querycgr.cgid = cgr->cgrid;
	qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCGR);
	while (!(mcr = qm_mc_result(&p->p)))
		cpu_relax();
	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYCGR);
	res = mcr->result;
	if (res == QM_MCR_RESULT_OK)
		*cgrd = mcr->querycgr;
	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();
	if (res != QM_MCR_RESULT_OK) {
		pr_err("QUERY_CGR failed: %s\n", mcr_result_str(res));
		return -EIO;
	}
	return 0;
}
EXPORT_SYMBOL(qman_query_cgr);

int qman_query_congestion(struct qm_mcr_querycongestion *congestion)
{
	struct qm_mc_result *mcr;
	struct qman_portal *p = get_affine_portal();
	unsigned long irqflags __maybe_unused;
	u8 res;

	PORTAL_IRQ_LOCK(p, irqflags);
	qm_mc_start(&p->p);
	qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION);
	while (!(mcr = qm_mc_result(&p->p)))
		cpu_relax();
	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
			QM_MCC_VERB_QUERYCONGESTION);
	res = mcr->result;
	if (res == QM_MCR_RESULT_OK)
		*congestion = mcr->querycongestion;
	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();
	if (res != QM_MCR_RESULT_OK) {
		pr_err("QUERY_CONGESTION failed: %s\n", mcr_result_str(res));
		return -EIO;
	}
	return 0;
}
EXPORT_SYMBOL(qman_query_congestion);

/* internal function used as a wait_event() expression */
static int set_p_vdqcr(struct qman_portal *p, struct qman_fq *fq, u32 vdqcr)
{
	unsigned long irqflags __maybe_unused;
	int ret = -EBUSY;
	PORTAL_IRQ_LOCK(p, irqflags);
	if (!p->vdqcr_owned) {
		FQLOCK(fq);
		if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
			goto escape;
		fq_set(fq, QMAN_FQ_STATE_VDQCR);
		FQUNLOCK(fq);
		p->vdqcr_owned = fq;
		ret = 0;
	}
escape:
	PORTAL_IRQ_UNLOCK(p, irqflags);
	if (!ret)
		qm_dqrr_vdqcr_set(&p->p, vdqcr);
	return ret;
}

static int set_vdqcr(struct qman_portal **p, struct qman_fq *fq, u32 vdqcr)
{
	int ret;
	*p = get_affine_portal();
	ret = set_p_vdqcr(*p, fq, vdqcr);
	put_affine_portal();
	return ret;
}

#ifdef CONFIG_FSL_DPA_CAN_WAIT
static int wait_p_vdqcr_start(struct qman_portal *p, struct qman_fq *fq,
				u32 vdqcr, u32 flags)
{
	int ret = 0;
	if (flags & QMAN_VOLATILE_FLAG_WAIT_INT)
		ret = wait_event_interruptible(affine_queue,
				!(ret = set_p_vdqcr(p, fq, vdqcr)));
	else
		wait_event(affine_queue, !(ret = set_p_vdqcr(p, fq, vdqcr)));
	return ret;
}

static int wait_vdqcr_start(struct qman_portal **p, struct qman_fq *fq,
				u32 vdqcr, u32 flags)
{
	int ret = 0;
	if (flags & QMAN_VOLATILE_FLAG_WAIT_INT)
		ret = wait_event_interruptible(affine_queue,
				!(ret = set_vdqcr(p, fq, vdqcr)));
	else
		wait_event(affine_queue, !(ret = set_vdqcr(p, fq, vdqcr)));
	return ret;
}
#endif

int qman_p_volatile_dequeue(struct qman_portal *p, struct qman_fq *fq,
					u32 flags __maybe_unused, u32 vdqcr)
{
	int ret;

	if ((fq->state != qman_fq_state_parked) &&
			(fq->state != qman_fq_state_retired))
		return -EINVAL;
	if (vdqcr & QM_VDQCR_FQID_MASK)
		return -EINVAL;
	if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
		return -EBUSY;
	vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid;
#ifdef CONFIG_FSL_DPA_CAN_WAIT
	if (flags & QMAN_VOLATILE_FLAG_WAIT)
		ret = wait_p_vdqcr_start(p, fq, vdqcr, flags);
	else
#endif
		ret = set_p_vdqcr(p, fq, vdqcr);
	if (ret)
		return ret;
	/* VDQCR is set */
#ifdef CONFIG_FSL_DPA_CAN_WAIT
	if (flags & QMAN_VOLATILE_FLAG_FINISH) {
		if (flags & QMAN_VOLATILE_FLAG_WAIT_INT)
			/* NB: don't propagate any error - the caller wouldn't
			 * know whether the VDQCR was issued or not. A signal
			 * could arrive after returning anyway, so the caller
			 * can check signal_pending() if that's an issue. */
			wait_event_interruptible(affine_queue,
				!fq_isset(fq, QMAN_FQ_STATE_VDQCR));
		else
			wait_event(affine_queue,
				!fq_isset(fq, QMAN_FQ_STATE_VDQCR));
	}
#endif
	return 0;
}
EXPORT_SYMBOL(qman_p_volatile_dequeue);

int qman_volatile_dequeue(struct qman_fq *fq, u32 flags __maybe_unused,
				u32 vdqcr)
{
	struct qman_portal *p;
	int ret;

	if ((fq->state != qman_fq_state_parked) &&
			(fq->state != qman_fq_state_retired))
		return -EINVAL;
	if (vdqcr & QM_VDQCR_FQID_MASK)
		return -EINVAL;
	if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
		return -EBUSY;
	vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid;
#ifdef CONFIG_FSL_DPA_CAN_WAIT
	if (flags & QMAN_VOLATILE_FLAG_WAIT)
		ret = wait_vdqcr_start(&p, fq, vdqcr, flags);
	else
#endif
		ret = set_vdqcr(&p, fq, vdqcr);
	if (ret)
		return ret;
	/* VDQCR is set */
#ifdef CONFIG_FSL_DPA_CAN_WAIT
	if (flags & QMAN_VOLATILE_FLAG_FINISH) {
		if (flags & QMAN_VOLATILE_FLAG_WAIT_INT)
			/* NB: don't propagate any error - the caller wouldn't
			 * know whether the VDQCR was issued or not. A signal
			 * could arrive after returning anyway, so the caller
			 * can check signal_pending() if that's an issue. */
			wait_event_interruptible(affine_queue,
				!fq_isset(fq, QMAN_FQ_STATE_VDQCR));
		else
			wait_event(affine_queue,
				!fq_isset(fq, QMAN_FQ_STATE_VDQCR));
	}
#endif
	return 0;
}
EXPORT_SYMBOL(qman_volatile_dequeue);

static noinline void update_eqcr_ci(struct qman_portal *p, u8 avail)
{
	if (avail)
		qm_eqcr_cce_prefetch(&p->p);
	else
		qm_eqcr_cce_update(&p->p);
}

int qman_eqcr_is_empty(void)
{
	unsigned long irqflags __maybe_unused;
	struct qman_portal *p = get_affine_portal();
	u8 avail;

	PORTAL_IRQ_LOCK(p, irqflags);
	update_eqcr_ci(p, 0);
	avail = qm_eqcr_get_fill(&p->p);
	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();
	return avail == 0;
}
EXPORT_SYMBOL(qman_eqcr_is_empty);

void qman_set_dc_ern(qman_cb_dc_ern handler, int affine)
{
	if (affine) {
		unsigned long irqflags __maybe_unused;
		struct qman_portal *p = get_affine_portal();
		PORTAL_IRQ_LOCK(p, irqflags);
		p->cb_dc_ern = handler;
		PORTAL_IRQ_UNLOCK(p, irqflags);
		put_affine_portal();
	} else
		cb_dc_ern = handler;
}
EXPORT_SYMBOL(qman_set_dc_ern);

static inline struct qm_eqcr_entry *try_p_eq_start(struct qman_portal *p,
					unsigned long *irqflags __maybe_unused,
					struct qman_fq *fq,
					const struct qm_fd *fd,
					u32 flags)
{
	struct qm_eqcr_entry *eq;
	u8 avail;
	PORTAL_IRQ_LOCK(p, (*irqflags));
#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
	if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) &&
			(flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) {
		if (p->eqci_owned) {
			PORTAL_IRQ_UNLOCK(p, (*irqflags));
			return NULL;
		}
		p->eqci_owned = fq;
	}
#endif
	if (p->use_eqcr_ci_stashing) {
		/*
		 * The stashing case is easy, only update if we need to in
		 * order to try and liberate ring entries.
		 */
		eq = qm_eqcr_start_stash(&p->p);
	} else {
		/*
		 * The non-stashing case is harder, need to prefetch ahead of
		 * time.
		 */
		avail = qm_eqcr_get_avail(&p->p);
		if (avail < 2)
			update_eqcr_ci(p, avail);
		eq = qm_eqcr_start_no_stash(&p->p);
	}

	if (unlikely(!eq)) {
#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
		if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) &&
				(flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC)))
			p->eqci_owned = NULL;
#endif
		PORTAL_IRQ_UNLOCK(p, (*irqflags));
		return NULL;
	}
	if (flags & QMAN_ENQUEUE_FLAG_DCA)
		eq->dca = QM_EQCR_DCA_ENABLE |
			((flags & QMAN_ENQUEUE_FLAG_DCA_PARK) ?
					QM_EQCR_DCA_PARK : 0) |
			((flags >> 8) & QM_EQCR_DCA_IDXMASK);
	eq->fqid = fq->fqid;
#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
	eq->tag = fq->key;
#else
	eq->tag = (u32)(uintptr_t)fq;
#endif
	eq->fd = *fd;
	return eq;
}

static inline struct qm_eqcr_entry *try_eq_start(struct qman_portal **p,
					unsigned long *irqflags __maybe_unused,
					struct qman_fq *fq,
					const struct qm_fd *fd,
					u32 flags)
{
	struct qm_eqcr_entry *eq;
	*p = get_affine_portal();
	eq = try_p_eq_start(*p, irqflags, fq, fd, flags);
	if (!eq)
		put_affine_portal();
	return eq;
}

#ifdef CONFIG_FSL_DPA_CAN_WAIT
static noinline struct qm_eqcr_entry *__wait_eq_start(struct qman_portal **p,
					unsigned long *irqflags __maybe_unused,
					struct qman_fq *fq,
					const struct qm_fd *fd,
					u32 flags)
{
	struct qm_eqcr_entry *eq = try_eq_start(p, irqflags, fq, fd, flags);
	if (!eq)
		qm_eqcr_set_ithresh(&(*p)->p, EQCR_ITHRESH);
	return eq;
}
static noinline struct qm_eqcr_entry *wait_eq_start(struct qman_portal **p,
					unsigned long *irqflags __maybe_unused,
					struct qman_fq *fq,
					const struct qm_fd *fd,
					u32 flags)
{
	struct qm_eqcr_entry *eq;
	if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT)
		/* NB: return NULL if signal occurs before completion. Signal
		 * can occur during return. Caller must check for signal */
		wait_event_interruptible(affine_queue,
			(eq = __wait_eq_start(p, irqflags, fq, fd, flags)));
	else
		wait_event(affine_queue,
			(eq = __wait_eq_start(p, irqflags, fq, fd, flags)));
	return eq;
}
static noinline struct qm_eqcr_entry *__wait_p_eq_start(struct qman_portal *p,
					unsigned long *irqflags __maybe_unused,
					struct qman_fq *fq,
					const struct qm_fd *fd,
					u32 flags)
{
	struct qm_eqcr_entry *eq = try_p_eq_start(p, irqflags, fq, fd, flags);
	if (!eq)
		qm_eqcr_set_ithresh(&p->p, EQCR_ITHRESH);
	return eq;
}
static noinline struct qm_eqcr_entry *wait_p_eq_start(struct qman_portal *p,
					unsigned long *irqflags __maybe_unused,
					struct qman_fq *fq,
					const struct qm_fd *fd,
					u32 flags)
{
	struct qm_eqcr_entry *eq;
	if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT)
		/* NB: return NULL if signal occurs before completion. Signal
		 * can occur during return. Caller must check for signal */
		wait_event_interruptible(affine_queue,
			(eq = __wait_p_eq_start(p, irqflags, fq, fd, flags)));
	else
		wait_event(affine_queue,
			(eq = __wait_p_eq_start(p, irqflags, fq, fd, flags)));
	return eq;
}
#endif

int qman_p_enqueue(struct qman_portal *p, struct qman_fq *fq,
				const struct qm_fd *fd, u32 flags)
{
	struct qm_eqcr_entry *eq;
	unsigned long irqflags __maybe_unused;

#ifdef CONFIG_FSL_DPA_CAN_WAIT
	if (flags & QMAN_ENQUEUE_FLAG_WAIT)
		eq = wait_p_eq_start(p, &irqflags, fq, fd, flags);
	else
#endif
	eq = try_p_eq_start(p, &irqflags, fq, fd, flags);
	if (!eq)
		return -EBUSY;
	/* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */
	qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE |
		(flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT)));
	/* Factor the below out, it's used from qman_enqueue_orp() too */
	PORTAL_IRQ_UNLOCK(p, irqflags);
#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
	if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) &&
			(flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) {
		if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT)
			/* NB: return success even if signal occurs before
			 * condition is true. pvb_commit guarantees success */
			wait_event_interruptible(affine_queue,
					(p->eqci_owned != fq));
		else
			wait_event(affine_queue, (p->eqci_owned != fq));
	}
#endif
	return 0;
}
EXPORT_SYMBOL(qman_p_enqueue);

int qman_enqueue(struct qman_fq *fq, const struct qm_fd *fd, u32 flags)
{
	struct qman_portal *p;
	struct qm_eqcr_entry *eq;
	unsigned long irqflags __maybe_unused;

#ifdef CONFIG_FSL_DPA_CAN_WAIT
	if (flags & QMAN_ENQUEUE_FLAG_WAIT)
		eq = wait_eq_start(&p, &irqflags, fq, fd, flags);
	else
#endif
	eq = try_eq_start(&p, &irqflags, fq, fd, flags);
	if (!eq)
		return -EBUSY;
	/* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */
	qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE |
		(flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT)));
	/* Factor the below out, it's used from qman_enqueue_orp() too */
	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();
#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
	if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) &&
			(flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) {
		if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT)
			/* NB: return success even if signal occurs before
			 * condition is true. pvb_commit guarantees success */
			wait_event_interruptible(affine_queue,
					(p->eqci_owned != fq));
		else
			wait_event(affine_queue, (p->eqci_owned != fq));
	}
#endif
	return 0;
}
EXPORT_SYMBOL(qman_enqueue);

int qman_p_enqueue_orp(struct qman_portal *p, struct qman_fq *fq,
				const struct qm_fd *fd, u32 flags,
				struct qman_fq *orp, u16 orp_seqnum)
{
	struct qm_eqcr_entry *eq;
	unsigned long irqflags __maybe_unused;

#ifdef CONFIG_FSL_DPA_CAN_WAIT
	if (flags & QMAN_ENQUEUE_FLAG_WAIT)
		eq = wait_p_eq_start(p, &irqflags, fq, fd, flags);
	else
#endif
	eq = try_p_eq_start(p, &irqflags, fq, fd, flags);
	if (!eq)
		return -EBUSY;
	/* Process ORP-specifics here */
	if (flags & QMAN_ENQUEUE_FLAG_NLIS)
		orp_seqnum |= QM_EQCR_SEQNUM_NLIS;
	else {
		orp_seqnum &= ~QM_EQCR_SEQNUM_NLIS;
		if (flags & QMAN_ENQUEUE_FLAG_NESN)
			orp_seqnum |= QM_EQCR_SEQNUM_NESN;
		else
			/* No need to check 4 QMAN_ENQUEUE_FLAG_HOLE */
			orp_seqnum &= ~QM_EQCR_SEQNUM_NESN;
	}
	eq->seqnum = orp_seqnum;
	eq->orp = orp->fqid;
	/* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */
	qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_ORP |
		((flags & (QMAN_ENQUEUE_FLAG_HOLE | QMAN_ENQUEUE_FLAG_NESN)) ?
				0 : QM_EQCR_VERB_CMD_ENQUEUE) |
		(flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT)));
	PORTAL_IRQ_UNLOCK(p, irqflags);
#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
	if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) &&
			(flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) {
		if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT)
			/* NB: return success even if signal occurs before
			 * condition is true. pvb_commit guarantees success */
			wait_event_interruptible(affine_queue,
					(p->eqci_owned != fq));
		else
			wait_event(affine_queue, (p->eqci_owned != fq));
	}
#endif
	return 0;
}
EXPORT_SYMBOL(qman_p_enqueue_orp);

int qman_enqueue_orp(struct qman_fq *fq, const struct qm_fd *fd, u32 flags,
			struct qman_fq *orp, u16 orp_seqnum)
{
	struct qman_portal *p;
	struct qm_eqcr_entry *eq;
	unsigned long irqflags __maybe_unused;

#ifdef CONFIG_FSL_DPA_CAN_WAIT
	if (flags & QMAN_ENQUEUE_FLAG_WAIT)
		eq = wait_eq_start(&p, &irqflags, fq, fd, flags);
	else
#endif
	eq = try_eq_start(&p, &irqflags, fq, fd, flags);
	if (!eq)
		return -EBUSY;
	/* Process ORP-specifics here */
	if (flags & QMAN_ENQUEUE_FLAG_NLIS)
		orp_seqnum |= QM_EQCR_SEQNUM_NLIS;
	else {
		orp_seqnum &= ~QM_EQCR_SEQNUM_NLIS;
		if (flags & QMAN_ENQUEUE_FLAG_NESN)
			orp_seqnum |= QM_EQCR_SEQNUM_NESN;
		else
			/* No need to check 4 QMAN_ENQUEUE_FLAG_HOLE */
			orp_seqnum &= ~QM_EQCR_SEQNUM_NESN;
	}
	eq->seqnum = orp_seqnum;
	eq->orp = orp->fqid;
	/* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */
	qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_ORP |
		((flags & (QMAN_ENQUEUE_FLAG_HOLE | QMAN_ENQUEUE_FLAG_NESN)) ?
				0 : QM_EQCR_VERB_CMD_ENQUEUE) |
		(flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT)));
	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();
#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
	if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) &&
			(flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) {
		if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT)
			/* NB: return success even if signal occurs before
			 * condition is true. pvb_commit guarantees success */
			wait_event_interruptible(affine_queue,
					(p->eqci_owned != fq));
		else
			wait_event(affine_queue, (p->eqci_owned != fq));
	}
#endif
	return 0;
}
EXPORT_SYMBOL(qman_enqueue_orp);

int qman_p_enqueue_precommit(struct qman_portal *p, struct qman_fq *fq,
				const struct qm_fd *fd, u32 flags,
				qman_cb_precommit cb, void *cb_arg)
{
	struct qm_eqcr_entry *eq;
	unsigned long irqflags __maybe_unused;

#ifdef CONFIG_FSL_DPA_CAN_WAIT
	if (flags & QMAN_ENQUEUE_FLAG_WAIT)
		eq = wait_p_eq_start(p, &irqflags, fq, fd, flags);
	else
#endif
	eq = try_p_eq_start(p, &irqflags, fq, fd, flags);
	if (!eq)
		return -EBUSY;
	/* invoke user supplied callback function before writing commit verb */
	if (cb(cb_arg)) {
		PORTAL_IRQ_UNLOCK(p, irqflags);
		return -EINVAL;
	}
	/* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */
	qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE |
		(flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT)));
	/* Factor the below out, it's used from qman_enqueue_orp() too */
	PORTAL_IRQ_UNLOCK(p, irqflags);
#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
	if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) &&
			(flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) {
		if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT)
			/* NB: return success even if signal occurs before
			 * condition is true. pvb_commit guarantees success */
			wait_event_interruptible(affine_queue,
					(p->eqci_owned != fq));
		else
			wait_event(affine_queue, (p->eqci_owned != fq));
	}
#endif
	return 0;
}
EXPORT_SYMBOL(qman_p_enqueue_precommit);

int qman_enqueue_precommit(struct qman_fq *fq, const struct qm_fd *fd,
		u32 flags, qman_cb_precommit cb, void *cb_arg)
{
	struct qman_portal *p;
	struct qm_eqcr_entry *eq;
	unsigned long irqflags __maybe_unused;

#ifdef CONFIG_FSL_DPA_CAN_WAIT
	if (flags & QMAN_ENQUEUE_FLAG_WAIT)
		eq = wait_eq_start(&p, &irqflags, fq, fd, flags);
	else
#endif
	eq = try_eq_start(&p, &irqflags, fq, fd, flags);
	if (!eq)
		return -EBUSY;
	/* invoke user supplied callback function before writing commit verb */
	if (cb(cb_arg)) {
		PORTAL_IRQ_UNLOCK(p, irqflags);
		put_affine_portal();
		return -EINVAL;
	}
	/* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */
	qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE |
		(flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT)));
	/* Factor the below out, it's used from qman_enqueue_orp() too */
	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();
#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
	if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) &&
			(flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) {
		if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT)
			/* NB: return success even if signal occurs before
			 * condition is true. pvb_commit guarantees success */
			wait_event_interruptible(affine_queue,
					(p->eqci_owned != fq));
		else
			wait_event(affine_queue, (p->eqci_owned != fq));
	}
#endif
	return 0;
}
EXPORT_SYMBOL(qman_enqueue_precommit);

int qman_modify_cgr(struct qman_cgr *cgr, u32 flags,
			struct qm_mcc_initcgr *opts)
{
	struct qm_mc_command *mcc;
	struct qm_mc_result *mcr;
	struct qman_portal *p = get_affine_portal();
	unsigned long irqflags __maybe_unused;
	u8 res;
	u8 verb = QM_MCC_VERB_MODIFYCGR;

	PORTAL_IRQ_LOCK(p, irqflags);
	mcc = qm_mc_start(&p->p);
	if (opts)
		mcc->initcgr = *opts;
	mcc->initcgr.cgid = cgr->cgrid;
	if (flags & QMAN_CGR_FLAG_USE_INIT)
		verb = QM_MCC_VERB_INITCGR;
	qm_mc_commit(&p->p, verb);
	while (!(mcr = qm_mc_result(&p->p)))
		cpu_relax();
	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == verb);
	res = mcr->result;
	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();
	return (res == QM_MCR_RESULT_OK) ? 0 : -EIO;
}
EXPORT_SYMBOL(qman_modify_cgr);

#define TARG_MASK(n) (0x80000000 >> (n->config->public_cfg.channel - \
					QM_CHANNEL_SWPORTAL0))
#define TARG_DCP_MASK(n) (0x80000000 >> (10 + n))
#define PORTAL_IDX(n) (n->config->public_cfg.channel - QM_CHANNEL_SWPORTAL0)

static u8 qman_cgr_cpus[__CGR_NUM];

int qman_create_cgr(struct qman_cgr *cgr, u32 flags,
			struct qm_mcc_initcgr *opts)
{
	unsigned long irqflags __maybe_unused;
	struct qm_mcr_querycgr cgr_state;
	struct qm_mcc_initcgr local_opts;
	int ret;
	struct qman_portal *p;

	/* We have to check that the provided CGRID is within the limits of the
	 * data-structures, for obvious reasons. However we'll let h/w take
	 * care of determining whether it's within the limits of what exists on
	 * the SoC. */
	if (cgr->cgrid >= __CGR_NUM)
		return -EINVAL;

	preempt_disable();
	p = get_affine_portal();
	qman_cgr_cpus[cgr->cgrid] = smp_processor_id();
	preempt_enable();

	memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr));
	cgr->chan = p->config->public_cfg.channel;
	spin_lock_irqsave(&p->cgr_lock, irqflags);

	/* if no opts specified, just add it to the list */
	if (!opts)
		goto add_list;

	ret = qman_query_cgr(cgr, &cgr_state);
	if (ret)
		goto release_lock;
	if (opts)
		local_opts = *opts;
	if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
		local_opts.cgr.cscn_targ_upd_ctrl =
			QM_CGR_TARG_UDP_CTRL_WRITE_BIT | PORTAL_IDX(p);
	else
		/* Overwrite TARG */
		local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ |
							TARG_MASK(p);
	local_opts.we_mask |= QM_CGR_WE_CSCN_TARG;

	/* send init if flags indicate so */
	if (opts && (flags & QMAN_CGR_FLAG_USE_INIT))
		ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT, &local_opts);
	else
		ret = qman_modify_cgr(cgr, 0, &local_opts);
	if (ret)
		goto release_lock;
add_list:
	list_add(&cgr->node, &p->cgr_cbs);

	/* Determine if newly added object requires its callback to be called */
	ret = qman_query_cgr(cgr, &cgr_state);
	if (ret) {
		/* we can't go back, so proceed and return success, but screen
		 * and wail to the log file */
		pr_crit("CGR HW state partially modified\n");
		ret = 0;
		goto release_lock;
	}
	if (cgr->cb && cgr_state.cgr.cscn_en && qman_cgrs_get(&p->cgrs[1],
							cgr->cgrid))
		cgr->cb(p, cgr, 1);
release_lock:
	spin_unlock_irqrestore(&p->cgr_lock, irqflags);
	put_affine_portal();
	return ret;
}
EXPORT_SYMBOL(qman_create_cgr);

int qman_create_cgr_to_dcp(struct qman_cgr *cgr, u32 flags, u16 dcp_portal,
					struct qm_mcc_initcgr *opts)
{
	unsigned long irqflags __maybe_unused;
	struct qm_mcc_initcgr local_opts;
	struct qm_mcr_querycgr cgr_state;
	int ret;

	/* We have to check that the provided CGRID is within the limits of the
	 * data-structures, for obvious reasons. However we'll let h/w take
	 * care of determining whether it's within the limits of what exists on
	 * the SoC.
	 */
	if (cgr->cgrid >= __CGR_NUM)
		return -EINVAL;

	ret = qman_query_cgr(cgr, &cgr_state);
	if (ret)
		return ret;

	memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr));
	if (opts)
		local_opts = *opts;

	if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
		local_opts.cgr.cscn_targ_upd_ctrl =
				QM_CGR_TARG_UDP_CTRL_WRITE_BIT |
				QM_CGR_TARG_UDP_CTRL_DCP | dcp_portal;
	else
		local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ |
					TARG_DCP_MASK(dcp_portal);
	local_opts.we_mask |= QM_CGR_WE_CSCN_TARG;

	/* send init if flags indicate so */
	if (opts && (flags & QMAN_CGR_FLAG_USE_INIT))
		ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT,
							&local_opts);
	else
		ret = qman_modify_cgr(cgr, 0, &local_opts);

	return ret;
}
EXPORT_SYMBOL(qman_create_cgr_to_dcp);

int qman_delete_cgr(struct qman_cgr *cgr)
{
	unsigned long irqflags __maybe_unused;
	struct qm_mcr_querycgr cgr_state;
	struct qm_mcc_initcgr local_opts;
	int ret = 0;
	struct qman_cgr *i;
	struct qman_portal *p = get_affine_portal();

	if (cgr->chan != p->config->public_cfg.channel) {
		pr_crit("Attempting to delete cgr from different portal "
			"than it was create: create 0x%x, delete 0x%x\n",
			cgr->chan, p->config->public_cfg.channel);
		ret = -EINVAL;
		goto put_portal;
	}
	memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr));
	spin_lock_irqsave(&p->cgr_lock, irqflags);
	list_del(&cgr->node);
	/*
	 * If there are no other CGR objects for this CGRID in the list, update
	 * CSCN_TARG accordingly
	 */
	list_for_each_entry(i, &p->cgr_cbs, node)
		if ((i->cgrid == cgr->cgrid) && i->cb)
			goto release_lock;
	ret = qman_query_cgr(cgr, &cgr_state);
	if (ret)  {
		/* add back to the list */
		list_add(&cgr->node, &p->cgr_cbs);
		goto release_lock;
	}
	/* Overwrite TARG */
	local_opts.we_mask = QM_CGR_WE_CSCN_TARG;
	if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
		local_opts.cgr.cscn_targ_upd_ctrl = PORTAL_IDX(p);
	else
		local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ &
							 ~(TARG_MASK(p));
	ret = qman_modify_cgr(cgr, 0, &local_opts);
	if (ret)
		/* add back to the list */
		list_add(&cgr->node, &p->cgr_cbs);
release_lock:
	spin_unlock_irqrestore(&p->cgr_lock, irqflags);
put_portal:
	put_affine_portal();
	return ret;
}
EXPORT_SYMBOL(qman_delete_cgr);

struct cgr_comp {
	struct qman_cgr *cgr;
	struct completion completion;
};

static int qman_delete_cgr_thread(void *p)
{
	struct cgr_comp *cgr_comp = (struct cgr_comp *)p;
	int res;

	res = qman_delete_cgr((struct qman_cgr *)cgr_comp->cgr);
	complete(&cgr_comp->completion);

	return res;
}

void qman_delete_cgr_safe(struct qman_cgr *cgr)
{
	struct task_struct *thread;
	struct cgr_comp cgr_comp;

	preempt_disable();
	if (qman_cgr_cpus[cgr->cgrid] != smp_processor_id()) {
		init_completion(&cgr_comp.completion);
		cgr_comp.cgr = cgr;
		thread = kthread_create(qman_delete_cgr_thread, &cgr_comp,
					"cgr_del");

		if (likely(!IS_ERR(thread))) {
			kthread_bind(thread, qman_cgr_cpus[cgr->cgrid]);
			wake_up_process(thread);
			wait_for_completion(&cgr_comp.completion);
			preempt_enable();
			return;
		}
	}
	qman_delete_cgr(cgr);
	preempt_enable();
}
EXPORT_SYMBOL(qman_delete_cgr_safe);

int qm_get_clock(u64 *clock_hz)
{
	if (!qman_clk) {
		pr_warn("Qman clock speed is unknown\n");
		return  -EINVAL;
	}
	*clock_hz = (u64)qman_clk;
	return 0;
}
EXPORT_SYMBOL(qm_get_clock);

int qm_set_clock(u64 clock_hz)
{
	if (qman_clk)
		return -1;
	qman_clk = (u32)clock_hz;
		return 0;
}
EXPORT_SYMBOL(qm_set_clock);

/* CEETM management command */
static int qman_ceetm_configure_lfqmt(struct qm_mcc_ceetm_lfqmt_config *opts)
{
	struct qm_mc_command *mcc;
	struct qm_mc_result *mcr;
	struct qman_portal *p;
	unsigned long irqflags __maybe_unused;
	u8 res;

	p = get_affine_portal();
	PORTAL_IRQ_LOCK(p, irqflags);

	mcc = qm_mc_start(&p->p);
	mcc->lfqmt_config = *opts;
	qm_mc_commit(&p->p, QM_CEETM_VERB_LFQMT_CONFIG);
	while (!(mcr = qm_mc_result(&p->p)))
		cpu_relax();
	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
					 QM_CEETM_VERB_LFQMT_CONFIG);
	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();

	res = mcr->result;
	if (res != QM_MCR_RESULT_OK) {
		pr_err("CEETM: CONFIGURE LFQMT failed\n");
		return -EIO;
	}
	return 0;
}

static int qman_ceetm_query_lfqmt(int lfqid,
			struct qm_mcr_ceetm_lfqmt_query *lfqmt_query)
{
	struct qm_mc_command *mcc;
	struct qm_mc_result *mcr;
	struct qman_portal *p;
	unsigned long irqflags __maybe_unused;
	u8 res;

	p = get_affine_portal();
	PORTAL_IRQ_LOCK(p, irqflags);

	mcc = qm_mc_start(&p->p);
	mcc->lfqmt_query.lfqid = lfqid;
	qm_mc_commit(&p->p, QM_CEETM_VERB_LFQMT_QUERY);
	while (!(mcr = qm_mc_result(&p->p)))
		cpu_relax();
	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_LFQMT_QUERY);
	res = mcr->result;
	if (res == QM_MCR_RESULT_OK)
		*lfqmt_query = mcr->lfqmt_query;

	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();
	if (res != QM_MCR_RESULT_OK) {
		pr_err("CEETM: QUERY LFQMT failed\n");
		return -EIO;
	}
	return 0;
}

static int qman_ceetm_configure_cq(struct qm_mcc_ceetm_cq_config *opts)
{
	struct qm_mc_command *mcc;
	struct qm_mc_result *mcr;
	struct qman_portal *p;
	unsigned long irqflags __maybe_unused;
	u8 res;

	p = get_affine_portal();
	PORTAL_IRQ_LOCK(p, irqflags);

	mcc = qm_mc_start(&p->p);
	mcc->cq_config = *opts;
	qm_mc_commit(&p->p, QM_CEETM_VERB_CQ_CONFIG);
	while (!(mcr = qm_mc_result(&p->p)))
		cpu_relax();
	res = mcr->result;
	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_CQ_CONFIG);

	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();

	if (res != QM_MCR_RESULT_OK) {
		pr_err("CEETM: CONFIGURE CQ failed\n");
		return -EIO;
	}
	return 0;
}

int qman_ceetm_query_cq(unsigned int cqid, unsigned int dcpid,
				struct qm_mcr_ceetm_cq_query *cq_query)
{
	struct qm_mc_command *mcc;
	struct qm_mc_result *mcr;
	struct qman_portal *p;
	unsigned long irqflags __maybe_unused;
	u8 res;

	p = get_affine_portal();
	PORTAL_IRQ_LOCK(p, irqflags);

	mcc = qm_mc_start(&p->p);
	mcc->cq_query.cqid = cqid;
	mcc->cq_query.dcpid = dcpid;
	qm_mc_commit(&p->p, QM_CEETM_VERB_CQ_QUERY);
	while (!(mcr = qm_mc_result(&p->p)))
		cpu_relax();
	res = mcr->result;
	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_CQ_QUERY);

	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();

	if (res != QM_MCR_RESULT_OK) {
		pr_err("CEETM: QUERY CQ failed\n");
		return -EIO;
	}

	*cq_query = mcr->cq_query;
	return 0;
}
EXPORT_SYMBOL(qman_ceetm_query_cq);

static int qman_ceetm_configure_dct(struct qm_mcc_ceetm_dct_config *opts)
{
	struct qm_mc_command *mcc;
	struct qm_mc_result *mcr;
	struct qman_portal *p;
	unsigned long irqflags __maybe_unused;
	u8 res;

	p = get_affine_portal();
	PORTAL_IRQ_LOCK(p, irqflags);

	mcc = qm_mc_start(&p->p);
	mcc->dct_config = *opts;
	qm_mc_commit(&p->p, QM_CEETM_VERB_DCT_CONFIG);
	while (!(mcr = qm_mc_result(&p->p)))
		cpu_relax();
	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_DCT_CONFIG);
	res = mcr->result;

	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();

	if (res != QM_MCR_RESULT_OK) {
		pr_err("CEETM: CONFIGURE DCT failed\n");
		return -EIO;
	}
	return 0;
}

static int qman_ceetm_query_dct(struct qm_mcc_ceetm_dct_query *opts,
			 struct qm_mcr_ceetm_dct_query *dct_query)
{
	struct qm_mc_command *mcc;
	struct qm_mc_result *mcr;
	struct qman_portal *p = get_affine_portal();
	unsigned long irqflags __maybe_unused;
	u8 res;

	PORTAL_IRQ_LOCK(p, irqflags);

	mcc = qm_mc_start(&p->p);
	mcc->dct_query = *opts;
	qm_mc_commit(&p->p, QM_CEETM_VERB_DCT_QUERY);
	while (!(mcr = qm_mc_result(&p->p)))
		cpu_relax();
	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_DCT_QUERY);
	res = mcr->result;

	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();

	if (res != QM_MCR_RESULT_OK) {
		pr_err("CEETM: QUERY DCT failed\n");
		return -EIO;
	}

	*dct_query = mcr->dct_query;
	return 0;
}

static int qman_ceetm_configure_class_scheduler(
			struct qm_mcc_ceetm_class_scheduler_config *opts)
{
	struct qm_mc_command *mcc;
	struct qm_mc_result *mcr;
	struct qman_portal *p;
	unsigned long irqflags __maybe_unused;
	u8 res;

	p = get_affine_portal();
	PORTAL_IRQ_LOCK(p, irqflags);

	mcc = qm_mc_start(&p->p);
	mcc->csch_config = *opts;
	qm_mc_commit(&p->p, QM_CEETM_VERB_CLASS_SCHEDULER_CONFIG);
	while (!(mcr = qm_mc_result(&p->p)))
		cpu_relax();
	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
					QM_CEETM_VERB_CLASS_SCHEDULER_CONFIG);
	res = mcr->result;

	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();

	if (res != QM_MCR_RESULT_OK) {
		pr_err("CEETM: CONFIGURE CLASS SCHEDULER failed\n");
		return -EIO;
	}
	return 0;
}

static int qman_ceetm_query_class_scheduler(struct qm_ceetm_channel *channel,
			struct qm_mcr_ceetm_class_scheduler_query *query)
{
	struct qm_mc_command *mcc;
	struct qm_mc_result *mcr;
	struct qman_portal *p;
	unsigned long irqflags __maybe_unused;
	u8 res;

	p = get_affine_portal();
	PORTAL_IRQ_LOCK(p, irqflags);

	mcc = qm_mc_start(&p->p);
	mcc->csch_query.cqcid = channel->idx;
	mcc->csch_query.dcpid = channel->dcp_idx;
	qm_mc_commit(&p->p, QM_CEETM_VERB_CLASS_SCHEDULER_QUERY);
	while (!(mcr = qm_mc_result(&p->p)))
		cpu_relax();
	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
				QM_CEETM_VERB_CLASS_SCHEDULER_QUERY);
	res = mcr->result;

	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();

	if (res != QM_MCR_RESULT_OK) {
		pr_err("CEETM: QUERY CLASS SCHEDULER failed\n");
		return -EIO;
	}
	*query = mcr->csch_query;
	return 0;
}

static int qman_ceetm_configure_mapping_shaper_tcfc(
		struct qm_mcc_ceetm_mapping_shaper_tcfc_config *opts)
{
	struct qm_mc_command *mcc;
	struct qm_mc_result *mcr;
	struct qman_portal *p;
	unsigned long irqflags __maybe_unused;
	u8 res;

	p = get_affine_portal();
	PORTAL_IRQ_LOCK(p, irqflags);

	mcc = qm_mc_start(&p->p);
	mcc->mst_config = *opts;
	qm_mc_commit(&p->p, QM_CEETM_VERB_MAPPING_SHAPER_TCFC_CONFIG);
	while (!(mcr = qm_mc_result(&p->p)))
		cpu_relax();
	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
				QM_CEETM_VERB_MAPPING_SHAPER_TCFC_CONFIG);
	res = mcr->result;

	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();

	if (res != QM_MCR_RESULT_OK) {
		pr_err("CEETM: CONFIGURE CHANNEL MAPPING failed\n");
		return -EIO;
	}
	return 0;
}

static int qman_ceetm_query_mapping_shaper_tcfc(
		struct qm_mcc_ceetm_mapping_shaper_tcfc_query *opts,
		struct qm_mcr_ceetm_mapping_shaper_tcfc_query *response)
{
	struct qm_mc_command *mcc;
	struct qm_mc_result *mcr;
	struct qman_portal *p;
	unsigned long irqflags __maybe_unused;
	u8 res;

	p = get_affine_portal();
	PORTAL_IRQ_LOCK(p, irqflags);

	mcc = qm_mc_start(&p->p);
	mcc->mst_query = *opts;
	qm_mc_commit(&p->p, QM_CEETM_VERB_MAPPING_SHAPER_TCFC_QUERY);
	while (!(mcr = qm_mc_result(&p->p)))
		cpu_relax();
	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
				QM_CEETM_VERB_MAPPING_SHAPER_TCFC_QUERY);
	res = mcr->result;

	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();

	if (res != QM_MCR_RESULT_OK) {
		pr_err("CEETM: QUERY CHANNEL MAPPING failed\n");
		return -EIO;
	}

	*response = mcr->mst_query;
	return 0;
}

static int qman_ceetm_configure_ccgr(struct qm_mcc_ceetm_ccgr_config *opts)
{
	struct qm_mc_command *mcc;
	struct qm_mc_result *mcr;
	struct qman_portal *p;
	unsigned long irqflags __maybe_unused;
	u8 res;

	p = get_affine_portal();
	PORTAL_IRQ_LOCK(p, irqflags);

	mcc = qm_mc_start(&p->p);
	mcc->ccgr_config = *opts;

	qm_mc_commit(&p->p, QM_CEETM_VERB_CCGR_CONFIG);
	while (!(mcr = qm_mc_result(&p->p)))
		cpu_relax();
	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_CCGR_CONFIG);

	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();

	res = mcr->result;
	if (res != QM_MCR_RESULT_OK) {
		pr_err("CEETM: CONFIGURE CCGR failed\n");
		return -EIO;
	}
	return 0;
}

int qman_ceetm_query_ccgr(struct qm_mcc_ceetm_ccgr_query *ccgr_query,
				struct qm_mcr_ceetm_ccgr_query *response)
{
	struct qm_mc_command *mcc;
	struct qm_mc_result *mcr;
	struct qman_portal *p;
	unsigned long irqflags __maybe_unused;
	u8 res;

	p = get_affine_portal();
	PORTAL_IRQ_LOCK(p, irqflags);

	mcc = qm_mc_start(&p->p);
	mcc->ccgr_query = *ccgr_query;
	qm_mc_commit(&p->p, QM_CEETM_VERB_CCGR_QUERY);

	while (!(mcr = qm_mc_result(&p->p)))
		cpu_relax();
	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_CCGR_QUERY);

	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();

	res = mcr->result;
	if (res != QM_MCR_RESULT_OK) {
		pr_err("CEETM: QUERY CCGR failed\n");
		return -EIO;
	}
	*response = mcr->ccgr_query;
	return 0;
}
EXPORT_SYMBOL(qman_ceetm_query_ccgr);

static int qman_ceetm_cq_peek_pop_xsfdrread(struct qm_ceetm_cq *cq,
			u8 command_type, u16 xsfdr,
			struct qm_mcr_ceetm_cq_peek_pop_xsfdrread *cq_ppxr)
{
	struct qm_mc_command *mcc;
	struct qm_mc_result *mcr;
	struct qman_portal *p;
	unsigned long irqflags __maybe_unused;
	u8 res;

	p = get_affine_portal();
	PORTAL_IRQ_LOCK(p, irqflags);

	mcc = qm_mc_start(&p->p);
	switch (command_type) {
	case 0:
	case 1:
		mcc->cq_ppxr.cqid = (cq->parent->idx << 4) | cq->idx;
		break;
	case 2:
		mcc->cq_ppxr.xsfdr = xsfdr;
		break;
	default:
		break;
	}
	mcc->cq_ppxr.ct = command_type;
	mcc->cq_ppxr.dcpid = cq->parent->dcp_idx;
	qm_mc_commit(&p->p, QM_CEETM_VERB_CQ_PEEK_POP_XFDRREAD);
	while (!(mcr = qm_mc_result(&p->p)))
		cpu_relax();
	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
				QM_CEETM_VERB_CQ_PEEK_POP_XFDRREAD);

	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();

	res = mcr->result;
	if (res != QM_MCR_RESULT_OK) {
		pr_err("CEETM: CQ PEEK/POP/XSFDR READ failed\n");
		return -EIO;
	}
	*cq_ppxr = mcr->cq_ppxr;
	return 0;
}

static int qman_ceetm_query_statistics(u16 cid,
			enum qm_dc_portal dcp_idx,
			u16 command_type,
			struct qm_mcr_ceetm_statistics_query *query_result)
{
	struct qm_mc_command *mcc;
	struct qm_mc_result *mcr;
	struct qman_portal *p;
	unsigned long irqflags __maybe_unused;
	u8 res;

	p = get_affine_portal();
	PORTAL_IRQ_LOCK(p, irqflags);

	mcc = qm_mc_start(&p->p);
	mcc->stats_query_write.cid = cid;
	mcc->stats_query_write.dcpid = dcp_idx;
	mcc->stats_query_write.ct = command_type;
	qm_mc_commit(&p->p, QM_CEETM_VERB_STATISTICS_QUERY_WRITE);

	while (!(mcr = qm_mc_result(&p->p)))
		cpu_relax();
	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
					 QM_CEETM_VERB_STATISTICS_QUERY_WRITE);

	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();

	res = mcr->result;
	if (res != QM_MCR_RESULT_OK) {
		pr_err("CEETM: STATISTICS QUERY failed\n");
		return -EIO;
	}
	*query_result = mcr->stats_query;
	return 0;
}

static int qman_ceetm_write_statistics(u16 cid, enum qm_dc_portal dcp_idx,
			u16 command_type, u64 frame_count, u64 byte_count)
{
	struct qm_mc_command *mcc;
	struct qm_mc_result *mcr;
	struct qman_portal *p;
	unsigned long irqflags __maybe_unused;
	u8 res;

	p = get_affine_portal();
	PORTAL_IRQ_LOCK(p, irqflags);

	mcc = qm_mc_start(&p->p);
	mcc->stats_query_write.cid = cid;
	mcc->stats_query_write.dcpid = dcp_idx;
	mcc->stats_query_write.ct = command_type;
	mcc->stats_query_write.frm_cnt = frame_count;
	mcc->stats_query_write.byte_cnt = byte_count;
	qm_mc_commit(&p->p, QM_CEETM_VERB_STATISTICS_QUERY_WRITE);

	while (!(mcr = qm_mc_result(&p->p)))
		cpu_relax();
	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
					 QM_CEETM_VERB_STATISTICS_QUERY_WRITE);

	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();

	res = mcr->result;
	if (res != QM_MCR_RESULT_OK) {
		pr_err("CEETM: STATISTICS WRITE failed\n");
		return -EIO;
	}
	return 0;
}

int qman_ceetm_bps2tokenrate(u64 bps, struct qm_ceetm_rate *token_rate,
							int rounding)
{
	u16 pres;
	u64 temp;
	u64 qman_freq;
	int ret;

	/* Read PRES from CEET_CFG_PRES register */
	ret = qman_ceetm_get_prescaler(&pres);
	if (ret)
		return -EINVAL;

	ret = qm_get_clock(&qman_freq);
	if (ret)
		return -EINVAL;

	/* token-rate = bytes-per-second * update-reference-period
	 *
	 * Where token-rate is N/8192 for a integer N, and
	 * update-reference-period is (2^22)/(PRES*QHz), where PRES
	 * is the prescalar value and QHz is the QMan clock frequency.
	 * So:
	 *
	 * token-rate = (byte-per-second*2^22)/PRES*QHZ)
	 *
	 * Converting to bits-per-second gives;
	 *
	 *	token-rate = (bps*2^19) / (PRES*QHZ)
	 *	N = (bps*2^32) / (PRES*QHz)
	 *
	 * And to avoid 64-bit overflow if 'bps' is larger than 4Gbps
	 * (yet minimise rounding error if 'bps' is small), we reorganise
	 * the formula to use two 16-bit shifts rather than 1 32-bit shift.
	 *      N = (((bps*2^16)/PRES)*2^16)/QHz
	 */
	temp = ROUNDING((bps << 16), pres, rounding);
	temp = ROUNDING((temp << 16), qman_freq, rounding);
	token_rate->whole = temp >> 13;
	token_rate->fraction = temp & (((u64)1 << 13) - 1);
	return 0;
}
EXPORT_SYMBOL(qman_ceetm_bps2tokenrate);

int qman_ceetm_tokenrate2bps(const struct qm_ceetm_rate *token_rate, u64 *bps,
							int rounding)
{
	u16 pres;
	u64 temp;
	u64 qman_freq;
	int ret;

	/* Read PRES from CEET_CFG_PRES register */
	ret = qman_ceetm_get_prescaler(&pres);
	if (ret)
		return -EINVAL;

	ret = qm_get_clock(&qman_freq);
	if (ret)
		return -EINVAL;

	/* bytes-per-second = token-rate / update-reference-period
	 *
	 * where "token-rate" is N/8192 for an integer N, and
	 * "update-reference-period" is (2^22)/(PRES*QHz), where PRES is
	 * the prescalar value and QHz is the QMan clock frequency. So;
	 *
	 * bytes-per-second = (N/8192) / (4194304/PRES*QHz)
	 *                  = N*PRES*QHz / (4194304*8192)
	 *                  = N*PRES*QHz / (2^35)
	 *
	 * Converting to bits-per-second gives;
	 *
	 *             bps = N*PRES*QHZ / (2^32)
	 *
	 * Note, the numerator has a maximum width of 72 bits! So to
	 * avoid 64-bit overflow errors, we calculate PRES*QHZ (maximum
	 * width 48 bits) divided by 2^9 (reducing to maximum 39 bits), before
	 * multiplying by N (goes to maximum of 63 bits).
	 *
	 *             temp = PRES*QHZ / (2^16)
	 *             kbps = temp*N / (2^16)
	 */
	temp = ROUNDING(qman_freq * pres, (u64)1 << 16 , rounding);
	temp *= ((token_rate->whole << 13) + token_rate->fraction);
	*bps = ROUNDING(temp, (u64)(1) << 16, rounding);
	return 0;
}
EXPORT_SYMBOL(qman_ceetm_tokenrate2bps);

int qman_ceetm_sp_claim(struct qm_ceetm_sp **sp, enum qm_dc_portal dcp_idx,
						unsigned int sp_idx)
{
	struct qm_ceetm_sp *p;

	DPA_ASSERT((dcp_idx ==  qm_dc_portal_fman0) ||
			(dcp_idx == qm_dc_portal_fman1));

	if ((sp_idx < qman_ceetms[dcp_idx].sp_range[0]) ||
		(sp_idx > (qman_ceetms[dcp_idx].sp_range[0] +
		qman_ceetms[dcp_idx].sp_range[1]))) {
		pr_err("Sub-portal index doesn't exist\n");
		return -EINVAL;
	}

	list_for_each_entry(p, &qman_ceetms[dcp_idx].sub_portals, node) {
		if ((p->idx == sp_idx) && (p->is_claimed == 0)) {
			p->is_claimed = 1;
			*sp = p;
			return 0;
		}
	}
	pr_err("The sub-portal#%d is not available!\n", sp_idx);
	return -ENODEV;
}
EXPORT_SYMBOL(qman_ceetm_sp_claim);

int qman_ceetm_sp_release(struct qm_ceetm_sp *sp)
{
	struct qm_ceetm_sp *p;

	if (sp->lni->is_claimed == 1) {
		pr_err("The dependency of sub-portal has not been released!\n");
		return -EBUSY;
	}

	list_for_each_entry(p, &qman_ceetms[sp->dcp_idx].sub_portals, node) {
		if (p->idx == sp->idx) {
			p->is_claimed = 0;
			p->lni = NULL;
		}
	}
	/* Disable CEETM mode of this sub-portal */
	qman_sp_disable_ceetm_mode(sp->dcp_idx, sp->idx);

	return 0;
}
EXPORT_SYMBOL(qman_ceetm_sp_release);

int qman_ceetm_lni_claim(struct qm_ceetm_lni **lni, enum qm_dc_portal dcp_idx,
							unsigned int lni_idx)
{
	struct qm_ceetm_lni *p;

	if ((lni_idx < qman_ceetms[dcp_idx].lni_range[0]) ||
		(lni_idx > (qman_ceetms[dcp_idx].lni_range[0] +
		qman_ceetms[dcp_idx].lni_range[1]))) {
		pr_err("The lni index is out of range\n");
		return -EINVAL;
	}

	list_for_each_entry(p, &qman_ceetms[dcp_idx].lnis, node) {
		if ((p->idx == lni_idx) && (p->is_claimed == 0)) {
			*lni = p;
			p->is_claimed = 1;
			return 0;
		}
	}

	pr_err("The LNI#%d is not available!\n", lni_idx);
	return -EINVAL;
}
EXPORT_SYMBOL(qman_ceetm_lni_claim);

int qman_ceetm_lni_release(struct qm_ceetm_lni *lni)
{
	struct qm_ceetm_lni *p;
	struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;

	if (!list_empty(&lni->channels)) {
		pr_err("The LNI dependencies are not released!\n");
		return -EBUSY;
	}

	list_for_each_entry(p, &qman_ceetms[lni->dcp_idx].lnis, node) {
		if (p->idx == lni->idx) {
			p->shaper_enable = 0;
			p->shaper_couple = 0;
			p->cr_token_rate.whole = 0;
			p->cr_token_rate.fraction = 0;
			p->er_token_rate.whole = 0;
			p->er_token_rate.fraction = 0;
			p->cr_token_bucket_limit = 0;
			p->er_token_bucket_limit = 0;
			p->is_claimed = 0;
		}
	}
	config_opts.cid = CEETM_COMMAND_LNI_SHAPER | lni->idx;
	config_opts.dcpid = lni->dcp_idx;
	memset(&config_opts.shaper_config, 0,
				sizeof(config_opts.shaper_config));
	return	qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
}
EXPORT_SYMBOL(qman_ceetm_lni_release);

int qman_ceetm_sp_set_lni(struct qm_ceetm_sp *sp, struct qm_ceetm_lni *lni)
{
	struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;

	config_opts.cid = CEETM_COMMAND_SP_MAPPING | sp->idx;
	config_opts.dcpid = sp->dcp_idx;
	config_opts.sp_mapping.map_lni_id = lni->idx;
	sp->lni = lni;

	if (qman_ceetm_configure_mapping_shaper_tcfc(&config_opts))
		return -EINVAL;

	/* Enable CEETM mode for this sub-portal */
	return qman_sp_enable_ceetm_mode(sp->dcp_idx, sp->idx);
}
EXPORT_SYMBOL(qman_ceetm_sp_set_lni);

int qman_ceetm_sp_get_lni(struct qm_ceetm_sp *sp, unsigned int *lni_idx)
{
	struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
	struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;

	query_opts.cid = CEETM_COMMAND_SP_MAPPING | sp->idx;
	query_opts.dcpid = sp->dcp_idx;
	if (qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result)) {
		pr_err("Can't get SP <-> LNI mapping\n");
		return -EINVAL;
	}
	*lni_idx = query_result.sp_mapping_query.map_lni_id;
	sp->lni->idx = query_result.sp_mapping_query.map_lni_id;
	return 0;
}
EXPORT_SYMBOL(qman_ceetm_sp_get_lni);

int qman_ceetm_lni_enable_shaper(struct qm_ceetm_lni *lni, int coupled,
								int oal)
{
	struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;

	lni->shaper_enable = 1;
	lni->shaper_couple = coupled;
	lni->oal = oal;

	config_opts.cid = CEETM_COMMAND_LNI_SHAPER | lni->idx;
	config_opts.dcpid = lni->dcp_idx;
	config_opts.shaper_config.cpl = (coupled << 7) | lni->oal;
	config_opts.shaper_config.crtcr = (lni->cr_token_rate.whole << 13) |
			 lni->cr_token_rate.fraction;
	config_opts.shaper_config.ertcr = (lni->er_token_rate.whole << 13) |
			 lni->er_token_rate.fraction;
	config_opts.shaper_config.crtbl = lni->cr_token_bucket_limit;
	config_opts.shaper_config.ertbl = lni->er_token_bucket_limit;
	config_opts.shaper_config.mps = 60;
	return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
}
EXPORT_SYMBOL(qman_ceetm_lni_enable_shaper);

int qman_ceetm_lni_disable_shaper(struct qm_ceetm_lni *lni)
{
	struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;

	if (!lni->shaper_enable) {
		pr_err("The shaper has been disabled\n");
		return -EINVAL;
	}

	config_opts.cid = CEETM_COMMAND_LNI_SHAPER | lni->idx;
	config_opts.dcpid = lni->dcp_idx;
	config_opts.shaper_config.cpl = (lni->shaper_couple << 7) | lni->oal;
	config_opts.shaper_config.crtbl = lni->cr_token_bucket_limit;
	config_opts.shaper_config.ertbl = lni->er_token_bucket_limit;
	/* Set CR/ER rate with all 1's to configure an infinite rate, thus
	 * disable the shaping.
	 */
	config_opts.shaper_config.crtcr = 0xFFFFFF;
	config_opts.shaper_config.ertcr = 0xFFFFFF;
	config_opts.shaper_config.mps = 60;
	lni->shaper_enable = 0;
	return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
}
EXPORT_SYMBOL(qman_ceetm_lni_disable_shaper);

int qman_ceetm_lni_is_shaper_enabled(struct qm_ceetm_lni *lni)
{
	return lni->shaper_enable;
}
EXPORT_SYMBOL(qman_ceetm_lni_is_shaper_enabled);

int qman_ceetm_lni_set_commit_rate(struct qm_ceetm_lni *lni,
				const struct qm_ceetm_rate *token_rate,
				u16 token_limit)
{
	struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
	struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
	struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
	int ret;

	lni->cr_token_rate.whole = token_rate->whole;
	lni->cr_token_rate.fraction = token_rate->fraction;
	lni->cr_token_bucket_limit = token_limit;
	if (lni->shaper_enable) {
		query_opts.cid = CEETM_COMMAND_LNI_SHAPER | lni->idx;
		query_opts.dcpid = lni->dcp_idx;
		ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts,
							&query_result);
		if (ret) {
			pr_err("Fail to get current LNI shaper setting\n");
			return -EINVAL;
		}

		config_opts.cid = CEETM_COMMAND_LNI_SHAPER | lni->idx;
		config_opts.dcpid = lni->dcp_idx;
		config_opts.shaper_config.crtcr = (token_rate->whole << 13) |
					 (token_rate->fraction);
		config_opts.shaper_config.crtbl = token_limit;
		config_opts.shaper_config.cpl = query_result.shaper_query.cpl;
		config_opts.shaper_config.ertcr =
					query_result.shaper_query.ertcr;
		config_opts.shaper_config.ertbl =
					query_result.shaper_query.ertbl;
		config_opts.shaper_config.mps = query_result.shaper_query.mps;
		return	qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
	} else {
		return 0;
	}
}
EXPORT_SYMBOL(qman_ceetm_lni_set_commit_rate);

int qman_ceetm_lni_set_commit_rate_bps(struct qm_ceetm_lni *lni,
				       u64 bps,
				       u16 token_limit)
{
	struct qm_ceetm_rate token_rate;
	int ret;

	ret = qman_ceetm_bps2tokenrate(bps, &token_rate, 0);
	if (ret) {
		pr_err("Can not convert bps to token rate\n");
		return -EINVAL;
	}

	return qman_ceetm_lni_set_commit_rate(lni, &token_rate, token_limit);
}
EXPORT_SYMBOL(qman_ceetm_lni_set_commit_rate_bps);

int qman_ceetm_lni_get_commit_rate(struct qm_ceetm_lni *lni,
				struct qm_ceetm_rate *token_rate,
				u16 *token_limit)
{
	struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
	struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
	int ret;

	query_opts.cid = CEETM_COMMAND_LNI_SHAPER | lni->idx;
	query_opts.dcpid = lni->dcp_idx;

	ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result);
	if (ret) {
		pr_err("The LNI CR rate or limit is not set\n");
		return -EINVAL;
	}
	token_rate->whole = query_result.shaper_query.crtcr >> 13;
	token_rate->fraction = query_result.shaper_query.crtcr & 0x1FFF;
	*token_limit = query_result.shaper_query.crtbl;
	return 0;
}
EXPORT_SYMBOL(qman_ceetm_lni_get_commit_rate);

int qman_ceetm_lni_get_commit_rate_bps(struct qm_ceetm_lni *lni,
				       u64 *bps, u16 *token_limit)
{
	struct qm_ceetm_rate token_rate;
	int ret;

	ret = qman_ceetm_lni_get_commit_rate(lni, &token_rate, token_limit);
	if (ret) {
		pr_err("The LNI CR rate or limit is not available\n");
		return -EINVAL;
	}

	return qman_ceetm_tokenrate2bps(&token_rate, bps, 0);
}
EXPORT_SYMBOL(qman_ceetm_lni_get_commit_rate_bps);

int qman_ceetm_lni_set_excess_rate(struct qm_ceetm_lni *lni,
					const struct qm_ceetm_rate *token_rate,
					u16 token_limit)
{
	struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
	struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
	struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
	int ret;

	lni->er_token_rate.whole = token_rate->whole;
	lni->er_token_rate.fraction = token_rate->fraction;
	lni->er_token_bucket_limit = token_limit;
	if (lni->shaper_enable) {
		query_opts.cid = CEETM_COMMAND_LNI_SHAPER | lni->idx;
		query_opts.dcpid = lni->dcp_idx;
		ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts,
							&query_result);
		if (ret) {
			pr_err("Fail to get current LNI shaper setting\n");
			return -EINVAL;
		}

		config_opts.cid = CEETM_COMMAND_LNI_SHAPER | lni->idx;
		config_opts.dcpid = lni->dcp_idx;
		config_opts.shaper_config.ertcr =
			 (token_rate->whole << 13) | (token_rate->fraction);
		config_opts.shaper_config.ertbl = token_limit;
		config_opts.shaper_config.cpl = query_result.shaper_query.cpl;
		config_opts.shaper_config.crtcr =
					query_result.shaper_query.crtcr;
		config_opts.shaper_config.crtbl =
					query_result.shaper_query.crtbl;
		config_opts.shaper_config.mps = query_result.shaper_query.mps;
		return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
	} else {
		return 0;
	}
}
EXPORT_SYMBOL(qman_ceetm_lni_set_excess_rate);

int qman_ceetm_lni_set_excess_rate_bps(struct qm_ceetm_lni *lni,
				       u64 bps,
				       u16 token_limit)
{
	struct qm_ceetm_rate token_rate;
	int ret;

	ret = qman_ceetm_bps2tokenrate(bps, &token_rate, 0);
	if (ret) {
		pr_err("Can not convert bps to token rate\n");
		return -EINVAL;
	}

	return qman_ceetm_lni_set_excess_rate(lni, &token_rate, token_limit);
}
EXPORT_SYMBOL(qman_ceetm_lni_set_excess_rate_bps);

int qman_ceetm_lni_get_excess_rate(struct qm_ceetm_lni *lni,
					struct qm_ceetm_rate *token_rate,
					u16 *token_limit)
{
	struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
	struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
	int ret;

	query_opts.cid = CEETM_COMMAND_LNI_SHAPER | lni->idx;
	query_opts.dcpid = lni->dcp_idx;
	ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result);
	if (ret) {
		pr_err("The LNI ER rate or limit is not set\n");
		return -EINVAL;
	}
	token_rate->whole = query_result.shaper_query.ertcr >> 13;
	token_rate->fraction = query_result.shaper_query.ertcr & 0x1FFF;
	*token_limit = query_result.shaper_query.ertbl;
	return 0;
}
EXPORT_SYMBOL(qman_ceetm_lni_get_excess_rate);

int qman_ceetm_lni_get_excess_rate_bps(struct qm_ceetm_lni *lni,
				       u64 *bps, u16 *token_limit)
{
	struct qm_ceetm_rate token_rate;
	int ret;

	ret = qman_ceetm_lni_get_excess_rate(lni, &token_rate, token_limit);
	if (ret) {
		pr_err("The LNI ER rate or limit is not available\n");
		return -EINVAL;
	}

	return qman_ceetm_tokenrate2bps(&token_rate, bps, 0);
}
EXPORT_SYMBOL(qman_ceetm_lni_get_excess_rate_bps);

#define QMAN_CEETM_LNITCFCC_CQ_LEVEL_SHIFT(n) ((15 - n) * 4)
#define QMAN_CEETM_LNITCFCC_ENABLE 0x8
int qman_ceetm_lni_set_tcfcc(struct qm_ceetm_lni *lni,
				unsigned int cq_level,
				int traffic_class)
{
	struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
	struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
	struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
	u64 lnitcfcc;

	if ((cq_level > 15) | (traffic_class > 7)) {
		pr_err("The CQ or traffic class id is out of range\n");
		return -EINVAL;
	}

	query_opts.cid = CEETM_COMMAND_TCFC | lni->idx;
	query_opts.dcpid = lni->dcp_idx;
	if (qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result)) {
		pr_err("Fail to query tcfcc\n");
		return -EINVAL;
	}

	lnitcfcc = query_result.tcfc_query.lnitcfcc;
	if (traffic_class == -1) {
		/* disable tcfc for this CQ */
		lnitcfcc &= ~((u64)QMAN_CEETM_LNITCFCC_ENABLE <<
				QMAN_CEETM_LNITCFCC_CQ_LEVEL_SHIFT(cq_level));
	} else {
		lnitcfcc &= ~((u64)0xF <<
				QMAN_CEETM_LNITCFCC_CQ_LEVEL_SHIFT(cq_level));
		lnitcfcc |= ((u64)(QMAN_CEETM_LNITCFCC_ENABLE |
				traffic_class)) <<
				QMAN_CEETM_LNITCFCC_CQ_LEVEL_SHIFT(cq_level);
	}
	config_opts.tcfc_config.lnitcfcc = lnitcfcc;
	config_opts.cid = CEETM_COMMAND_TCFC | lni->idx;
	config_opts.dcpid = lni->dcp_idx;
	return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
}
EXPORT_SYMBOL(qman_ceetm_lni_set_tcfcc);

#define QMAN_CEETM_LNITCFCC_TC_MASK 0x7
int qman_ceetm_lni_get_tcfcc(struct qm_ceetm_lni *lni, unsigned int cq_level,
						int *traffic_class)
{
	struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
	struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
	int ret;
	u8 lnitcfcc;

	if (cq_level > 15) {
		pr_err("the CQ level is out of range\n");
		return -EINVAL;
	}

	query_opts.cid = CEETM_COMMAND_TCFC | lni->idx;
	query_opts.dcpid = lni->dcp_idx;
	ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result);
	if (ret)
		return ret;
	lnitcfcc = (u8)(query_result.tcfc_query.lnitcfcc >>
				QMAN_CEETM_LNITCFCC_CQ_LEVEL_SHIFT(cq_level));
	if (lnitcfcc & QMAN_CEETM_LNITCFCC_ENABLE)
		*traffic_class = lnitcfcc & QMAN_CEETM_LNITCFCC_TC_MASK;
	else
		*traffic_class = -1;
	return 0;
}
EXPORT_SYMBOL(qman_ceetm_lni_get_tcfcc);

#define QMAN_CEETM_ENABLE_CHANNEL_SHAPER 0x80
int qman_ceetm_channel_claim(struct qm_ceetm_channel **channel,
				struct qm_ceetm_lni *lni)
{
	struct qm_ceetm_channel *p;
	u32 channel_idx;
	int ret = 0;
	struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
	static u8 map;

	if (lni->dcp_idx == qm_dc_portal_fman0) {
		ret = qman_alloc_ceetm0_channel(&channel_idx);
	} else if (lni->dcp_idx == qm_dc_portal_fman1) {
		ret = qman_alloc_ceetm1_channel(&channel_idx);
	} else {
		pr_err("dcp_idx %u does not correspond to a known fman in this driver\n",
			lni->dcp_idx);
		return -EINVAL;
	}

	if (ret) {
		pr_err("The is no channel available for LNI#%d\n", lni->idx);
		return -ENODEV;
	}

	p = kzalloc(sizeof(*p), GFP_KERNEL);
	if (!p)
		return -ENOMEM;
	p->idx = channel_idx;
	p->dcp_idx = lni->dcp_idx;
	list_add_tail(&p->node, &lni->channels);
	INIT_LIST_HEAD(&p->class_queues);
	INIT_LIST_HEAD(&p->ccgs);
	config_opts.cid = CEETM_COMMAND_CHANNEL_MAPPING | channel_idx;
	config_opts.dcpid = lni->dcp_idx;
	map = (u8)~QMAN_CEETM_ENABLE_CHANNEL_SHAPER;
	map &= lni->idx;
	config_opts.channel_mapping.map = map;
	if (qman_ceetm_configure_mapping_shaper_tcfc(&config_opts)) {
		pr_err("Can't map channel#%d for LNI#%d\n",
						channel_idx, lni->idx);
		return -EINVAL;
	}
	*channel = p;
	return 0;
}
EXPORT_SYMBOL(qman_ceetm_channel_claim);

int qman_ceetm_channel_release(struct qm_ceetm_channel *channel)
{
	struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
	if (!list_empty(&channel->class_queues)) {
		pr_err("CEETM channel#%d has class queue unreleased!\n",
						channel->idx);
		return -EBUSY;
	}
	if (!list_empty(&channel->ccgs)) {
		pr_err("CEETM channel#%d has ccg unreleased!\n",
						channel->idx);
		return -EBUSY;
	}

	/* channel->dcp_idx corresponds to known fman validation */
	if ((channel->dcp_idx != qm_dc_portal_fman0) &&
	    (channel->dcp_idx != qm_dc_portal_fman1)) {
		pr_err("dcp_idx %u does not correspond to a known fman in this driver\n",
			channel->dcp_idx);
		return -EINVAL;
	}

	config_opts.cid = CEETM_COMMAND_CHANNEL_SHAPER | channel->idx;
	config_opts.dcpid = channel->dcp_idx;
	memset(&config_opts.shaper_config, 0,
				sizeof(config_opts.shaper_config));
	if (qman_ceetm_configure_mapping_shaper_tcfc(&config_opts)) {
		pr_err("Can't reset channel shapping parameters\n");
		return -EINVAL;
	}

	if (channel->dcp_idx == qm_dc_portal_fman0) {
		qman_release_ceetm0_channelid(channel->idx);
	} else if (channel->dcp_idx == qm_dc_portal_fman1) {
		qman_release_ceetm1_channelid(channel->idx);
	} else {
		pr_err("dcp_idx %u does not correspond to a known fman in this driver\n",
			channel->dcp_idx);
		return -EINVAL;
	}
	list_del(&channel->node);
	kfree(channel);

	return 0;
}
EXPORT_SYMBOL(qman_ceetm_channel_release);

int qman_ceetm_channel_enable_shaper(struct qm_ceetm_channel *channel,
								int coupled)
{
	struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
	struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
	struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
	u8 map;

	if (channel->shaper_enable == 1) {
		pr_err("This channel shaper has been enabled!\n");
		return -EINVAL;
	}

	channel->shaper_enable = 1;
	channel->shaper_couple = coupled;

	query_opts.cid = (u16)(CEETM_COMMAND_CHANNEL_MAPPING | channel->idx);
	query_opts.dcpid = (u8)channel->dcp_idx;

	if (qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result)) {
		pr_err("Can't query channel mapping\n");
		return -EINVAL;
	}

	map = query_result.channel_mapping_query.map;
	map |= QMAN_CEETM_ENABLE_CHANNEL_SHAPER;

	config_opts.cid = CEETM_COMMAND_CHANNEL_MAPPING | channel->idx;
	config_opts.dcpid = channel->dcp_idx;
	config_opts.channel_mapping.map = map;
	if (qman_ceetm_configure_mapping_shaper_tcfc(&config_opts)) {
		pr_err("Can't enable shaper for channel #%d\n",
						channel->idx);
		return -EINVAL;
	}

	config_opts.cid = CEETM_COMMAND_CHANNEL_SHAPER | channel->idx;
	config_opts.shaper_config.cpl = coupled << 7;
	config_opts.shaper_config.crtcr = (channel->cr_token_rate.whole << 13) |
					channel->cr_token_rate.fraction;
	config_opts.shaper_config.ertcr = (channel->er_token_rate.whole << 13) |
					channel->er_token_rate.fraction;
	config_opts.shaper_config.crtbl = channel->cr_token_bucket_limit;
	config_opts.shaper_config.ertbl = channel->er_token_bucket_limit;
	return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
}
EXPORT_SYMBOL(qman_ceetm_channel_enable_shaper);

int qman_ceetm_channel_disable_shaper(struct qm_ceetm_channel *channel)
{
	struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
	struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
	struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
	u8 map;

	query_opts.cid = CEETM_COMMAND_CHANNEL_MAPPING | channel->idx;
	query_opts.dcpid = channel->dcp_idx;

	if (qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result)) {
		pr_err("Can't query channel mapping\n");
		return -EINVAL;
	}

	map = query_result.channel_mapping_query.map;
	map &= ~QMAN_CEETM_ENABLE_CHANNEL_SHAPER;

	config_opts.cid = CEETM_COMMAND_CHANNEL_MAPPING | channel->idx;
	config_opts.dcpid = channel->dcp_idx;
	config_opts.channel_mapping.map = map;
	return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
}
EXPORT_SYMBOL(qman_ceetm_channel_disable_shaper);

int qman_ceetm_channel_is_shaper_enabled(struct qm_ceetm_channel *channel)
{
	struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
	struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
	u8 map;

	query_opts.cid = CEETM_COMMAND_CHANNEL_MAPPING | channel->idx;
	query_opts.dcpid = channel->dcp_idx;

	if (qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result)) {
		pr_err("Can't query channel mapping\n");
		return -EINVAL;
	}

	map = query_result.channel_mapping_query.map;
	return (map & QMAN_CEETM_ENABLE_CHANNEL_SHAPER) ? 1 : 0;
}
EXPORT_SYMBOL(qman_ceetm_channel_is_shaper_enabled);

int qman_ceetm_channel_set_commit_rate(struct qm_ceetm_channel *channel,
				const struct qm_ceetm_rate *token_rate,
				u16 token_limit)
{
	struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
	struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
	struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
	int ret;

	query_opts.cid = CEETM_COMMAND_CHANNEL_SHAPER | channel->idx;
	query_opts.dcpid = channel->dcp_idx;

	ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result);
	if (ret) {
		pr_err("Fail to get the current channel shaper setting\n");
		return -EINVAL;
	}

	channel->cr_token_rate.whole = token_rate->whole;
	channel->cr_token_rate.fraction = token_rate->fraction;
	channel->cr_token_bucket_limit = token_limit;
	config_opts.cid = CEETM_COMMAND_CHANNEL_SHAPER | channel->idx;
	config_opts.dcpid = channel->dcp_idx;
	config_opts.shaper_config.crtcr = (token_rate->whole << 13) |
				 (token_rate->fraction);
	config_opts.shaper_config.crtbl = token_limit;
	config_opts.shaper_config.cpl = query_result.shaper_query.cpl;
	config_opts.shaper_config.ertcr = query_result.shaper_query.ertcr;
	config_opts.shaper_config.ertbl = query_result.shaper_query.ertbl;
	return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
}
EXPORT_SYMBOL(qman_ceetm_channel_set_commit_rate);

int qman_ceetm_channel_set_commit_rate_bps(struct qm_ceetm_channel *channel,
					   u64 bps, u16 token_limit)
{
	struct qm_ceetm_rate token_rate;
	int ret;

	ret = qman_ceetm_bps2tokenrate(bps, &token_rate, 0);
	if (ret) {
		pr_err("Can not convert bps to token rate\n");
		return -EINVAL;
	}
	return qman_ceetm_channel_set_commit_rate(channel, &token_rate,
						  token_limit);
}
EXPORT_SYMBOL(qman_ceetm_channel_set_commit_rate_bps);

int qman_ceetm_channel_get_commit_rate(struct qm_ceetm_channel *channel,
				struct qm_ceetm_rate *token_rate,
				u16 *token_limit)
{
	struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
	struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
	int ret;

	query_opts.cid = CEETM_COMMAND_CHANNEL_SHAPER | channel->idx;
	query_opts.dcpid = channel->dcp_idx;

	ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result);
	if (ret | !query_result.shaper_query.crtcr |
			 !query_result.shaper_query.crtbl) {
		pr_err("The channel commit rate or limit is not set\n");
		return -EINVAL;
	}
	token_rate->whole = query_result.shaper_query.crtcr >> 13;
	token_rate->fraction = query_result.shaper_query.crtcr & 0x1FFF;
	*token_limit = query_result.shaper_query.crtbl;
	return 0;
}
EXPORT_SYMBOL(qman_ceetm_channel_get_commit_rate);

int qman_ceetm_channel_get_commit_rate_bps(struct qm_ceetm_channel *channel,
					   u64 *bps, u16 *token_limit)
{
	struct qm_ceetm_rate token_rate;
	int ret;

	ret = qman_ceetm_channel_get_commit_rate(channel, &token_rate,
						 token_limit);
	if (ret) {
		pr_err("The channel CR rate or limit is not available\n");
		return -EINVAL;
	}

	return qman_ceetm_tokenrate2bps(&token_rate, bps, 0);
}
EXPORT_SYMBOL(qman_ceetm_channel_get_commit_rate_bps);

int qman_ceetm_channel_set_excess_rate(struct qm_ceetm_channel *channel,
					const struct qm_ceetm_rate *token_rate,
					u16 token_limit)
{
	struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
	struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
	struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
	int ret;

	query_opts.cid = CEETM_COMMAND_CHANNEL_SHAPER | channel->idx;
	query_opts.dcpid = channel->dcp_idx;
	ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result);
	if (ret) {
		pr_err("Fail to get the current channel shaper setting\n");
		return -EINVAL;
	}

	channel->er_token_rate.whole = token_rate->whole;
	channel->er_token_rate.fraction = token_rate->fraction;
	channel->er_token_bucket_limit = token_limit;
	config_opts.cid = CEETM_COMMAND_CHANNEL_SHAPER | channel->idx;
	config_opts.dcpid = channel->dcp_idx;
	config_opts.shaper_config.ertcr =
			 (token_rate->whole << 13) | (token_rate->fraction);
	config_opts.shaper_config.ertbl = token_limit;
	config_opts.shaper_config.cpl = query_result.shaper_query.cpl;
	config_opts.shaper_config.crtcr = query_result.shaper_query.crtcr;
	config_opts.shaper_config.crtbl = query_result.shaper_query.crtbl;
	return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
}
EXPORT_SYMBOL(qman_ceetm_channel_set_excess_rate);

int qman_ceetm_channel_set_excess_rate_bps(struct qm_ceetm_channel *channel,
					   u64 bps, u16 token_limit)
{
	struct qm_ceetm_rate token_rate;
	int ret;

	ret = qman_ceetm_bps2tokenrate(bps, &token_rate, 0);
	if (ret) {
		pr_err("Can not convert bps to token rate\n");
		return -EINVAL;
	}
	return qman_ceetm_channel_set_excess_rate(channel, &token_rate,
						  token_limit);
}
EXPORT_SYMBOL(qman_ceetm_channel_set_excess_rate_bps);

int qman_ceetm_channel_get_excess_rate(struct qm_ceetm_channel *channel,
					struct qm_ceetm_rate *token_rate,
					u16 *token_limit)
{
	struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
	struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
	int ret;

	query_opts.cid = CEETM_COMMAND_CHANNEL_SHAPER | channel->idx;
	query_opts.dcpid = channel->dcp_idx;
	ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result);
	if (ret | !query_result.shaper_query.ertcr |
			 !query_result.shaper_query.ertbl) {
		pr_err("The channel excess rate or limit is not set\n");
		return -EINVAL;
	}
	token_rate->whole = query_result.shaper_query.ertcr >> 13;
	token_rate->fraction = query_result.shaper_query.ertcr & 0x1FFF;
	*token_limit = query_result.shaper_query.ertbl;
	return 0;
}
EXPORT_SYMBOL(qman_ceetm_channel_get_excess_rate);

int qman_ceetm_channel_get_excess_rate_bps(struct qm_ceetm_channel *channel,
					   u64 *bps, u16 *token_limit)
{
	struct qm_ceetm_rate token_rate;
	int ret;

	ret = qman_ceetm_channel_get_excess_rate(channel, &token_rate,
						 token_limit);
	if (ret) {
		pr_err("The channel ER rate or limit is not available\n");
		return -EINVAL;
	}

	return qman_ceetm_tokenrate2bps(&token_rate, bps, 0);
}
EXPORT_SYMBOL(qman_ceetm_channel_get_excess_rate_bps);

int qman_ceetm_channel_set_weight(struct qm_ceetm_channel *channel,
						u16 token_limit)
{
	struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;

	if (channel->shaper_enable) {
		pr_err("This channel is a shaped one\n");
		return -EINVAL;
	}

	channel->cr_token_bucket_limit = token_limit;
	config_opts.cid = CEETM_COMMAND_CHANNEL_SHAPER | channel->idx;
	config_opts.dcpid = channel->dcp_idx;
	config_opts.shaper_config.crtbl = token_limit;
	return	qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
}
EXPORT_SYMBOL(qman_ceetm_channel_set_weight);

int qman_ceetm_channel_get_weight(struct qm_ceetm_channel *channel,
					u16 *token_limit)
{
	struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
	struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
	int ret;

	query_opts.cid = CEETM_COMMAND_CHANNEL_SHAPER | channel->idx;
	query_opts.dcpid = channel->dcp_idx;
	ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result);
	if (ret | !query_result.shaper_query.crtbl) {
		pr_err("This unshaped channel's uFQ wight is unavailable\n");
		return -EINVAL;
	}
	*token_limit = query_result.shaper_query.crtbl;
	return 0;
}
EXPORT_SYMBOL(qman_ceetm_channel_get_weight);

int qman_ceetm_channel_set_group(struct qm_ceetm_channel *channel, int group_b,
				unsigned int prio_a, unsigned int prio_b)
{
	struct qm_mcc_ceetm_class_scheduler_config config_opts;
	struct qm_mcr_ceetm_class_scheduler_query query_result;
	int i;

	if (!prio_a || (prio_a > 7)) {
		pr_err("The priority of group A is out of range\n");
		return -EINVAL;
	}
	if ((group_b && !prio_b) || (prio_b > 7)) {
		pr_err("The priority of group B is out of range\n");
		return -EINVAL;
	}

	if (qman_ceetm_query_class_scheduler(channel, &query_result)) {
		pr_err("Can't query channel#%d's scheduler!\n", channel->idx);
		return -EINVAL;
	}

	config_opts.cqcid = channel->idx;
	config_opts.dcpid = channel->dcp_idx;
	if (!group_b)
		config_opts.gpc = (u8)((1 << 6) | prio_a);
	else
		config_opts.gpc = (u8)((prio_b << 3) | prio_a);

	for (i = 0; i < 8; i++)
		config_opts.w[i] = query_result.w[i];
	config_opts.crem = query_result.crem;
	config_opts.erem = query_result.erem;

	return qman_ceetm_configure_class_scheduler(&config_opts);
}
EXPORT_SYMBOL(qman_ceetm_channel_set_group);

int qman_ceetm_channel_get_group(struct qm_ceetm_channel *channel, int *group_b,
				unsigned int *prio_a, unsigned int *prio_b)
{
	struct qm_mcr_ceetm_class_scheduler_query query_result;

	if (qman_ceetm_query_class_scheduler(channel, &query_result)) {
		pr_err("Can't query channel#%d's scheduler!\n", channel->idx);
		return -EINVAL;
	}
	*group_b = (query_result.gpc >> 6) & 0x1;
	*prio_a = query_result.gpc & 0x7;
	*prio_b = (query_result.gpc >> 3) & 0x7;
	return 0;
}
EXPORT_SYMBOL(qman_ceetm_channel_get_group);

#define GROUP_A_ELIGIBILITY_SET		(1 << 8)
#define GROUP_B_ELIGIBILITY_SET		(1 << 9)
#define CQ_ELIGIBILITY_SET(n)		(1 << (7 - n))
int qman_ceetm_channel_set_group_cr_eligibility(struct qm_ceetm_channel
				*channel, int group_b, int cre)
{
	struct qm_mcc_ceetm_class_scheduler_config csch_config;
	struct qm_mcr_ceetm_class_scheduler_query csch_query;
	int i;

	if (qman_ceetm_query_class_scheduler(channel, &csch_query)) {
		pr_err("Cannot get the channel %d scheduler setting.\n",
						channel->idx);
		return -EINVAL;
	}
	csch_config.cqcid = channel->idx;
	csch_config.dcpid = channel->dcp_idx;
	csch_config.gpc = csch_query.gpc;
	for (i = 0; i < 8; i++)
		csch_config.w[i] = csch_query.w[i];
	csch_config.erem = csch_query.erem;
	if (group_b)
		csch_config.crem = (csch_query.crem & ~GROUP_B_ELIGIBILITY_SET)
					| (cre ? GROUP_B_ELIGIBILITY_SET : 0);
	else
		csch_config.crem = (csch_query.crem & ~GROUP_A_ELIGIBILITY_SET)
					| (cre ? GROUP_A_ELIGIBILITY_SET : 0);

	if (qman_ceetm_configure_class_scheduler(&csch_config)) {
		pr_err("Cannot config channel %d's scheduler with "
			"group_%c's cr eligibility\n", channel->idx,
			group_b ? 'b' : 'a');
		return -EINVAL;
	}

	return 0;
}
EXPORT_SYMBOL(qman_ceetm_channel_set_group_cr_eligibility);

int qman_ceetm_channel_set_group_er_eligibility(struct qm_ceetm_channel
				*channel, int group_b, int ere)
{
	struct qm_mcc_ceetm_class_scheduler_config csch_config;
	struct qm_mcr_ceetm_class_scheduler_query csch_query;
	int i;

	if (qman_ceetm_query_class_scheduler(channel, &csch_query)) {
		pr_err("Cannot get the channel %d scheduler setting.\n",
						channel->idx);
		return -EINVAL;
	}
	csch_config.cqcid = channel->idx;
	csch_config.dcpid = channel->dcp_idx;
	csch_config.gpc = csch_query.gpc;
	for (i = 0; i < 8; i++)
		csch_config.w[i] = csch_query.w[i];
	csch_config.crem = csch_query.crem;
	if (group_b)
		csch_config.erem = (csch_query.erem & ~GROUP_B_ELIGIBILITY_SET)
					| (ere ? GROUP_B_ELIGIBILITY_SET : 0);
	else
		csch_config.erem = (csch_query.erem & ~GROUP_A_ELIGIBILITY_SET)
					| (ere ? GROUP_A_ELIGIBILITY_SET : 0);

	if (qman_ceetm_configure_class_scheduler(&csch_config)) {
		pr_err("Cannot config channel %d's scheduler with "
			"group_%c's er eligibility\n", channel->idx,
			group_b ? 'b' : 'a');
		return -EINVAL;
	}

	return 0;
}
EXPORT_SYMBOL(qman_ceetm_channel_set_group_er_eligibility);

int qman_ceetm_channel_set_cq_cr_eligibility(struct qm_ceetm_channel *channel,
						unsigned int idx, int cre)
{
	struct qm_mcc_ceetm_class_scheduler_config csch_config;
	struct qm_mcr_ceetm_class_scheduler_query csch_query;
	int i;

	if (idx > 7) {
		pr_err("CQ index is out of range\n");
		return -EINVAL;
	}
	if (qman_ceetm_query_class_scheduler(channel, &csch_query)) {
		pr_err("Cannot get the channel %d scheduler setting.\n",
						channel->idx);
		return -EINVAL;
	}
	csch_config.cqcid = channel->idx;
	csch_config.dcpid = channel->dcp_idx;
	csch_config.gpc = csch_query.gpc;
	for (i = 0; i < 8; i++)
		csch_config.w[i] = csch_query.w[i];
	csch_config.erem = csch_query.erem;
	csch_config.crem = (csch_query.crem & ~CQ_ELIGIBILITY_SET(idx)) |
					(cre ? CQ_ELIGIBILITY_SET(idx) : 0);
	if (qman_ceetm_configure_class_scheduler(&csch_config)) {
		pr_err("Cannot config channel scheduler to set "
			"cr eligibility mask for CQ#%d\n", idx);
		return -EINVAL;
	}

	return 0;
}
EXPORT_SYMBOL(qman_ceetm_channel_set_cq_cr_eligibility);

int qman_ceetm_channel_set_cq_er_eligibility(struct qm_ceetm_channel *channel,
						unsigned int idx, int ere)
{
	struct qm_mcc_ceetm_class_scheduler_config csch_config;
	struct qm_mcr_ceetm_class_scheduler_query csch_query;
	int i;

	if (idx > 7) {
		pr_err("CQ index is out of range\n");
		return -EINVAL;
	}
	if (qman_ceetm_query_class_scheduler(channel, &csch_query)) {
		pr_err("Cannot get the channel %d scheduler setting.\n",
						channel->idx);
		return -EINVAL;
	}
	csch_config.cqcid = channel->idx;
	csch_config.dcpid = channel->dcp_idx;
	csch_config.gpc = csch_query.gpc;
	for (i = 0; i < 8; i++)
		csch_config.w[i] = csch_query.w[i];
	csch_config.crem = csch_query.crem;
	csch_config.erem = (csch_query.erem & ~CQ_ELIGIBILITY_SET(idx)) |
					(ere ? CQ_ELIGIBILITY_SET(idx) : 0);
	if (qman_ceetm_configure_class_scheduler(&csch_config)) {
		pr_err("Cannot config channel scheduler to set "
			"er eligibility mask for CQ#%d\n", idx);
		return -EINVAL;
	}
	return 0;
}
EXPORT_SYMBOL(qman_ceetm_channel_set_cq_er_eligibility);

int qman_ceetm_cq_claim(struct qm_ceetm_cq **cq,
		struct qm_ceetm_channel *channel, unsigned int idx,
		struct qm_ceetm_ccg *ccg)
{
	struct qm_ceetm_cq *p;
	struct qm_mcc_ceetm_cq_config cq_config;

	if (idx > 7) {
		pr_err("The independent class queue id is out of range\n");
		return -EINVAL;
	}

	list_for_each_entry(p, &channel->class_queues, node) {
		if (p->idx == idx) {
			pr_err("The CQ#%d has been claimed!\n", idx);
			return -EINVAL;
		}
	}

	p = kmalloc(sizeof(*p), GFP_KERNEL);
	if (!p) {
		pr_err("Can't allocate memory for CQ#%d!\n", idx);
		return -ENOMEM;
	}

	list_add_tail(&p->node, &channel->class_queues);
	p->idx = idx;
	p->is_claimed = 1;
	p->parent = channel;
	INIT_LIST_HEAD(&p->bound_lfqids);

	if (ccg) {
		cq_config.cqid = (channel->idx << 4) | idx;
		cq_config.dcpid = channel->dcp_idx;
		cq_config.ccgid = ccg->idx;
		if (qman_ceetm_configure_cq(&cq_config)) {
			pr_err("Can't configure the CQ#%d with CCGRID#%d\n",
						 idx, ccg->idx);
		return -EINVAL;
		}
	}

	*cq = p;
	return 0;
}
EXPORT_SYMBOL(qman_ceetm_cq_claim);

int qman_ceetm_cq_claim_A(struct qm_ceetm_cq **cq,
		struct qm_ceetm_channel *channel, unsigned int idx,
		struct qm_ceetm_ccg *ccg)
{
	struct qm_ceetm_cq *p;
	struct qm_mcc_ceetm_cq_config cq_config;

	if ((idx < 7) || (idx > 15)) {
		pr_err("This grouped class queue id is out of range\n");
		return -EINVAL;
	}

	list_for_each_entry(p, &channel->class_queues, node) {
		if (p->idx == idx) {
			pr_err("The CQ#%d has been claimed!\n", idx);
			return -EINVAL;
		}
	}

	p = kmalloc(sizeof(*p), GFP_KERNEL);
	if (!p) {
		pr_err("Can't allocate memory for CQ#%d!\n", idx);
		return -ENOMEM;
	}

	list_add_tail(&p->node, &channel->class_queues);
	p->idx = idx;
	p->is_claimed = 1;
	p->parent = channel;
	INIT_LIST_HEAD(&p->bound_lfqids);

	if (ccg) {
		cq_config.cqid = (channel->idx << 4) | idx;
		cq_config.dcpid = channel->dcp_idx;
		cq_config.ccgid = ccg->idx;
		if (qman_ceetm_configure_cq(&cq_config)) {
			pr_err("Can't configure the CQ#%d with CCGRID#%d\n",
						 idx, ccg->idx);
			return -EINVAL;
		}
	}
	*cq = p;
	return 0;
}
EXPORT_SYMBOL(qman_ceetm_cq_claim_A);

int qman_ceetm_cq_claim_B(struct qm_ceetm_cq **cq,
		struct qm_ceetm_channel *channel, unsigned int idx,
		struct qm_ceetm_ccg *ccg)
{
	struct qm_ceetm_cq *p;
	struct qm_mcc_ceetm_cq_config cq_config;

	if ((idx < 11) || (idx > 15)) {
		pr_err("This grouped class queue id is out of range\n");
		return -EINVAL;
	}

	list_for_each_entry(p, &channel->class_queues, node) {
		if (p->idx == idx) {
			pr_err("The CQ#%d has been claimed!\n", idx);
			return -EINVAL;
		}
	}

	p = kmalloc(sizeof(*p), GFP_KERNEL);
	if (!p) {
		pr_err("Can't allocate memory for CQ#%d!\n", idx);
		return -ENOMEM;
	}

	list_add_tail(&p->node, &channel->class_queues);
	p->idx = idx;
	p->is_claimed = 1;
	p->parent = channel;
	INIT_LIST_HEAD(&p->bound_lfqids);

	if (ccg) {
		cq_config.cqid = (channel->idx << 4) | idx;
		cq_config.dcpid = channel->dcp_idx;
		cq_config.ccgid = ccg->idx;
		if (qman_ceetm_configure_cq(&cq_config)) {
			pr_err("Can't configure the CQ#%d with CCGRID#%d\n",
					 idx, ccg->idx);
		return -EINVAL;
		}
	}
	*cq = p;
	return 0;
}
EXPORT_SYMBOL(qman_ceetm_cq_claim_B);

int qman_ceetm_cq_release(struct qm_ceetm_cq *cq)
{
	if (!list_empty(&cq->bound_lfqids)) {
		pr_err("The CQ#%d has unreleased LFQID\n", cq->idx);
		return -EBUSY;
	}
	list_del(&cq->node);
	qman_ceetm_drain_cq(cq);
	kfree(cq);
	return 0;
}
EXPORT_SYMBOL(qman_ceetm_cq_release);

int qman_ceetm_set_queue_weight(struct qm_ceetm_cq *cq,
				struct qm_ceetm_weight_code *weight_code)
{
	struct qm_mcc_ceetm_class_scheduler_config config_opts;
	struct qm_mcr_ceetm_class_scheduler_query query_result;
	int i;

	if (cq->idx < 8) {
		pr_err("Can not set weight for ungrouped class queue\n");
		return -EINVAL;
	}

	if (qman_ceetm_query_class_scheduler(cq->parent, &query_result)) {
		pr_err("Can't query channel#%d's scheduler!\n",
						cq->parent->idx);
		return -EINVAL;
	}

	config_opts.cqcid = cq->parent->idx;
	config_opts.dcpid = cq->parent->dcp_idx;
	config_opts.crem = query_result.crem;
	config_opts.erem = query_result.erem;
	config_opts.gpc = query_result.gpc;
	for (i = 0; i < 8; i++)
		config_opts.w[i] = query_result.w[i];
	config_opts.w[cq->idx - 8] = ((weight_code->y << 3) |
						(weight_code->x & 0x7));
	return qman_ceetm_configure_class_scheduler(&config_opts);
}
EXPORT_SYMBOL(qman_ceetm_set_queue_weight);

int qman_ceetm_get_queue_weight(struct qm_ceetm_cq *cq,
				struct qm_ceetm_weight_code *weight_code)
{
	struct qm_mcr_ceetm_class_scheduler_query query_result;

	if (cq->idx < 8) {
		pr_err("Can not get weight for ungrouped class queue\n");
		return -EINVAL;
	}

	if (qman_ceetm_query_class_scheduler(cq->parent,
						&query_result)) {
		pr_err("Can't get the weight code for CQ#%d!\n", cq->idx);
		return -EINVAL;
	}
	weight_code->y = query_result.w[cq->idx - 8] >> 3;
	weight_code->x = query_result.w[cq->idx - 8] & 0x7;

	return 0;
}
EXPORT_SYMBOL(qman_ceetm_get_queue_weight);

/* The WBFS code is represent as {x,y}, the effect wieght can be calculated as:
 *	effective weight = 2^x / (1 - (y/64))
 *			 = 2^(x+6) / (64 - y)
 */
static void reduce_fraction(u32 *n, u32 *d)
{
	u32 factor = 2;
	u32 lesser = (*n < *d) ? *n : *d;
	/* If factor exceeds the square-root of the lesser of *n and *d,
	 * then there's no point continuing. Proof: if there was a factor
	 * bigger than the square root, that would imply there exists
	 * another factor smaller than the square-root with which it
	 * multiplies to give 'lesser' - but that's a contradiction
	 * because the other factor would have already been found and
	 * divided out.
	 */
	while ((factor * factor) <= lesser) {
		/* If 'factor' is a factor of *n and *d, divide them both
		 * by 'factor' as many times as possible.
		 */
		while (!(*n % factor) && !(*d % factor)) {
			*n /= factor;
			*d /= factor;
			lesser /= factor;
		}
		if (factor == 2)
			factor = 3;
		else
			factor += 2;
	}
}

int qman_ceetm_wbfs2ratio(struct qm_ceetm_weight_code *weight_code,
				u32 *numerator,
				u32 *denominator)
{
	*numerator = (u32) 1 << (weight_code->x + 6);
	*denominator = 64 - weight_code->y;
	reduce_fraction(numerator, denominator);
	return 0;
}
EXPORT_SYMBOL(qman_ceetm_wbfs2ratio);

/* For a given x, the weight is between 2^x (inclusive) and 2^(x+1) (exclusive).
 * So find 'x' by range, and then estimate 'y' using:
 *		64 - y	= 2^(x + 6) / weight
 *			= 2^(x + 6) / (n/d)
 *			= d * 2^(x+6) / n
 *		y = 64 - (d * 2^(x+6) / n)
 */
int qman_ceetm_ratio2wbfs(u32 numerator,
				u32 denominator,
				struct qm_ceetm_weight_code *weight_code,
				int rounding)
{
	unsigned int y, x = 0;
	/* search incrementing 'x' until:
	 * weight < 2^(x+1)
	 *    n/d < 2^(x+1)
	 *	n < d * 2^(x+1)
	 */
	while ((x < 8) && (numerator >= (denominator << (x + 1))))
		x++;
	if (x >= 8)
		return -ERANGE;
	/* because of the subtraction, use '-rounding' */
	y = 64 - ROUNDING(denominator << (x + 6), numerator, -rounding);
	if (y >= 32)
		return -ERANGE;
	weight_code->x = x;
	weight_code->y = y;
	return 0;
}
EXPORT_SYMBOL(qman_ceetm_ratio2wbfs);

int qman_ceetm_set_queue_weight_in_ratio(struct qm_ceetm_cq *cq, u32 ratio)
{
	struct qm_ceetm_weight_code weight_code;

	if (qman_ceetm_ratio2wbfs(ratio, 100, &weight_code, 0)) {
		pr_err("Cannot get wbfs code for cq %x\n", cq->idx);
		return -EINVAL;
	}
	return qman_ceetm_set_queue_weight(cq, &weight_code);
}
EXPORT_SYMBOL(qman_ceetm_set_queue_weight_in_ratio);

int qman_ceetm_get_queue_weight_in_ratio(struct qm_ceetm_cq *cq, u32 *ratio)
{
	struct qm_ceetm_weight_code weight_code;
	u32 n, d;

	if (qman_ceetm_get_queue_weight(cq, &weight_code)) {
		pr_err("Cannot query the weight code for cq%x\n", cq->idx);
		return -EINVAL;
	}

	if (qman_ceetm_wbfs2ratio(&weight_code, &n, &d)) {
		pr_err("Cannot get the ratio with wbfs code\n");
		return -EINVAL;
	}

	*ratio = (n * (u32)100) / d;
	return 0;
}
EXPORT_SYMBOL(qman_ceetm_get_queue_weight_in_ratio);

int qman_ceetm_cq_get_dequeue_statistics(struct qm_ceetm_cq *cq, u32 flags,
					u64 *frame_count, u64 *byte_count)
{
	struct qm_mcr_ceetm_statistics_query result;
	u16 cid, command_type;
	enum qm_dc_portal dcp_idx;
	int ret;

	cid = (cq->parent->idx << 4) | cq->idx;
	dcp_idx = cq->parent->dcp_idx;
	if (flags == QMAN_CEETM_FLAG_CLEAR_STATISTICS_COUNTER)
		command_type = CEETM_QUERY_DEQUEUE_CLEAR_STATISTICS;
	else
		command_type = CEETM_QUERY_DEQUEUE_STATISTICS;

	ret = qman_ceetm_query_statistics(cid, dcp_idx, command_type, &result);
	if (ret) {
		pr_err("Can't query the statistics of CQ#%d!\n", cq->idx);
		return -EINVAL;
	}

	*frame_count = result.frm_cnt;
	*byte_count = result.byte_cnt;
	return 0;
}
EXPORT_SYMBOL(qman_ceetm_cq_get_dequeue_statistics);

int qman_ceetm_drain_cq(struct qm_ceetm_cq *cq)
{
	struct qm_mcr_ceetm_cq_peek_pop_xsfdrread ppxr;
	int ret;

	do {
		ret = qman_ceetm_cq_peek_pop_xsfdrread(cq, 1, 0, &ppxr);
		if (ret) {
			pr_err("Failed to pop frame from CQ\n");
			return -EINVAL;
		}
	} while (!(ppxr.stat & 0x2));

	return 0;
}
EXPORT_SYMBOL(qman_ceetm_drain_cq);

#define CEETM_LFQMT_LFQID_MSB 0xF00000
#define CEETM_LFQMT_LFQID_LSB 0x000FFF
int qman_ceetm_lfq_claim(struct qm_ceetm_lfq **lfq,
					struct qm_ceetm_cq *cq)
{
	struct qm_ceetm_lfq *p;
	u32 lfqid;
	int ret = 0;
	struct qm_mcc_ceetm_lfqmt_config lfqmt_config;

	if (cq->parent->dcp_idx == qm_dc_portal_fman0) {
		ret = qman_alloc_ceetm0_lfqid(&lfqid);
	} else if (cq->parent->dcp_idx == qm_dc_portal_fman1) {
		ret = qman_alloc_ceetm1_lfqid(&lfqid);
	} else {
		pr_err("dcp_idx %u does not correspond to a known fman in this driver\n",
			cq->parent->dcp_idx);
		return -EINVAL;
	}

	if (ret) {
		pr_err("There is no lfqid avalaible for CQ#%d!\n", cq->idx);
		return -ENODEV;
	}
	p = kmalloc(sizeof(*p), GFP_KERNEL);
	if (!p)
		return -ENOMEM;
	p->idx = lfqid;
	p->dctidx = (u16)(lfqid & CEETM_LFQMT_LFQID_LSB);
	p->parent = cq->parent;
	list_add_tail(&p->node, &cq->bound_lfqids);

	lfqmt_config.lfqid = CEETM_LFQMT_LFQID_MSB |
				(cq->parent->dcp_idx << 16) |
				(lfqid & CEETM_LFQMT_LFQID_LSB);
	lfqmt_config.cqid = (cq->parent->idx << 4) | (cq->idx);
	lfqmt_config.dctidx = p->dctidx;
	if (qman_ceetm_configure_lfqmt(&lfqmt_config)) {
		pr_err("Can't configure LFQMT for LFQID#%d @ CQ#%d\n",
				lfqid, cq->idx);
		return -EINVAL;
	}
	*lfq = p;
	return 0;
}
EXPORT_SYMBOL(qman_ceetm_lfq_claim);

int qman_ceetm_lfq_release(struct qm_ceetm_lfq *lfq)
{
	if (lfq->parent->dcp_idx == qm_dc_portal_fman0) {
		qman_release_ceetm0_lfqid(lfq->idx);
	} else if (lfq->parent->dcp_idx == qm_dc_portal_fman1) {
		qman_release_ceetm1_lfqid(lfq->idx);
	} else {
		pr_err("dcp_idx %u does not correspond to a known fman in this driver\n",
			lfq->parent->dcp_idx);
		return -EINVAL;
	}
	list_del(&lfq->node);
	kfree(lfq);
	return 0;
}
EXPORT_SYMBOL(qman_ceetm_lfq_release);

int qman_ceetm_lfq_set_context(struct qm_ceetm_lfq *lfq, u64 context_a,
							u32 context_b)
{
	struct qm_mcc_ceetm_dct_config dct_config;
	lfq->context_a = context_a;
	lfq->context_b = context_b;
	dct_config.dctidx = (u16)lfq->dctidx;
	dct_config.dcpid = lfq->parent->dcp_idx;
	dct_config.context_b = context_b;
	dct_config.context_a = context_a;
	return qman_ceetm_configure_dct(&dct_config);
}
EXPORT_SYMBOL(qman_ceetm_lfq_set_context);

int qman_ceetm_lfq_get_context(struct qm_ceetm_lfq *lfq, u64 *context_a,
							u32 *context_b)
{
	struct qm_mcc_ceetm_dct_query dct_query;
	struct qm_mcr_ceetm_dct_query query_result;

	dct_query.dctidx = (u16)lfq->dctidx;
	dct_query.dcpid = lfq->parent->dcp_idx;
	if (qman_ceetm_query_dct(&dct_query, &query_result)) {
		pr_err("Can't query LFQID#%d's context!\n", lfq->idx);
		return -EINVAL;
	}
	*context_a = query_result.context_a;
	*context_b = query_result.context_b;
	return 0;
}
EXPORT_SYMBOL(qman_ceetm_lfq_get_context);

int qman_ceetm_create_fq(struct qm_ceetm_lfq *lfq, struct qman_fq *fq)
{
	spin_lock_init(&fq->fqlock);
	fq->fqid = lfq->idx;
	fq->flags = QMAN_FQ_FLAG_NO_MODIFY;
	if (lfq->ern)
		fq->cb.ern = lfq->ern;
#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
	if (unlikely(find_empty_fq_table_entry(&fq->key, fq)))
		return -ENOMEM;
#endif
	return 0;
}
EXPORT_SYMBOL(qman_ceetm_create_fq);

int qman_ceetm_ccg_claim(struct qm_ceetm_ccg **ccg,
				struct qm_ceetm_channel *channel,
				unsigned int idx,
				void (*cscn)(struct qm_ceetm_ccg *,
					void *cb_ctx,
					int congested),
				void *cb_ctx)
{
	struct qm_ceetm_ccg *p;

	if (idx > 15) {
		pr_err("The given ccg index is out of range\n");
		return -EINVAL;
	}

	list_for_each_entry(p, &channel->ccgs, node) {
		if (p->idx == idx) {
			pr_err("The CCG#%d has been claimed\n", idx);
			return -EINVAL;
		}
	}

	p = kmalloc(sizeof(*p), GFP_KERNEL);
	if (!p) {
		pr_err("Can't allocate memory for CCG#%d!\n", idx);
		return -ENOMEM;
	}

	list_add_tail(&p->node, &channel->ccgs);

	p->idx = idx;
	p->parent = channel;
	p->cb = cscn;
	p->cb_ctx = cb_ctx;
	INIT_LIST_HEAD(&p->cb_node);

	*ccg = p;
	return 0;
}
EXPORT_SYMBOL(qman_ceetm_ccg_claim);

int qman_ceetm_ccg_release(struct qm_ceetm_ccg *ccg)
{
	unsigned long irqflags __maybe_unused;
	struct qm_mcc_ceetm_ccgr_config config_opts;
	int ret = 0;
	struct qman_portal *p = get_affine_portal();

	memset(&config_opts, 0, sizeof(struct qm_mcc_ceetm_ccgr_config));
	spin_lock_irqsave(&p->ccgr_lock, irqflags);
	if (!list_empty(&ccg->cb_node))
		list_del(&ccg->cb_node);
	config_opts.ccgrid = CEETM_CCGR_CM_CONFIGURE |
				(ccg->parent->idx << 4) | ccg->idx;
	config_opts.dcpid = ccg->parent->dcp_idx;
	config_opts.we_mask = QM_CCGR_WE_CSCN_TUPD;
	config_opts.cm_config.cscn_tupd = PORTAL_IDX(p);
	ret = qman_ceetm_configure_ccgr(&config_opts);
	spin_unlock_irqrestore(&p->ccgr_lock, irqflags);

	list_del(&ccg->node);
	kfree(ccg);
	return ret;
}
EXPORT_SYMBOL(qman_ceetm_ccg_release);

int qman_ceetm_ccg_set(struct qm_ceetm_ccg *ccg, u16 we_mask,
				const struct qm_ceetm_ccg_params *params)
{
	struct qm_mcc_ceetm_ccgr_config config_opts;
	unsigned long irqflags __maybe_unused;
	int ret;
	struct qman_portal *p;

	if (((ccg->parent->idx << 4) | ccg->idx) >= (2 * __CGR_NUM))
		return -EINVAL;

	p = get_affine_portal();

	memset(&config_opts, 0, sizeof(struct qm_mcc_ceetm_ccgr_config));
	spin_lock_irqsave(&p->ccgr_lock, irqflags);

	config_opts.ccgrid = CEETM_CCGR_CM_CONFIGURE |
				(ccg->parent->idx << 4) | ccg->idx;
	config_opts.dcpid = ccg->parent->dcp_idx;
	config_opts.we_mask = we_mask;
	if (we_mask & QM_CCGR_WE_CSCN_EN) {
		config_opts.we_mask |= QM_CCGR_WE_CSCN_TUPD;
		config_opts.cm_config.cscn_tupd =
			QM_CGR_TARG_UDP_CTRL_WRITE_BIT | PORTAL_IDX(p);
	}
	config_opts.cm_config.ctl = (params->wr_en_g << 6) |
				(params->wr_en_y << 5) |
				(params->wr_en_r << 4) |
				(params->td_en << 3)   |
				(params->td_mode << 2) |
				(params->cscn_en << 1) |
				(params->mode);
	config_opts.cm_config.oal = params->oal;
	config_opts.cm_config.cs_thres = params->cs_thres_in;
	config_opts.cm_config.cs_thres_x = params->cs_thres_out;
	config_opts.cm_config.td_thres = params->td_thres;
	config_opts.cm_config.wr_parm_g = params->wr_parm_g;
	config_opts.cm_config.wr_parm_y = params->wr_parm_y;
	config_opts.cm_config.wr_parm_r = params->wr_parm_r;
	ret = qman_ceetm_configure_ccgr(&config_opts);
	if (ret) {
		pr_err("Configure CCGR CM failed!\n");
		goto release_lock;
	}

	if (we_mask & QM_CCGR_WE_CSCN_EN)
		if (list_empty(&ccg->cb_node))
			list_add(&ccg->cb_node,
				&p->ccgr_cbs[ccg->parent->dcp_idx]);
release_lock:
	spin_unlock_irqrestore(&p->ccgr_lock, irqflags);
	put_affine_portal();
	return ret;
}
EXPORT_SYMBOL(qman_ceetm_ccg_set);

#define CEETM_CCGR_CTL_MASK 0x01
int qman_ceetm_ccg_get(struct qm_ceetm_ccg *ccg,
				struct qm_ceetm_ccg_params *params)
{
	struct qm_mcc_ceetm_ccgr_query query_opts;
	struct qm_mcr_ceetm_ccgr_query query_result;

	query_opts.ccgrid = CEETM_CCGR_CM_QUERY |
				(ccg->parent->idx << 4) | ccg->idx;
	query_opts.dcpid = ccg->parent->dcp_idx;

	if (qman_ceetm_query_ccgr(&query_opts, &query_result)) {
		pr_err("Can't query CCGR#%d\n", ccg->idx);
		return -EINVAL;
	}

	params->wr_parm_r = query_result.cm_query.wr_parm_r;
	params->wr_parm_y = query_result.cm_query.wr_parm_y;
	params->wr_parm_g = query_result.cm_query.wr_parm_g;
	params->td_thres = query_result.cm_query.td_thres;
	params->cs_thres_out = query_result.cm_query.cs_thres_x;
	params->cs_thres_in = query_result.cm_query.cs_thres;
	params->oal = query_result.cm_query.oal;
	params->wr_en_g = (query_result.cm_query.ctl >> 6) &
					 CEETM_CCGR_CTL_MASK;
	params->wr_en_y = (query_result.cm_query.ctl >> 5) &
					 CEETM_CCGR_CTL_MASK;
	params->wr_en_r = (query_result.cm_query.ctl >> 4) &
					 CEETM_CCGR_CTL_MASK;
	params->td_en = (query_result.cm_query.ctl >> 3) &
					 CEETM_CCGR_CTL_MASK;
	params->td_mode = (query_result.cm_query.ctl >> 2) &
					 CEETM_CCGR_CTL_MASK;
	params->cscn_en = (query_result.cm_query.ctl >> 1) &
					 CEETM_CCGR_CTL_MASK;
	params->mode = (query_result.cm_query.ctl & CEETM_CCGR_CTL_MASK);

	return 0;
}
EXPORT_SYMBOL(qman_ceetm_ccg_get);

int qman_ceetm_ccg_get_reject_statistics(struct qm_ceetm_ccg *ccg, u32 flags,
					u64 *frame_count, u64 *byte_count)
{
	struct qm_mcr_ceetm_statistics_query result;
	u16 cid, command_type;
	enum qm_dc_portal dcp_idx;
	int ret;

	cid = (ccg->parent->idx << 4) | ccg->idx;
	dcp_idx = ccg->parent->dcp_idx;
	if (flags == QMAN_CEETM_FLAG_CLEAR_STATISTICS_COUNTER)
		command_type = CEETM_QUERY_REJECT_CLEAR_STATISTICS;
	else
		command_type = CEETM_QUERY_REJECT_STATISTICS;

	ret = qman_ceetm_query_statistics(cid, dcp_idx, command_type, &result);
	if (ret) {
		pr_err("Can't query the statistics of CCG#%d!\n", ccg->idx);
		return -EINVAL;
	}

	*frame_count = result.frm_cnt;
	*byte_count = result.byte_cnt;
	return 0;
}
EXPORT_SYMBOL(qman_ceetm_ccg_get_reject_statistics);

int qman_ceetm_cscn_swp_get(struct qm_ceetm_ccg *ccg,
					u16 swp_idx,
					unsigned int *cscn_enabled)
{
	struct qm_mcc_ceetm_ccgr_query query_opts;
	struct qm_mcr_ceetm_ccgr_query query_result;
	int i;

	DPA_ASSERT(swp_idx < 127);
	query_opts.ccgrid = CEETM_CCGR_CM_QUERY |
				(ccg->parent->idx << 4) | ccg->idx;
	query_opts.dcpid = ccg->parent->dcp_idx;

	if (qman_ceetm_query_ccgr(&query_opts, &query_result)) {
		pr_err("Can't query CCGR#%d\n", ccg->idx);
		return -EINVAL;
	}

	i = swp_idx / 32;
	i = 3 - i;
	*cscn_enabled = (query_result.cm_query.cscn_targ_swp[i] >>
							(31 - swp_idx % 32));

	return 0;
}
EXPORT_SYMBOL(qman_ceetm_cscn_swp_get);

int qman_ceetm_cscn_dcp_set(struct qm_ceetm_ccg *ccg,
				u16 dcp_idx,
				u8 vcgid,
				unsigned int cscn_enabled,
				u16 we_mask,
				const struct qm_ceetm_ccg_params *params)
{
	struct qm_mcc_ceetm_ccgr_config config_opts;
	int ret;

	config_opts.ccgrid = CEETM_CCGR_CM_CONFIGURE |
				(ccg->parent->idx << 4) | ccg->idx;
	config_opts.dcpid = ccg->parent->dcp_idx;
	config_opts.we_mask = we_mask | QM_CCGR_WE_CSCN_TUPD | QM_CCGR_WE_CDV;
	config_opts.cm_config.cdv = vcgid;
	config_opts.cm_config.cscn_tupd = (cscn_enabled << 15) |
					QM_CGR_TARG_UDP_CTRL_DCP | dcp_idx;
	config_opts.cm_config.ctl = (params->wr_en_g << 6) |
				(params->wr_en_y << 5) |
				(params->wr_en_r << 4) |
				(params->td_en << 3)   |
				(params->td_mode << 2) |
				(params->cscn_en << 1) |
				(params->mode);
	config_opts.cm_config.cs_thres = params->cs_thres_in;
	config_opts.cm_config.cs_thres_x = params->cs_thres_out;
	config_opts.cm_config.td_thres = params->td_thres;
	config_opts.cm_config.wr_parm_g = params->wr_parm_g;
	config_opts.cm_config.wr_parm_y = params->wr_parm_y;
	config_opts.cm_config.wr_parm_r = params->wr_parm_r;

	ret = qman_ceetm_configure_ccgr(&config_opts);
	if (ret) {
		pr_err("Configure CSCN_TARG_DCP failed!\n");
		return -EINVAL;
	}
	return 0;
}
EXPORT_SYMBOL(qman_ceetm_cscn_dcp_set);

int qman_ceetm_cscn_dcp_get(struct qm_ceetm_ccg *ccg,
				u16 dcp_idx,
				u8 *vcgid,
				unsigned int *cscn_enabled)
{
	struct qm_mcc_ceetm_ccgr_query query_opts;
	struct qm_mcr_ceetm_ccgr_query query_result;

	query_opts.ccgrid = CEETM_CCGR_CM_QUERY |
				(ccg->parent->idx << 4) | ccg->idx;
	query_opts.dcpid = ccg->parent->dcp_idx;

	if (qman_ceetm_query_ccgr(&query_opts, &query_result)) {
		pr_err("Can't query CCGR#%d\n", ccg->idx);
		return -EINVAL;
	}

	*vcgid = query_result.cm_query.cdv;
	*cscn_enabled = (query_result.cm_query.cscn_targ_dcp >>
							dcp_idx) & 0x1;
	return 0;
}
EXPORT_SYMBOL(qman_ceetm_cscn_dcp_get);

int qman_ceetm_querycongestion(struct __qm_mcr_querycongestion *ccg_state,
							unsigned int dcp_idx)
{
	struct qm_mc_command *mcc;
	struct qm_mc_result *mcr;
	struct qman_portal *p;
	unsigned long irqflags __maybe_unused;
	u8 res;
	int i;

	p = get_affine_portal();
	PORTAL_IRQ_LOCK(p, irqflags);

	mcc = qm_mc_start(&p->p);
	for (i = 0; i < 2; i++) {
		mcc->ccgr_query.ccgrid = CEETM_QUERY_CONGESTION_STATE | i;
		mcc->ccgr_query.dcpid = dcp_idx;
		qm_mc_commit(&p->p, QM_CEETM_VERB_CCGR_QUERY);

		while (!(mcr = qm_mc_result(&p->p)))
			cpu_relax();
		DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
						QM_CEETM_VERB_CCGR_QUERY);
		res = mcr->result;
		if (res == QM_MCR_RESULT_OK) {
			*(ccg_state + i) =
				mcr->ccgr_query.congestion_state.state;
		} else {
			pr_err("QUERY CEETM CONGESTION STATE failed\n");
			return -EIO;
		}
	}
	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();
	return 0;
}

int qman_set_wpm(int wpm_enable)
{
	return qm_set_wpm(wpm_enable);
}
EXPORT_SYMBOL(qman_set_wpm);

int qman_get_wpm(int *wpm_enable)
{
	return qm_get_wpm(wpm_enable);
}
EXPORT_SYMBOL(qman_get_wpm);

int qman_shutdown_fq(u32 fqid)
{
	struct qman_portal *p;
	unsigned long irqflags __maybe_unused;
	int ret;
	struct qm_portal *low_p;
	p = get_affine_portal();
	PORTAL_IRQ_LOCK(p, irqflags);
	low_p = &p->p;
	ret = qm_shutdown_fq(&low_p, 1, fqid);
	PORTAL_IRQ_UNLOCK(p, irqflags);
	put_affine_portal();
	return ret;
}

const struct qm_portal_config *qman_get_qm_portal_config(
						struct qman_portal *portal)
{
	return portal->sharing_redirect ? NULL : portal->config;
}