1 files changed, 32 insertions, 16 deletions

diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c
index 218494c..8ccf1b9 100644
--- a/drivers/infiniband/sw/rdmavt/qp.c
+++ b/drivers/infiniband/sw/rdmavt/qp.c
@@ -1579,6 +1579,7 @@ static int rvt_post_one_wr(struct rvt_qp *qp,
 	int ret;
 	size_t cplen;
 	bool reserved_op;
+	int local_ops_delayed = 0;
 
 	BUILD_BUG_ON(IB_QPT_MAX >= (sizeof(u32) * BITS_PER_BYTE));
 
@@ -1592,25 +1593,37 @@ static int rvt_post_one_wr(struct rvt_qp *qp,
 	cplen = ret;
 
 	/*
-	 * Local operations including fast register and local invalidate
-	 * can be processed immediately w/o being posted to the send queue
-	 * if neither fencing nor completion generation is needed. However,
-	 * once fencing or completion is requested, direct processing of
-	 * following local operations must be disabled until all the local
-	 * operations posted to the send queue have completed. This is
-	 * necessary to ensure the correct ordering.
+	 * Local operations include fast register and local invalidate.
+	 * Fast register needs to be processed immediately because the
+	 * registered lkey may be used by following work requests and the
+	 * lkey needs to be valid at the time those requests are posted.
+	 * Local invalidate can be processed immediately if fencing is
+	 * not required and no previous local invalidate ops are pending.
+	 * Signaled local operations that have been processed immediately
+	 * need to have requests with "completion only" flags set posted
+	 * to the send queue in order to generate completions.
 	 */
-	if ((rdi->post_parms[wr->opcode].flags & RVT_OPERATION_LOCAL) &&
-	    !(wr->send_flags & (IB_SEND_FENCE | IB_SEND_SIGNALED)) &&
-	    !atomic_read(&qp->local_ops_pending)) {
-		struct ib_reg_wr *reg = reg_wr(wr);
-
+	if ((rdi->post_parms[wr->opcode].flags & RVT_OPERATION_LOCAL)) {
 		switch (wr->opcode) {
 		case IB_WR_REG_MR:
-			return rvt_fast_reg_mr(qp, reg->mr, reg->key,
-					       reg->access);
+			ret = rvt_fast_reg_mr(qp,
+					      reg_wr(wr)->mr,
+					      reg_wr(wr)->key,
+					      reg_wr(wr)->access);
+			if (ret || !(wr->send_flags & IB_SEND_SIGNALED))
+				return ret;
+			break;
 		case IB_WR_LOCAL_INV:
-			return rvt_invalidate_rkey(qp, wr->ex.invalidate_rkey);
+			if ((wr->send_flags & IB_SEND_FENCE) ||
+			    atomic_read(&qp->local_ops_pending)) {
+				local_ops_delayed = 1;
+			} else {
+				ret = rvt_invalidate_rkey(
+					qp, wr->ex.invalidate_rkey);
+				if (ret || !(wr->send_flags & IB_SEND_SIGNALED))
+					return ret;
+			}
+			break;
 		default:
 			return -EINVAL;
 		}
@@ -1675,7 +1688,10 @@ static int rvt_post_one_wr(struct rvt_qp *qp,
 	}
 
 	if (rdi->post_parms[wr->opcode].flags & RVT_OPERATION_LOCAL) {
-		atomic_inc(&qp->local_ops_pending);
+		if (local_ops_delayed)
+			atomic_inc(&qp->local_ops_pending);
+		else
+			wqe->wr.send_flags |= RVT_SEND_COMPLETION_ONLY;
 		wqe->ssn = 0;
 		wqe->psn = 0;
 		wqe->lpsn = 0;