Commit eb2ec1b0 authored by Ben Walker's avatar Ben Walker Committed by Daniel Verkamp
Browse files

nvme/rdma: Reap multiple completions per poll 



This is more CPU efficient than only grabbing one
completion per call to ibv_poll_cq.

Change-Id: I0c70d33639f0f345482d9e7c810f9c6723937058
Signed-off-by: default avatarBen Walker <benjamin.walker@intel.com>
parent 378fc778

lib/nvme/nvme_rdma.c

+54 −62
Original line number Diff line number Diff line
@@ -1374,93 +1374,85 @@ nvme_rdma_qpair_fail(struct spdk_nvme_qpair *qpair) 
	return 0;

}



#define MAX_COMPLETIONS_PER_POLL 128



int

nvme_rdma_qpair_process_completions(struct spdk_nvme_qpair *qpair,

				    uint32_t max_completions)

{

	struct nvme_rdma_qpair *rqpair;

	struct ibv_wc wc;

	uint32_t size;

	int rc;

	uint32_t io_completed = 0;



	rqpair = nvme_rdma_qpair(qpair);

	size = rqpair->num_entries - 1U;

	if (!max_completions || max_completions > size) {

		max_completions = size;

	}

	struct nvme_rdma_qpair 	*rqpair = nvme_rdma_qpair(qpair);

	struct ibv_wc 		wc[MAX_COMPLETIONS_PER_POLL];

	int			i, rc, batch_size;

	uint32_t 		reaped;



	/* poll the send_cq */

	while (true) {

		rc = ibv_poll_cq(rqpair->cm_id->send_cq, 1, &wc);

		if (rc == 0) {

			break;

	if (max_completions == 0) {

		max_completions = rqpair->num_entries;

	} else {

		max_completions = nvme_min(max_completions, rqpair->num_entries);

	}



	/* Consume all send completions */

	reaped = 0;

	do {

		batch_size = nvme_min((max_completions - reaped),

				      MAX_COMPLETIONS_PER_POLL);

		rc = ibv_poll_cq(rqpair->cm_id->send_cq, batch_size, wc);

		if (rc < 0) {

			SPDK_ERRLOG("Poll CQ error!(%d): %s\n",

			SPDK_ERRLOG("Error polling CQ! (%d): %s\n",

				    errno, strerror(errno));

			return -1;

		}



		if (wc.status) {

			SPDK_ERRLOG("CQ completion error status %d, exiting handler\n",

				    wc.status);

			break;

		}



		if (wc.opcode == IBV_WC_SEND) {

			SPDK_TRACELOG(SPDK_TRACE_DEBUG, "CQ send completion\n");

		} else {

			SPDK_ERRLOG("Poll cq opcode type unknown!!!!! completion\n");

			return -1;

		}

	}



	/* poll the recv_cq */

	while (true) {

		rc = ibv_poll_cq(rqpair->cm_id->recv_cq, 1, &wc);

		if (rc == 0) {

		} else if (rc == 0) {

			/* Ran out of completions */

			break;

		}

		reaped += rc;

	} while (reaped < max_completions);



	/* Poll for recv completions */

	reaped = 0;

	do {

		batch_size = nvme_min((max_completions - reaped),

				      MAX_COMPLETIONS_PER_POLL);

		rc = ibv_poll_cq(rqpair->cm_id->recv_cq, batch_size, wc);

		if (rc < 0) {

			SPDK_ERRLOG("Poll CQ error!(%d): %s\n",

			SPDK_ERRLOG("Error polling CQ! (%d): %s\n",

				    errno, strerror(errno));

			return -1;

		} else if (rc == 0) {

			/* Ran out of completions */

			break;

		}



		if (wc.status) {

			SPDK_ERRLOG("CQ Completion Error For Response %lu: %d (%s)\n",

				    wc.wr_id, wc.status, ibv_wc_status_str(wc.status));

			break;

		reaped += rc;

		for (i = 0; i < rc; i++) {

			if (wc[i].status) {

				SPDK_ERRLOG("CQ error on Queue Pair %p, Response Index %lu (%d): %s\n",

					    qpair, wc[i].wr_id, wc[i].status, ibv_wc_status_str(wc[i].status));

				return -1;

			}



		if (wc.opcode == IBV_WC_RECV) {

			switch (wc[i].opcode) {

			case IBV_WC_RECV:

				SPDK_TRACELOG(SPDK_TRACE_DEBUG, "CQ recv completion\n");

			if (wc.byte_len < sizeof(struct spdk_nvme_cpl)) {

				SPDK_ERRLOG("recv length %u less than expected response size\n", wc.byte_len);

				if (wc[i].byte_len < sizeof(struct spdk_nvme_cpl)) {

					SPDK_ERRLOG("recv length %u less than expected response size\n", wc[i].byte_len);

					return -1;

				}



			rc = nvme_rdma_recv(rqpair, wc.wr_id);

			if (rc) {

				if (nvme_rdma_recv(rqpair, wc[i].wr_id)) {

					SPDK_ERRLOG("nvme_rdma_recv processing failure\n");



					return -1;

				}

			io_completed++;

		} else {

			SPDK_ERRLOG("Poll cq opcode type unknown!!!!! completion\n");

			return -1;

		}



		if (io_completed == max_completions) {

				break;



			default:

				SPDK_ERRLOG("Received an unexpected opcode on the CQ: %d\n", wc[i].opcode);

				return -1;

			}

		}

	} while (reaped < max_completions);



	return io_completed;

	return reaped;

}



uint32_t