Commit cce99060 authored by Shuhei Matsumoto's avatar Shuhei Matsumoto Committed by Tomasz Zawadzki
Browse files

nvme_rdma: Factor out send/recv completion from cq_process_completions() 



Factor out processing recv completion and send completion into helper
functions to make the following patches simpler.

Additionally, invert if condition to check if both send and recv are
completed to make the following patches simpler.

Signed-off-by: default avatarShuhei Matsumoto <smatsumoto@nvidia.com>
Change-Id: Idcd951adc7b42594e33e195e82122f6fe55bc4aa
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/14419


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarAleksey Marchuk <alexeymar@nvidia.com>
parent d7ad7bca

lib/nvme/nvme_rdma.c

+104 −82
Original line number Diff line number Diff line
@@ -2435,78 +2435,63 @@ nvme_rdma_log_wc_status(struct nvme_rdma_qpair *rqpair, struct ibv_wc *wc) 
	}

}



static int

nvme_rdma_cq_process_completions(struct ibv_cq *cq, uint32_t batch_size,

				 struct nvme_rdma_poller *poller,

				 struct nvme_rdma_qpair *rdma_qpair,

				 uint64_t *rdma_completions)

static inline int

nvme_rdma_process_recv_completion(struct ibv_wc *wc, struct nvme_rdma_wr *rdma_wr)

{

	struct ibv_wc			wc[MAX_COMPLETIONS_PER_POLL];

	struct nvme_rdma_qpair		*rqpair;

	struct spdk_nvme_rdma_req	*rdma_req;

	struct spdk_nvme_rdma_rsp	*rdma_rsp;

	struct nvme_rdma_wr		*rdma_wr;

	uint32_t			reaped = 0;

	int				completion_rc = 0;

	int				rc, i;



	rc = ibv_poll_cq(cq, batch_size, wc);

	if (rc < 0) {

		SPDK_ERRLOG("Error polling CQ! (%d): %s\n",

			    errno, spdk_strerror(errno));

		return -ECANCELED;

	} else if (rc == 0) {

		return 0;

	}



	for (i = 0; i < rc; i++) {

		rdma_wr = (struct nvme_rdma_wr *)wc[i].wr_id;

		switch (rdma_wr->type) {

		case RDMA_WR_TYPE_RECV:

	rdma_rsp = SPDK_CONTAINEROF(rdma_wr, struct spdk_nvme_rdma_rsp, rdma_wr);

	rqpair = rdma_rsp->rqpair;

	assert(rqpair->current_num_recvs > 0);

	rqpair->current_num_recvs--;



			if (wc[i].status) {

				nvme_rdma_log_wc_status(rqpair, &wc[i]);

	if (wc->status) {

		nvme_rdma_log_wc_status(rqpair, wc);

		nvme_rdma_fail_qpair(&rqpair->qpair, 0);

				completion_rc = -ENXIO;

				continue;

		return -ENXIO;

	}



	SPDK_DEBUGLOG(nvme, "CQ recv completion\n");



			if (wc[i].byte_len < sizeof(struct spdk_nvme_cpl)) {

				SPDK_ERRLOG("recv length %u less than expected response size\n", wc[i].byte_len);

	if (wc->byte_len < sizeof(struct spdk_nvme_cpl)) {

		SPDK_ERRLOG("recv length %u less than expected response size\n", wc->byte_len);

		nvme_rdma_fail_qpair(&rqpair->qpair, 0);

				completion_rc = -ENXIO;

				continue;

		return -ENXIO;

	}

	rdma_req = &rqpair->rdma_reqs[rdma_rsp->cpl.cid];

	rdma_req->completion_flags |= NVME_RDMA_RECV_COMPLETED;

	rdma_req->rsp_idx = rdma_rsp->idx;



			if ((rdma_req->completion_flags & NVME_RDMA_SEND_COMPLETED) != 0) {

	if ((rdma_req->completion_flags & NVME_RDMA_SEND_COMPLETED) == 0) {

		return 0;

	}



	if (spdk_unlikely(nvme_rdma_request_ready(rqpair, rdma_req))) {

		SPDK_ERRLOG("Unable to re-post rx descriptor\n");

		nvme_rdma_fail_qpair(&rqpair->qpair, 0);

					completion_rc = -ENXIO;

					continue;

		return -ENXIO;

	}

				reaped++;

	rqpair->num_completions++;

	return 1;

}

			break;



		case RDMA_WR_TYPE_SEND:

static inline int

nvme_rdma_process_send_completion(struct nvme_rdma_poller *poller,

				  struct nvme_rdma_qpair *rdma_qpair,

				  struct ibv_wc *wc, struct nvme_rdma_wr *rdma_wr)

{

	struct nvme_rdma_qpair		*rqpair;

	struct spdk_nvme_rdma_req	*rdma_req;



	rdma_req = SPDK_CONTAINEROF(rdma_wr, struct spdk_nvme_rdma_req, rdma_wr);



	/* If we are flushing I/O */

			if (wc[i].status) {

	if (wc->status) {

		rqpair = rdma_req->req ? nvme_rdma_qpair(rdma_req->req->qpair) : NULL;

		if (!rqpair) {

					rqpair = rdma_qpair != NULL ? rdma_qpair : get_rdma_qpair_from_wc(poller->group, &wc[i]);

			rqpair = rdma_qpair != NULL ? rdma_qpair : get_rdma_qpair_from_wc(poller->group, wc);

		}

		if (!rqpair) {

			/* When poll_group is used, several qpairs share the same CQ and it is possible to
@@ -2514,14 +2499,13 @@ nvme_rdma_cq_process_completions(struct ibv_cq *cq, uint32_t batch_size, 
			 * That happens due to qpair is destroyed while there are submitted but not completed send/receive

			 * Work Requests */

			assert(poller);

					continue;

			return 0;

		}

		assert(rqpair->current_num_sends > 0);

		rqpair->current_num_sends--;

				nvme_rdma_log_wc_status(rqpair, &wc[i]);

		nvme_rdma_log_wc_status(rqpair, wc);

		nvme_rdma_fail_qpair(&rqpair->qpair, 0);

				completion_rc = -ENXIO;

				continue;

		return -ENXIO;

	}



	/* We do not support Soft Roce anymore. Other than Soft Roce's bug, we should not
@@ -2534,22 +2518,60 @@ nvme_rdma_cq_process_completions(struct ibv_cq *cq, uint32_t batch_size, 
	assert(rqpair->current_num_sends > 0);

	rqpair->current_num_sends--;



			if ((rdma_req->completion_flags & NVME_RDMA_RECV_COMPLETED) != 0) {

	if ((rdma_req->completion_flags & NVME_RDMA_RECV_COMPLETED) == 0) {

		return 0;

	}



	if (spdk_unlikely(nvme_rdma_request_ready(rqpair, rdma_req))) {

		SPDK_ERRLOG("Unable to re-post rx descriptor\n");

		nvme_rdma_fail_qpair(&rqpair->qpair, 0);

					completion_rc = -ENXIO;

					continue;

		return -ENXIO;

	}

				reaped++;

	rqpair->num_completions++;

	return 1;

}



static int

nvme_rdma_cq_process_completions(struct ibv_cq *cq, uint32_t batch_size,

				 struct nvme_rdma_poller *poller,

				 struct nvme_rdma_qpair *rdma_qpair,

				 uint64_t *rdma_completions)

{

	struct ibv_wc			wc[MAX_COMPLETIONS_PER_POLL];

	struct nvme_rdma_wr		*rdma_wr;

	uint32_t			reaped = 0;

	int				completion_rc = 0;

	int				rc, _rc, i;



	rc = ibv_poll_cq(cq, batch_size, wc);

	if (rc < 0) {

		SPDK_ERRLOG("Error polling CQ! (%d): %s\n",

			    errno, spdk_strerror(errno));

		return -ECANCELED;

	} else if (rc == 0) {

		return 0;

	}



	for (i = 0; i < rc; i++) {

		rdma_wr = (struct nvme_rdma_wr *)wc[i].wr_id;

		switch (rdma_wr->type) {

		case RDMA_WR_TYPE_RECV:

			_rc = nvme_rdma_process_recv_completion(&wc[i], rdma_wr);

			break;



		case RDMA_WR_TYPE_SEND:

			_rc = nvme_rdma_process_send_completion(poller, rdma_qpair, &wc[i], rdma_wr);

			break;



		default:

			SPDK_ERRLOG("Received an unexpected opcode on the CQ: %d\n", rdma_wr->type);

			return -ECANCELED;

		}

		if (spdk_likely(_rc >= 0)) {

			reaped += _rc;

		} else {

			completion_rc = _rc;

		}

	}



	*rdma_completions += rc;