Commit 6b314fb5 authored by Seth Howell's avatar Seth Howell Committed by Jim Harris
Browse files

nvme_rdma: properly separate alloc_reqs and register_reqs. 



The way these two functions were separated previously represented a
pretty sserious bug when doing a controller reset.

If there were any outstanding requests in the rqpair, they would get
overwritten during the call to nvme_rdma_qpair_register_reqs and the
application would never get a completion for the higher level requests.
The only thing that we need to do in this function is assign the proper
lkeys.

Change-Id: I304c70646daf9b563cd00badba7141e5e8653aad
Signed-off-by: default avatarSeth Howell <seth.howell@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/471659


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarAlexey Marchuk <alexeymar@mellanox.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
parent 4c1a18c4

lib/nvme/nvme_rdma.c

+24 −19
Original line number Diff line number Diff line
@@ -587,6 +587,8 @@ nvme_rdma_free_reqs(struct nvme_rdma_qpair *rqpair) 
static int

nvme_rdma_alloc_reqs(struct nvme_rdma_qpair *rqpair)

{

	int i;



	rqpair->rdma_reqs = calloc(rqpair->num_entries, sizeof(struct spdk_nvme_rdma_req));

	if (rqpair->rdma_reqs == NULL) {

		SPDK_ERRLOG("Failed to allocate rdma_reqs\n");
@@ -599,23 +601,6 @@ nvme_rdma_alloc_reqs(struct nvme_rdma_qpair *rqpair) 
		goto fail;

	}



	return 0;

fail:

	nvme_rdma_free_reqs(rqpair);

	return -ENOMEM;

}



static int

nvme_rdma_register_reqs(struct nvme_rdma_qpair *rqpair)

{

	int i;



	rqpair->cmd_mr = rdma_reg_msgs(rqpair->cm_id, rqpair->cmds,

				       rqpair->num_entries * sizeof(*rqpair->cmds));

	if (!rqpair->cmd_mr) {

		SPDK_ERRLOG("Unable to register cmd_mr\n");

		goto fail;

	}



	TAILQ_INIT(&rqpair->free_reqs);

	TAILQ_INIT(&rqpair->outstanding_reqs);
@@ -633,8 +618,6 @@ nvme_rdma_register_reqs(struct nvme_rdma_qpair *rqpair) 
		 * an NVMe-oF SGL is required, the length of

		 * this element may change. */

		rdma_req->send_sgl[0].addr = (uint64_t)cmd;

		rdma_req->send_sgl[0].lkey = rqpair->cmd_mr->lkey;



		rdma_req->send_wr.wr_id = (uint64_t)rdma_req;

		rdma_req->send_wr.next = NULL;

		rdma_req->send_wr.opcode = IBV_WR_SEND;
@@ -645,6 +628,28 @@ nvme_rdma_register_reqs(struct nvme_rdma_qpair *rqpair) 
		TAILQ_INSERT_TAIL(&rqpair->free_reqs, rdma_req, link);

	}



	return 0;

fail:

	nvme_rdma_free_reqs(rqpair);

	return -ENOMEM;

}



static int

nvme_rdma_register_reqs(struct nvme_rdma_qpair *rqpair)

{

	int i;



	rqpair->cmd_mr = rdma_reg_msgs(rqpair->cm_id, rqpair->cmds,

				       rqpair->num_entries * sizeof(*rqpair->cmds));

	if (!rqpair->cmd_mr) {

		SPDK_ERRLOG("Unable to register cmd_mr\n");

		goto fail;

	}



	for (i = 0; i < rqpair->num_entries; i++) {

		rqpair->rdma_reqs[i].send_sgl[0].lkey = rqpair->cmd_mr->lkey;

	}



	return 0;



fail: