Commit 4999a985 authored by Shuhei Matsumoto's avatar Shuhei Matsumoto Committed by Jim Harris
Browse files

nvme_rdma: Move responses from rdma_qpair into a separate object 



Move parallel arrays of response buffers and response SGLs from
qpair to a new responses object.

Use options to create the responses object.

Use spdk_zmalloc() to allocate the responses object because qpair
is also allocated by spdk_zmalloc().

The purpose is to share the code and the data structure between
SRQ is enabled and disabled.

Signed-off-by: default avatarShuhei Matsumoto <smatsumoto@nvidia.com>
Signed-off-by: default avatarDenis Nagorny <denisn@nvidia.com>
Signed-off-by: default avatarEvgeniy Kochetov <evgeniik@nvidia.com>
Change-Id: Ia23fe7328ae1f2f551fed5863fd1414f8567d602
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/14172


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarAleksey Marchuk <alexeymar@nvidia.com>
parent 9cdbd9e4

lib/nvme/nvme_rdma.c

+82 −52
Original line number Diff line number Diff line
@@ -167,6 +167,25 @@ struct nvme_rdma_qpair; 


typedef int (*nvme_rdma_cm_event_cb)(struct nvme_rdma_qpair *rqpair, int ret);



struct nvme_rdma_rsp_opts {

	uint16_t				num_entries;

	struct nvme_rdma_qpair			*rqpair;

	struct spdk_rdma_mem_map		*mr_map;

};



struct nvme_rdma_rsps {

	/* Parallel arrays of response buffers + response SGLs of size num_entries */

	struct ibv_sge				*rsp_sgls;

	struct spdk_nvme_rdma_rsp		*rsps;



	struct ibv_recv_wr			*rsp_recv_wrs;



	/* Count of outstanding recv objects */

	uint16_t				current_num_recvs;



	uint16_t				num_entries;

};



/* NVMe RDMA qpair extensions for spdk_nvme_qpair */

struct nvme_rdma_qpair {

	struct spdk_nvme_qpair			qpair;
@@ -187,11 +206,8 @@ struct nvme_rdma_qpair { 


	uint32_t				num_completions;



	/* Parallel arrays of response buffers + response SGLs of size num_entries */

	struct ibv_sge				*rsp_sgls;

	struct spdk_nvme_rdma_rsp		*rsps;

	struct nvme_rdma_rsps			*rsps;



	struct ibv_recv_wr			*rsp_recv_wrs;

	/*

	 * Array of num_entries NVMe commands registered as RDMA message buffers.

	 * Indexed by rdma_req->id.
@@ -205,8 +221,7 @@ struct nvme_rdma_qpair { 


	struct nvme_rdma_memory_domain		*memory_domain;



	/* Counts of outstanding send and recv objects */

	uint16_t				current_num_recvs;

	/* Count of outstanding send objects */

	uint16_t				current_num_sends;



	/* Placed at the end of the struct since it is not used frequently */
@@ -762,7 +777,6 @@ nvme_rdma_qpair_init(struct nvme_rdma_qpair *rqpair) 
	/* ibv_create_qp will change the values in attr.cap. Make sure we store the proper value. */

	rqpair->max_send_sge = spdk_min(NVME_RDMA_DEFAULT_TX_SGE, attr.cap.max_send_sge);

	rqpair->max_recv_sge = spdk_min(NVME_RDMA_DEFAULT_RX_SGE, attr.cap.max_recv_sge);

	rqpair->current_num_recvs = 0;

	rqpair->current_num_sends = 0;



	rqpair->cm_id->context = rqpair;
@@ -784,14 +798,14 @@ nvme_rdma_reset_failed_sends(struct nvme_rdma_qpair *rqpair, 
}



static void

nvme_rdma_reset_failed_recvs(struct nvme_rdma_qpair *rqpair,

nvme_rdma_reset_failed_recvs(struct nvme_rdma_rsps *rsps,

			     struct ibv_recv_wr *bad_recv_wr, int rc)

{

	SPDK_ERRLOG("Failed to post WRs on receive queue, errno %d (%s), bad_wr %p\n",

		    rc, spdk_strerror(rc), bad_recv_wr);

	while (bad_recv_wr != NULL) {

		assert(rqpair->current_num_recvs > 0);

		rqpair->current_num_recvs--;

		assert(rsps->current_num_recvs > 0);

		rsps->current_num_recvs--;

		bad_recv_wr = bad_recv_wr->next;

	}

}
@@ -819,7 +833,7 @@ nvme_rdma_qpair_submit_recvs(struct nvme_rdma_qpair *rqpair) 


	rc = spdk_rdma_qp_flush_recv_wrs(rqpair->rdma_qp, &bad_recv_wr);

	if (spdk_unlikely(rc)) {

		nvme_rdma_reset_failed_recvs(rqpair, bad_recv_wr, rc);

		nvme_rdma_reset_failed_recvs(rqpair->rsps, bad_recv_wr, rc);

	}



	return rc;
@@ -832,58 +846,64 @@ nvme_rdma_qpair_submit_recvs(struct nvme_rdma_qpair *rqpair) 
	}



static void

nvme_rdma_free_rsps(struct nvme_rdma_qpair *rqpair)

nvme_rdma_free_rsps(struct nvme_rdma_rsps *rsps)

{

	spdk_free(rqpair->rsps);

	rqpair->rsps = NULL;

	spdk_free(rqpair->rsp_sgls);

	rqpair->rsp_sgls = NULL;

	spdk_free(rqpair->rsp_recv_wrs);

	rqpair->rsp_recv_wrs = NULL;

	if (!rsps) {

		return;

	}



static int

nvme_rdma_create_rsps(struct nvme_rdma_qpair *rqpair)

	spdk_free(rsps->rsps);

	spdk_free(rsps->rsp_sgls);

	spdk_free(rsps->rsp_recv_wrs);

	spdk_free(rsps);

}



static struct nvme_rdma_rsps *

nvme_rdma_create_rsps(struct nvme_rdma_rsp_opts *opts)

{

	struct nvme_rdma_rsps *rsps;

	struct spdk_rdma_memory_translation translation;

	uint16_t i;

	int rc;



	rqpair->rsps = NULL;

	rqpair->rsp_recv_wrs = NULL;

	rsps = spdk_zmalloc(sizeof(*rsps), 0, NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA);

	if (!rsps) {

		SPDK_ERRLOG("Failed to allocate rsps object\n");

		return NULL;

	}



	rqpair->rsp_sgls = spdk_zmalloc(rqpair->num_entries * sizeof(*rqpair->rsp_sgls), 0, NULL,

	rsps->rsp_sgls = spdk_zmalloc(opts->num_entries * sizeof(*rsps->rsp_sgls), 0, NULL,

				      SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA);

	if (!rqpair->rsp_sgls) {

	if (!rsps->rsp_sgls) {

		SPDK_ERRLOG("Failed to allocate rsp_sgls\n");

		goto fail;

	}



	rqpair->rsp_recv_wrs = spdk_zmalloc(rqpair->num_entries * sizeof(*rqpair->rsp_recv_wrs), 0, NULL,

	rsps->rsp_recv_wrs = spdk_zmalloc(opts->num_entries * sizeof(*rsps->rsp_recv_wrs), 0, NULL,

					  SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA);

	if (!rqpair->rsp_recv_wrs) {

	if (!rsps->rsp_recv_wrs) {

		SPDK_ERRLOG("Failed to allocate rsp_recv_wrs\n");

		goto fail;

	}



	rqpair->rsps = spdk_zmalloc(rqpair->num_entries * sizeof(*rqpair->rsps), 0, NULL,

	rsps->rsps = spdk_zmalloc(opts->num_entries * sizeof(*rsps->rsps), 0, NULL,

				  SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA);

	if (!rqpair->rsps) {

	if (!rsps->rsps) {

		SPDK_ERRLOG("can not allocate rdma rsps\n");

		goto fail;

	}



	for (i = 0; i < rqpair->num_entries; i++) {

		struct ibv_sge *rsp_sgl = &rqpair->rsp_sgls[i];

		struct spdk_nvme_rdma_rsp *rsp = &rqpair->rsps[i];

		struct ibv_recv_wr *recv_wr = &rqpair->rsp_recv_wrs[i];

	for (i = 0; i < opts->num_entries; i++) {

		struct ibv_sge *rsp_sgl = &rsps->rsp_sgls[i];

		struct spdk_nvme_rdma_rsp *rsp = &rsps->rsps[i];

		struct ibv_recv_wr *recv_wr = &rsps->rsp_recv_wrs[i];



		rsp->rqpair = rqpair;

		rsp->rqpair = opts->rqpair;

		rsp->rdma_wr.type = RDMA_WR_TYPE_RECV;

		rsp->recv_wr = recv_wr;

		rsp_sgl->addr = (uint64_t)rsp;

		rsp_sgl->length = sizeof(struct spdk_nvme_cpl);

		rc = spdk_rdma_get_translation(rqpair->mr_map, rsp, sizeof(*rsp), &translation);

		rc = spdk_rdma_get_translation(opts->mr_map, rsp, sizeof(*rsp), &translation);

		if (rc) {

			goto fail;

		}
@@ -896,15 +916,16 @@ nvme_rdma_create_rsps(struct nvme_rdma_qpair *rqpair) 


		nvme_rdma_trace_ibv_sge(recv_wr->sg_list);



		spdk_rdma_qp_queue_recv_wrs(rqpair->rdma_qp, recv_wr);

		spdk_rdma_qp_queue_recv_wrs(opts->rqpair->rdma_qp, recv_wr);

	}



	rqpair->current_num_recvs = rqpair->num_entries;

	rsps->num_entries = opts->num_entries;

	rsps->current_num_recvs = opts->num_entries;



	return 0;

	return rsps;

fail:

	nvme_rdma_free_rsps(rqpair);

	return -ENOMEM;

	nvme_rdma_free_rsps(rsps);

	return NULL;

}



static void
@@ -1082,6 +1103,8 @@ static int nvme_rdma_stale_conn_retry(struct nvme_rdma_qpair *rqpair); 
static int

nvme_rdma_connect_established(struct nvme_rdma_qpair *rqpair, int ret)

{

	struct nvme_rdma_rsp_opts opts = {};



	if (ret == -ESTALE) {

		return nvme_rdma_stale_conn_retry(rqpair);

	} else if (ret) {
@@ -1104,9 +1127,12 @@ nvme_rdma_connect_established(struct nvme_rdma_qpair *rqpair, int ret) 
	}

	SPDK_DEBUGLOG(nvme, "RDMA requests created\n");



	ret = nvme_rdma_create_rsps(rqpair);

	SPDK_DEBUGLOG(nvme, "rc =%d\n", ret);

	if (ret < 0) {

	opts.num_entries = rqpair->num_entries;

	opts.rqpair = rqpair;

	opts.mr_map = rqpair->mr_map;



	rqpair->rsps = nvme_rdma_create_rsps(&opts);

	if (!rqpair->rsps) {

		SPDK_ERRLOG("Unable to create rqpair RDMA responses\n");

		return -1;

	}
@@ -1862,7 +1888,11 @@ nvme_rdma_qpair_disconnected(struct nvme_rdma_qpair *rqpair, int ret) 
		goto quiet;

	}



	if (rqpair->current_num_sends != 0 || rqpair->current_num_recvs != 0) {

	if (rqpair->rsps == NULL) {

		goto quiet;

	}



	if (rqpair->current_num_sends != 0 || rqpair->rsps->current_num_recvs != 0) {

		rqpair->state = NVME_RDMA_QPAIR_STATE_LINGERING;

		rqpair->evt_timeout_ticks = (NVME_RDMA_DISCONNECTED_QPAIR_TIMEOUT_US * spdk_get_ticks_hz()) /

					    SPDK_SEC_TO_USEC + spdk_get_ticks();
@@ -1883,7 +1913,7 @@ static int 
nvme_rdma_qpair_wait_until_quiet(struct nvme_rdma_qpair *rqpair)

{

	if (spdk_get_ticks() < rqpair->evt_timeout_ticks &&

	    (rqpair->current_num_sends != 0 || rqpair->current_num_recvs != 0)) {

	    (rqpair->current_num_sends != 0 || rqpair->rsps->current_num_recvs != 0)) {

		return -EAGAIN;

	}
@@ -2046,7 +2076,7 @@ nvme_rdma_ctrlr_delete_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ 
	nvme_rdma_put_memory_domain(rqpair->memory_domain);



	nvme_rdma_free_reqs(rqpair);

	nvme_rdma_free_rsps(rqpair);

	nvme_rdma_free_rsps(rqpair->rsps);

	spdk_free(rqpair);



	return 0;
@@ -2336,8 +2366,8 @@ nvme_rdma_request_ready(struct nvme_rdma_qpair *rqpair, struct spdk_nvme_rdma_re 


	nvme_rdma_req_complete(rdma_req, &rdma_rsp->cpl, true);



	assert(rqpair->current_num_recvs < rqpair->num_entries);

	rqpair->current_num_recvs++;

	assert(rqpair->rsps->current_num_recvs < rqpair->rsps->num_entries);

	rqpair->rsps->current_num_recvs++;



	recv_wr->next = NULL;

	nvme_rdma_trace_ibv_sge(recv_wr->sg_list);
@@ -2409,8 +2439,8 @@ nvme_rdma_process_recv_completion(struct ibv_wc *wc, struct nvme_rdma_wr *rdma_w 


	rdma_rsp = SPDK_CONTAINEROF(rdma_wr, struct spdk_nvme_rdma_rsp, rdma_wr);

	rqpair = rdma_rsp->rqpair;

	assert(rqpair->current_num_recvs > 0);

	rqpair->current_num_recvs--;

	assert(rqpair->rsps->current_num_recvs > 0);

	rqpair->rsps->current_num_recvs--;



	if (wc->status) {

		nvme_rdma_log_wc_status(rqpair, wc);

test/unit/lib/nvme/nvme_rdma.c/nvme_rdma_ut.c

+21 −20
Original line number Diff line number Diff line
@@ -510,31 +510,33 @@ test_nvme_rdma_create_reqs(void) 
static void

test_nvme_rdma_create_rsps(void)

{

	struct nvme_rdma_qpair rqpair = {};

	int rc;

	struct nvme_rdma_rsp_opts opts = {};

	struct nvme_rdma_rsps *rsps;

	struct spdk_rdma_qp *rdma_qp = (struct spdk_rdma_qp *)0xfeedf00d;

	struct nvme_rdma_qpair rqpair = { .rdma_qp = rdma_qp, };



	memset(&g_nvme_hooks, 0, sizeof(g_nvme_hooks));



	opts.rqpair = &rqpair;



	/* Test case 1 calloc false */

	rqpair.num_entries = 0;

	rc = nvme_rdma_create_rsps(&rqpair);

	CU_ASSERT(rqpair.rsp_sgls == NULL);

	SPDK_CU_ASSERT_FATAL(rc == -ENOMEM);

	opts.num_entries = 0;

	rsps = nvme_rdma_create_rsps(&opts);

	SPDK_CU_ASSERT_FATAL(rsps == NULL);



	/* Test case 2 calloc success */

	memset(&rqpair, 0, sizeof(rqpair));

	rqpair.num_entries = 1;



	rc = nvme_rdma_create_rsps(&rqpair);

	CU_ASSERT(rc == 0);

	CU_ASSERT(rqpair.rsp_sgls != NULL);

	CU_ASSERT(rqpair.rsp_recv_wrs != NULL);

	CU_ASSERT(rqpair.rsps != NULL);

	CU_ASSERT(rqpair.rsp_sgls[0].lkey == g_rdma_mr.lkey);

	CU_ASSERT(rqpair.rsp_sgls[0].addr == (uint64_t)&rqpair.rsps[0]);

	CU_ASSERT(rqpair.rsp_recv_wrs[0].wr_id == (uint64_t)&rqpair.rsps[0].rdma_wr);



	nvme_rdma_free_rsps(&rqpair);

	opts.num_entries = 1;



	rsps = nvme_rdma_create_rsps(&opts);

	SPDK_CU_ASSERT_FATAL(rsps != NULL);

	CU_ASSERT(rsps->rsp_sgls != NULL);

	CU_ASSERT(rsps->rsp_recv_wrs != NULL);

	CU_ASSERT(rsps->rsps != NULL);

	CU_ASSERT(rsps->rsp_sgls[0].lkey == g_rdma_mr.lkey);

	CU_ASSERT(rsps->rsp_sgls[0].addr == (uint64_t)&rsps->rsps[0]);

	CU_ASSERT(rsps->rsp_recv_wrs[0].wr_id == (uint64_t)&rsps->rsps[0].rdma_wr);



	nvme_rdma_free_rsps(rsps);

}



static void
@@ -1004,7 +1006,6 @@ test_nvme_rdma_qpair_init(void) 
	CU_ASSERT(rqpair.max_send_sge == NVME_RDMA_DEFAULT_TX_SGE);

	CU_ASSERT(rqpair.max_recv_sge == NVME_RDMA_DEFAULT_RX_SGE);

	CU_ASSERT(rqpair.current_num_sends == 0);

	CU_ASSERT(rqpair.current_num_recvs == 0);

	CU_ASSERT(rqpair.cq == (struct ibv_cq *)0xFEEDBEEF);

	CU_ASSERT(rqpair.memory_domain != NULL);