Commit 34f6147c authored by Alexey Marchuk's avatar Alexey Marchuk Committed by Tomasz Zawadzki
Browse files

nvme/rdma: Use rdma_utils to manage memory domains 



Signed-off-by: default avatarAlexey Marchuk <alexeymar@nvidia.com>
Change-Id: Ia50cf1389798593a90a141b5f49641b9ce6b072d
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/23095


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Reviewed-by: default avatarShuhei Matsumoto <smatsumoto@nvidia.com>
Reviewed-by: default avatarBen Walker <ben@nvidia.com>
parent 0a9c0239

lib/nvme/nvme_rdma.c

+8 −86
Original line number Diff line number Diff line
@@ -76,14 +76,6 @@ 
#define NVME_RDMA_POLL_GROUP_CHECK_QPN(_rqpair, qpn)				\

	((_rqpair)->rdma_qp && (_rqpair)->rdma_qp->qp->qp_num == (qpn))	\



struct nvme_rdma_memory_domain {

	TAILQ_ENTRY(nvme_rdma_memory_domain) link;

	uint32_t ref;

	struct ibv_pd *pd;

	struct spdk_memory_domain *domain;

	struct spdk_memory_domain_rdma_ctx rdma_ctx;

};



enum nvme_rdma_wr_type {

	RDMA_WR_TYPE_RECV,

	RDMA_WR_TYPE_SEND,
@@ -227,7 +219,7 @@ struct nvme_rdma_qpair { 
	TAILQ_HEAD(, spdk_nvme_rdma_req)	free_reqs;

	TAILQ_HEAD(, spdk_nvme_rdma_req)	outstanding_reqs;



	struct nvme_rdma_memory_domain		*memory_domain;

	struct spdk_memory_domain		*memory_domain;



	/* Count of outstanding send objects */

	uint16_t				current_num_sends;
@@ -315,79 +307,6 @@ static struct nvme_rdma_poller *nvme_rdma_poll_group_get_poller(struct nvme_rdma 
static void nvme_rdma_poll_group_put_poller(struct nvme_rdma_poll_group *group,

		struct nvme_rdma_poller *poller);



static TAILQ_HEAD(, nvme_rdma_memory_domain) g_memory_domains = TAILQ_HEAD_INITIALIZER(

			g_memory_domains);

static pthread_mutex_t g_memory_domains_lock = PTHREAD_MUTEX_INITIALIZER;



static struct nvme_rdma_memory_domain *

nvme_rdma_get_memory_domain(struct ibv_pd *pd)

{

	struct nvme_rdma_memory_domain *domain = NULL;

	struct spdk_memory_domain_ctx ctx;

	int rc;



	pthread_mutex_lock(&g_memory_domains_lock);



	TAILQ_FOREACH(domain, &g_memory_domains, link) {

		if (domain->pd == pd) {

			domain->ref++;

			pthread_mutex_unlock(&g_memory_domains_lock);

			return domain;

		}

	}



	domain = calloc(1, sizeof(*domain));

	if (!domain) {

		SPDK_ERRLOG("Memory allocation failed\n");

		pthread_mutex_unlock(&g_memory_domains_lock);

		return NULL;

	}



	domain->rdma_ctx.size = sizeof(domain->rdma_ctx);

	domain->rdma_ctx.ibv_pd = pd;

	ctx.size = sizeof(ctx);

	ctx.user_ctx = &domain->rdma_ctx;



	rc = spdk_memory_domain_create(&domain->domain, SPDK_DMA_DEVICE_TYPE_RDMA, &ctx,

				       SPDK_RDMA_DMA_DEVICE);

	if (rc) {

		SPDK_ERRLOG("Failed to create memory domain\n");

		free(domain);

		pthread_mutex_unlock(&g_memory_domains_lock);

		return NULL;

	}



	domain->pd = pd;

	domain->ref = 1;

	TAILQ_INSERT_TAIL(&g_memory_domains, domain, link);



	pthread_mutex_unlock(&g_memory_domains_lock);



	return domain;

}



static void

nvme_rdma_put_memory_domain(struct nvme_rdma_memory_domain *device)

{

	if (!device) {

		return;

	}



	pthread_mutex_lock(&g_memory_domains_lock);



	assert(device->ref > 0);



	device->ref--;



	if (device->ref == 0) {

		spdk_memory_domain_destroy(device->domain);

		TAILQ_REMOVE(&g_memory_domains, device, link);

		free(device);

	}



	pthread_mutex_unlock(&g_memory_domains_lock);

}



static int nvme_rdma_ctrlr_delete_io_qpair(struct spdk_nvme_ctrlr *ctrlr,

		struct spdk_nvme_qpair *qpair);
@@ -806,7 +725,7 @@ nvme_rdma_qpair_init(struct nvme_rdma_qpair *rqpair) 
		return -1;

	}



	rqpair->memory_domain = nvme_rdma_get_memory_domain(rqpair->rdma_qp->qp->pd);

	rqpair->memory_domain = spdk_rdma_utils_get_memory_domain(rqpair->rdma_qp->qp->pd);

	if (!rqpair->memory_domain) {

		SPDK_ERRLOG("Failed to get memory domain\n");

		return -1;
@@ -1443,7 +1362,7 @@ nvme_rdma_get_memory_translation(struct nvme_request *req, struct nvme_rdma_qpai 


		rc = spdk_memory_domain_translate_data(req->payload.opts->memory_domain,

						       req->payload.opts->memory_domain_ctx,

						       rqpair->memory_domain->domain, &ctx, _ctx->addr,

						       rqpair->memory_domain, &ctx, _ctx->addr,

						       _ctx->length, &dma_translation);

		if (spdk_unlikely(rc) || dma_translation.iov_count != 1) {

			SPDK_ERRLOG("DMA memory translation failed, rc %d, iov count %u\n", rc, dma_translation.iov_count);
@@ -2138,7 +2057,10 @@ nvme_rdma_ctrlr_delete_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ 
	nvme_rdma_qpair_abort_reqs(qpair, 0);

	nvme_qpair_deinit(qpair);



	nvme_rdma_put_memory_domain(rqpair->memory_domain);

	if (spdk_rdma_utils_put_memory_domain(rqpair->memory_domain) != 0) {

		SPDK_ERRLOG("Failed to release memory domain\n");

		assert(0);

	}



	spdk_free(rqpair);
@@ -3340,7 +3262,7 @@ nvme_rdma_ctrlr_get_memory_domains(const struct spdk_nvme_ctrlr *ctrlr, 
	struct nvme_rdma_qpair *rqpair = nvme_rdma_qpair(ctrlr->adminq);



	if (domains && array_size > 0) {

		domains[0] = rqpair->memory_domain->domain;

		domains[0] = rqpair->memory_domain;

	}



	return 1;

test/common/lib/test_rdma.c

+3 −0
Original line number Diff line number Diff line
@@ -43,6 +43,9 @@ DEFINE_STUB(spdk_rdma_provider_qp_flush_recv_wrs, int, (struct spdk_rdma_provide 
DEFINE_STUB(spdk_rdma_utils_create_mem_map, struct spdk_rdma_utils_mem_map *, (struct ibv_pd *pd,

		struct spdk_nvme_rdma_hooks *hooks, uint32_t access_flags), NULL)

DEFINE_STUB_V(spdk_rdma_utils_free_mem_map, (struct spdk_rdma_utils_mem_map **map));

DEFINE_STUB(spdk_rdma_utils_get_memory_domain, struct spdk_memory_domain *, (struct ibv_pd *pd),

	    NULL);

DEFINE_STUB(spdk_rdma_utils_put_memory_domain, int, (struct spdk_memory_domain *domain), 0);



/* used to mock out having to split an SGL over a memory region */

size_t g_mr_size;

test/unit/lib/nvme/nvme_rdma.c/nvme_rdma_ut.c

+11 −70
Original line number Diff line number Diff line
@@ -960,6 +960,7 @@ test_nvme_rdma_qpair_init(void) 
	struct nvme_rdma_qpair		rqpair = {};

	struct rdma_cm_id		cm_id = {};

	struct ibv_pd			*pd = (struct ibv_pd *)0xfeedbeef;

	struct spdk_memory_domain	*domain = (struct spdk_memory_domain *)0xf00dfeed;

	struct ibv_qp			qp = { .pd = pd };

	struct nvme_rdma_ctrlr		rctrlr = {};

	int				rc = 0;
@@ -971,6 +972,7 @@ test_nvme_rdma_qpair_init(void) 
	rqpair.qpair.ctrlr = &rctrlr.ctrlr;

	g_spdk_rdma_qp.qp = &qp;

	MOCK_SET(spdk_rdma_utils_get_pd, pd);

	MOCK_SET(spdk_rdma_utils_get_memory_domain, domain);



	rc = nvme_rdma_qpair_init(&rqpair);

	CU_ASSERT(rc == 0);
@@ -980,9 +982,10 @@ test_nvme_rdma_qpair_init(void) 
	CU_ASSERT(rqpair.max_recv_sge == NVME_RDMA_DEFAULT_RX_SGE);

	CU_ASSERT(rqpair.current_num_sends == 0);

	CU_ASSERT(rqpair.cq == (struct ibv_cq *)0xFEEDBEEF);

	CU_ASSERT(rqpair.memory_domain != NULL);

	CU_ASSERT(rqpair.memory_domain == domain);



	MOCK_CLEAR(spdk_rdma_utils_get_pd);

	MOCK_CLEAR(spdk_rdma_utils_get_memory_domain);

}



static void
@@ -1027,72 +1030,15 @@ test_nvme_rdma_qpair_submit_request(void) 
	nvme_rdma_free_reqs(&rqpair);

}



static void

test_nvme_rdma_memory_domain(void)

{

	struct nvme_rdma_memory_domain *domain_1 = NULL, *domain_2 = NULL, *domain_tmp;

	struct ibv_pd *pd_1 = (struct ibv_pd *)0x1, *pd_2 = (struct ibv_pd *)0x2;

	/* Counters below are used to check the number of created/destroyed rdma_dma_device objects.

	 * Since other unit tests may create dma_devices, we can't just check that the queue is empty or not */

	uint32_t dma_dev_count_start = 0, dma_dev_count = 0, dma_dev_count_end = 0;



	TAILQ_FOREACH(domain_tmp, &g_memory_domains, link) {

		dma_dev_count_start++;

	}



	/* spdk_memory_domain_create failed, expect fail */

	MOCK_SET(spdk_memory_domain_create, -1);

	domain_1 = nvme_rdma_get_memory_domain(pd_1);

	CU_ASSERT(domain_1 == NULL);

	MOCK_CLEAR(spdk_memory_domain_create);



	/* Normal scenario */

	domain_1 = nvme_rdma_get_memory_domain(pd_1);

	SPDK_CU_ASSERT_FATAL(domain_1 != NULL);

	CU_ASSERT(domain_1->domain != NULL);

	CU_ASSERT(domain_1->pd == pd_1);

	CU_ASSERT(domain_1->ref == 1);



	/* Request the same pd, ref counter increased */

	CU_ASSERT(nvme_rdma_get_memory_domain(pd_1) == domain_1);

	CU_ASSERT(domain_1->ref == 2);



	/* Request another pd */

	domain_2 = nvme_rdma_get_memory_domain(pd_2);

	SPDK_CU_ASSERT_FATAL(domain_2 != NULL);

	CU_ASSERT(domain_2->domain != NULL);

	CU_ASSERT(domain_2->pd == pd_2);

	CU_ASSERT(domain_2->ref == 1);



	TAILQ_FOREACH(domain_tmp, &g_memory_domains, link) {

		dma_dev_count++;

	}

	CU_ASSERT(dma_dev_count == dma_dev_count_start + 2);



	/* put domain_1, decrement refcount */

	nvme_rdma_put_memory_domain(domain_1);



	/* Release both devices */

	CU_ASSERT(domain_2->ref == 1);

	nvme_rdma_put_memory_domain(domain_1);

	nvme_rdma_put_memory_domain(domain_2);



	TAILQ_FOREACH(domain_tmp, &g_memory_domains, link) {

		dma_dev_count_end++;

	}

	CU_ASSERT(dma_dev_count_start == dma_dev_count_end);

}



static void

test_rdma_ctrlr_get_memory_domains(void)

{

	struct nvme_rdma_ctrlr rctrlr = {};

	struct nvme_rdma_qpair rqpair = {};

	struct spdk_memory_domain *domain = (struct spdk_memory_domain *)0xbaadbeef;

	struct nvme_rdma_memory_domain rdma_domain = { .domain = domain };

	struct spdk_memory_domain *domains[1] = {NULL};



	rqpair.memory_domain = &rdma_domain;

	rqpair.memory_domain = domain;

	rqpair.qpair.trtype = SPDK_NVME_TRANSPORT_RDMA;

	rctrlr.ctrlr.adminq = &rqpair.qpair;
@@ -1127,8 +1073,7 @@ test_rdma_get_memory_translation(void) 
	};

	int rc;



	rqpair.memory_domain = nvme_rdma_get_memory_domain(rqpair.rdma_qp->qp->pd);

	SPDK_CU_ASSERT_FATAL(rqpair.memory_domain != NULL);

	rqpair.memory_domain = (struct spdk_memory_domain *) 0xfeedbeef;



	/* case 1, using extended IO opts with DMA device.

	 * Test 1 - spdk_dma_translate_data error, expect fail */
@@ -1164,9 +1109,6 @@ test_rdma_get_memory_translation(void) 
	CU_ASSERT(rc == 0);

	CU_ASSERT(ctx.lkey == RDMA_UT_LKEY);

	CU_ASSERT(ctx.rkey == RDMA_UT_RKEY);



	/* Cleanup */

	nvme_rdma_put_memory_domain(rqpair.memory_domain);

}



static void
@@ -1450,7 +1392,6 @@ main(int argc, char **argv) 
	CU_ADD_TEST(suite, test_nvme_rdma_validate_cm_event);

	CU_ADD_TEST(suite, test_nvme_rdma_qpair_init);

	CU_ADD_TEST(suite, test_nvme_rdma_qpair_submit_request);

	CU_ADD_TEST(suite, test_nvme_rdma_memory_domain);

	CU_ADD_TEST(suite, test_rdma_ctrlr_get_memory_domains);

	CU_ADD_TEST(suite, test_rdma_get_memory_translation);

	CU_ADD_TEST(suite, test_get_rdma_qpair_from_wc);