Commit e6ddb7df authored by Seth Howell's avatar Seth Howell Committed by Ben Walker
Browse files

rdma: use the new common poll group data buffer cache. 



This change is aimed at addressing github issue #555

Change-Id: I5112ac38c59f2f0a17d0c560e7e2f640a11f58a9
Signed-off-by: default avatarSeth Howell <seth.howell@intel.com>
Reviewed-on: https://review.gerrithub.io/c/440419


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Chandler-Test-Pool: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: default avatarZiye Yang <optimistyzy@gmail.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: default avatarDarek Stojaczyk <dariusz.stojaczyk@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
parent 8cb172f2

lib/nvmf/rdma.c

+48 −20
Original line number Diff line number Diff line
@@ -1115,6 +1115,26 @@ spdk_nvmf_rdma_check_contiguous_entries(uint64_t addr_1, uint64_t addr_2) 
	return addr_1 == addr_2;

}



static void

spdk_nvmf_rdma_request_free_buffers(struct spdk_nvmf_rdma_request *rdma_req,

				    struct spdk_nvmf_transport_poll_group *group, struct spdk_nvmf_transport *transport)

{

	for (uint32_t i = 0; i < rdma_req->req.iovcnt; i++) {

		if (group->buf_cache_count < group->buf_cache_size) {

			STAILQ_INSERT_HEAD(&group->buf_cache,

					   (struct spdk_nvmf_transport_pg_cache_buf *)rdma_req->data.buffers[i], link);

			group->buf_cache_count++;

		} else {

			spdk_mempool_put(transport->data_buf_pool, rdma_req->data.buffers[i]);

		}

		rdma_req->req.iov[i].iov_base = NULL;

		rdma_req->data.buffers[i] = NULL;

		rdma_req->req.iov[i].iov_len = 0;



	}

	rdma_req->data_from_pool = false;

}



typedef enum spdk_nvme_data_transfer spdk_nvme_data_transfer_t;



static spdk_nvme_data_transfer_t
@@ -1182,19 +1202,30 @@ spdk_nvmf_rdma_request_fill_iovs(struct spdk_nvmf_rdma_transport *rtransport, 
				 struct spdk_nvmf_rdma_device *device,

				 struct spdk_nvmf_rdma_request *rdma_req)

{

	struct spdk_nvmf_rdma_qpair		*rqpair;

	struct spdk_nvmf_rdma_poll_group	*rgroup;

	void					*buf = NULL;

	uint32_t				length = rdma_req->req.length;

	uint64_t				translation_len;

	uint32_t				i = 0;

	int					rc = 0;



	rqpair = SPDK_CONTAINEROF(rdma_req->req.qpair, struct spdk_nvmf_rdma_qpair, qpair);

	rgroup = rqpair->poller->group;

	rdma_req->req.iovcnt = 0;

	while (length) {

		if (!(STAILQ_EMPTY(&rgroup->group.buf_cache))) {

			rgroup->group.buf_cache_count--;

			buf = STAILQ_FIRST(&rgroup->group.buf_cache);

			STAILQ_REMOVE_HEAD(&rgroup->group.buf_cache, link);

			assert(buf != NULL);

		} else {

			buf = spdk_mempool_get(rtransport->transport.data_buf_pool);

			if (!buf) {

				rc = -ENOMEM;

				goto err_exit;

			}

		}



		rdma_req->req.iov[i].iov_base = (void *)((uintptr_t)(buf + NVMF_DATA_BUFFER_MASK) &

						~NVMF_DATA_BUFFER_MASK);
@@ -1221,12 +1252,9 @@ spdk_nvmf_rdma_request_fill_iovs(struct spdk_nvmf_rdma_transport *rtransport, 
	return rc;



err_exit:

	spdk_nvmf_rdma_request_free_buffers(rdma_req, &rgroup->group, &rtransport->transport);

	while (i) {

		i--;

		spdk_mempool_put(rtransport->transport.data_buf_pool, rdma_req->data.buffers[i]);

		rdma_req->req.iov[i].iov_base = NULL;

		rdma_req->req.iov[i].iov_len = 0;



		rdma_req->data.wr.sg_list[i].addr = 0;

		rdma_req->data.wr.sg_list[i].length = 0;

		rdma_req->data.wr.sg_list[i].lkey = 0;
@@ -1331,14 +1359,14 @@ static void 
nvmf_rdma_request_free(struct spdk_nvmf_rdma_request *rdma_req,

		       struct spdk_nvmf_rdma_transport	*rtransport)

{

	struct spdk_nvmf_rdma_qpair		*rqpair;

	struct spdk_nvmf_rdma_poll_group	*rgroup;



	if (rdma_req->data_from_pool) {

		/* Put the buffer/s back in the pool */

		for (uint32_t i = 0; i < rdma_req->req.iovcnt; i++) {

			spdk_mempool_put(rtransport->transport.data_buf_pool, rdma_req->data.buffers[i]);

			rdma_req->req.iov[i].iov_base = NULL;

			rdma_req->data.buffers[i] = NULL;

		}

		rdma_req->data_from_pool = false;

		rqpair = SPDK_CONTAINEROF(rdma_req->req.qpair, struct spdk_nvmf_rdma_qpair, qpair);

		rgroup = rqpair->poller->group;



		spdk_nvmf_rdma_request_free_buffers(rdma_req, &rgroup->group, &rtransport->transport);

	}

	rdma_req->req.length = 0;

	rdma_req->req.iovcnt = 0;

test/unit/lib/nvmf/rdma.c/rdma_ut.c

+98 −0
Original line number Diff line number Diff line
@@ -114,16 +114,27 @@ test_spdk_nvmf_rdma_request_parse_sgl(void) 
	struct spdk_nvmf_rdma_device device;

	struct spdk_nvmf_rdma_request rdma_req;

	struct spdk_nvmf_rdma_recv recv;

	struct spdk_nvmf_rdma_poll_group group;

	struct spdk_nvmf_rdma_qpair rqpair;

	struct spdk_nvmf_rdma_poller poller;

	union nvmf_c2h_msg cpl;

	union nvmf_h2c_msg cmd;

	struct spdk_nvme_sgl_descriptor *sgl;

	struct spdk_nvmf_transport_pg_cache_buf bufs[4];

	int rc, i;



	STAILQ_INIT(&group.group.buf_cache);

	group.group.buf_cache_size = 0;

	group.group.buf_cache_count = 0;

	poller.group = &group;

	rqpair.poller = &poller;



	sgl = &cmd.nvme_cmd.dptr.sgl1;

	rdma_req.recv = &recv;

	rdma_req.req.cmd = &cmd;

	rdma_req.req.rsp = &cpl;

	rdma_req.data.wr.sg_list = rdma_req.data.sgl;

	rdma_req.req.qpair = &rqpair.qpair;



	rtransport.transport.opts = g_rdma_ut_transport_opts;
@@ -227,6 +238,93 @@ test_spdk_nvmf_rdma_request_parse_sgl(void) 
	rc = spdk_nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req);



	CU_ASSERT(rc == -1);

	/* Test 3: use PG buffer cache */

	sgl->generic.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK;

	sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS;

	sgl->address = 0xFFFF;

	rdma_req.recv->buf = (void *)0xDDDD;

	g_rdma_mr.lkey = 0xABCD;

	sgl->keyed.key = 0xEEEE;



	for (i = 0; i < 4; i++) {

		STAILQ_INSERT_TAIL(&group.group.buf_cache, &bufs[i], link);

	}



	/* part 1: use the four buffers from the pg cache */



	group.group.buf_cache_size = 4;

	group.group.buf_cache_count = 4;

	MOCK_SET(spdk_mempool_get, (void *)0x2000);

	reset_nvmf_rdma_request(&rdma_req);

	sgl->keyed.length = rtransport.transport.opts.io_unit_size * 4;

	rc = spdk_nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req);



	SPDK_CU_ASSERT_FATAL(rc == 0);

	CU_ASSERT(rdma_req.data_from_pool == true);

	CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 4);

	CU_ASSERT((uint64_t)rdma_req.req.data == (((uint64_t)&bufs[0] + NVMF_DATA_BUFFER_MASK) &

			~NVMF_DATA_BUFFER_MASK));

	CU_ASSERT(rdma_req.data.wr.num_sge == 4);

	CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE);

	CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF);

	CU_ASSERT(group.group.buf_cache_count == 0);

	CU_ASSERT(STAILQ_EMPTY(&group.group.buf_cache));

	for (i = 0; i < 4; i++) {

		CU_ASSERT((uint64_t)rdma_req.data.buffers[i] == (uint64_t)&bufs[i]);

		CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (((uint64_t)&bufs[i] + NVMF_DATA_BUFFER_MASK) &

				~NVMF_DATA_BUFFER_MASK));

		CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size);

	}

	/* part 2: now that we have used the buffers from the cache, try again. We should get mempool buffers. */



	reset_nvmf_rdma_request(&rdma_req);

	rc = spdk_nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req);



	SPDK_CU_ASSERT_FATAL(rc == 0);

	CU_ASSERT(rdma_req.data_from_pool == true);

	CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 4);

	CU_ASSERT((uint64_t)rdma_req.req.data == 0x2000);

	CU_ASSERT(rdma_req.data.wr.num_sge == 4);

	CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE);

	CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF);

	CU_ASSERT(group.group.buf_cache_count == 0);

	CU_ASSERT(STAILQ_EMPTY(&group.group.buf_cache));

	for (i = 0; i < 4; i++) {

		CU_ASSERT((uint64_t)rdma_req.data.buffers[i] == 0x2000);

		CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000);

		CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size);

		CU_ASSERT(group.group.buf_cache_count == 0);

	}



	/* part 3: half and half */

	group.group.buf_cache_count = 2;



	for (i = 0; i < 2; i++) {

		STAILQ_INSERT_TAIL(&group.group.buf_cache, &bufs[i], link);

	}

	reset_nvmf_rdma_request(&rdma_req);

	rc = spdk_nvmf_rdma_request_parse_sgl(&rtransport, &device, &rdma_req);



	SPDK_CU_ASSERT_FATAL(rc == 0);

	CU_ASSERT(rdma_req.data_from_pool == true);

	CU_ASSERT(rdma_req.req.length == rtransport.transport.opts.io_unit_size * 4);

	CU_ASSERT((uint64_t)rdma_req.req.data == (((uint64_t)&bufs[0] + NVMF_DATA_BUFFER_MASK) &

			~NVMF_DATA_BUFFER_MASK));

	CU_ASSERT(rdma_req.data.wr.num_sge == 4);

	CU_ASSERT(rdma_req.data.wr.wr.rdma.rkey == 0xEEEE);

	CU_ASSERT(rdma_req.data.wr.wr.rdma.remote_addr == 0xFFFF);

	CU_ASSERT(group.group.buf_cache_count == 0);

	for (i = 0; i < 2; i++) {

		CU_ASSERT((uint64_t)rdma_req.data.buffers[i] == (uint64_t)&bufs[i]);

		CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == (((uint64_t)&bufs[i] + NVMF_DATA_BUFFER_MASK) &

				~NVMF_DATA_BUFFER_MASK));

		CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size);

	}

	for (i = 2; i < 4; i++) {

		CU_ASSERT((uint64_t)rdma_req.data.buffers[i] == 0x2000);

		CU_ASSERT(rdma_req.data.wr.sg_list[i].addr == 0x2000);

		CU_ASSERT(rdma_req.data.wr.sg_list[i].length == rtransport.transport.opts.io_unit_size);

	}

}



int main(int argc, char **argv)