Commit 2696886c authored by Alexey Marchuk's avatar Alexey Marchuk Committed by Jim Harris
Browse files

dma: Update translation result to hold iovec pointer 



In some cases a single virtually contriguos memory
buffer can be translated to several chunks of memory.
To make such translation possible, update structure
spdk_memory_domain_translation_result to use a pointer
to iovec.
Add a single iov structure or cases where translation
is always 1:1, it will make easier translation callback
implementation. For RDMA transport translation of address
is always 1:1, so treat iovcnt other than 1 as an
error.

Change-Id: I65605575d43a490490eba72c1eb19f3a09d55ec6
Signed-off-by: default avatarAlexey Marchuk <alexeymar@mellanox.com>
Signed-off-by: default avatarMax Gurtovoy <mgurtovoy@nvidia.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/9779


Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
parent 549bcdc0

include/spdk/dma.h

+8 −4
Original line number Diff line number Diff line
@@ -120,10 +120,14 @@ typedef int (*spdk_memory_domain_push_data_cb)(struct spdk_memory_domain *dst_do 
struct spdk_memory_domain_translation_result {

	/** size of this structure in bytes */

	size_t size;

	/** Address of data buffer translated into destination memory domain space */

	void *addr;

	/** Size of the data buffer */

	size_t len;

	/** Number of elements in iov */

	uint32_t iov_count;

	/** Translation results, holds single address, length pair. Should only be used if \b iov_count is 1 */

	struct iovec iov;

	/** Translation results, array of addresses and lengths. Should only be used if \b iov_count is

	 * bigger than 1. The implementer of the translation callback is responsible for allocating and

	 * storing of this array until IO request completes */

	struct iovec *iovs;

	/** Destination domain passed to translation function */

	struct spdk_memory_domain *dst_domain;

	union {

lib/nvme/nvme_rdma.c

+4 −4
Original line number Diff line number Diff line
@@ -1375,15 +1375,15 @@ nvme_rdma_get_memory_translation(struct nvme_request *req, struct nvme_rdma_qpai 
						       req->payload.opts->memory_domain_ctx,

						       rqpair->memory_domain->domain, &ctx, _ctx->addr,

						       _ctx->length, &dma_translation);

		if (spdk_unlikely(rc)) {

			SPDK_ERRLOG("DMA memory translation failed, rc %d\n", rc);

		if (spdk_unlikely(rc) || dma_translation.iov_count != 1) {

			SPDK_ERRLOG("DMA memory translation failed, rc %d, iov count %u\n", rc, dma_translation.iov_count);

			return rc;

		}



		_ctx->lkey = dma_translation.rdma.lkey;

		_ctx->rkey = dma_translation.rdma.rkey;

		_ctx->addr = dma_translation.addr;

		_ctx->length = dma_translation.len;

		_ctx->addr = dma_translation.iov.iov_base;

		_ctx->length = dma_translation.iov.iov_len;

	} else {

		rc = spdk_rdma_get_translation(rqpair->mr_map, _ctx->addr, _ctx->length, &rdma_translation);

		if (spdk_unlikely(rc)) {

test/dma/test_dma/test_dma.c

+3 −2
Original line number Diff line number Diff line
@@ -94,8 +94,9 @@ dma_test_translate_memory_cb(struct spdk_memory_domain *src_domain, void *src_do 
		return -1;

	}



	result->len = len;

	result->addr = addr;

	result->iov.iov_base = addr;

	result->iov.iov_len = len;

	result->iov_count = 1;

	result->rdma.lkey = ctx->mr->lkey;

	result->rdma.rkey = ctx->mr->rkey;

	result->dst_domain = dst_domain;

test/unit/lib/nvme/nvme_rdma.c/nvme_rdma_ut.c

+6 −0
Original line number Diff line number Diff line
@@ -1274,12 +1274,18 @@ test_rdma_get_memory_translation(void) 
	MOCK_CLEAR(spdk_memory_domain_translate_data);



	/* Test 2 - expect pass */

	g_memory_translation_translation.iov_count = 1;

	g_memory_translation_translation.iov.iov_base = ctx.addr + 1;

	g_memory_translation_translation.iov.iov_len = ctx.length;

	g_memory_translation_translation.rdma.lkey = 123;

	g_memory_translation_translation.rdma.rkey = 321;



	rc = nvme_rdma_get_memory_translation(&req, &rqpair, &ctx);

	CU_ASSERT(rc == 0);

	CU_ASSERT(ctx.lkey == g_memory_translation_translation.rdma.lkey);

	CU_ASSERT(ctx.rkey == g_memory_translation_translation.rdma.rkey);

	CU_ASSERT(ctx.addr == g_memory_translation_translation.iov.iov_base);

	CU_ASSERT(ctx.length == g_memory_translation_translation.iov.iov_len);



	/* case 2, using rdma translation

	 * Test 1 - spdk_rdma_get_translation error, expect fail */