Commit 141dc943 authored by Alexey Marchuk's avatar Alexey Marchuk Committed by Konrad Sztyber
Browse files

lib/dma: Add API to get user context 



When a memory domain is created, the user can
pass a pointer to a context which contains another
pointer to a domain-specific context (aka user
context). For RDMA memory domain this user context
contains a pointer to a protection domain.
Further patches will use this user context in IO
path, but with current API we'll have to dereference
two pointers whcih isn't a performance way.
To have less pointer dereferences, embed the user
context at the end of memory domain structure and
add API to get opaque pointer to the context

Signed-off-by: default avatarAlexey Marchuk <alexeymar@nvidia.com>
Change-Id: I55c247e21c6e990dc3c2d7a02b2ff6e73548a898
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/24690


Reviewed-by: default avatarShuhei Matsumoto <smatsumoto@nvidia.com>
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarBen Walker <ben@nvidia.com>
parent ae7bf749

include/spdk/dma.h

+11 −0
Original line number Diff line number Diff line
@@ -206,6 +206,8 @@ struct spdk_memory_domain_ctx { 
	 * Depending on memory domain type, this pointer can be cast to a specific structure,

	 * e.g. to spdk_memory_domain_rdma_ctx structure for RDMA memory domain */

	void *user_ctx;

	/** size of \b user_ctx in bytes */

	size_t user_ctx_size;

};



/**
@@ -288,6 +290,15 @@ void spdk_memory_domain_set_memzero(struct spdk_memory_domain *domain, 
 */

struct spdk_memory_domain_ctx *spdk_memory_domain_get_context(struct spdk_memory_domain *domain);



/**

 * Get an opaque pointer to the user context and its size.

 *

 * \param domain Memory domain

 * \param ctx_size Stores size of the user context. NULL pointer is not allowed

 * \return User context pointer

 */

void *spdk_memory_domain_get_user_context(struct spdk_memory_domain *domain, size_t *ctx_size);



/**

 * Get type of the DMA device that can access this memory domain

 *

lib/dma/Makefile

+2 −2
Original line number Diff line number Diff line
@@ -6,8 +6,8 @@ 
SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)

include $(SPDK_ROOT_DIR)/mk/spdk.common.mk



SO_VER := 4

SO_MINOR := 1

SO_VER := 5

SO_MINOR := 0

SO_SUFFIX := $(SO_VER).$(SO_MINOR)



LIBNAME = dma

lib/dma/dma.c

+32 −5
Original line number Diff line number Diff line
@@ -22,6 +22,8 @@ struct spdk_memory_domain { 
	TAILQ_ENTRY(spdk_memory_domain) link;

	struct spdk_memory_domain_ctx *ctx;

	char *id;

	size_t user_ctx_size;

	uint8_t user_ctx[];

};



static struct spdk_memory_domain g_system_domain = {
@@ -47,18 +49,24 @@ spdk_memory_domain_create(struct spdk_memory_domain **_domain, enum spdk_dma_dev 
			  struct spdk_memory_domain_ctx *ctx, const char *id)

{

	struct spdk_memory_domain *domain;

	size_t ctx_size;

	size_t ctx_size, user_ctx_size = 0;



	if (!_domain) {

		return -EINVAL;

	}



	if (ctx && ctx->size == 0) {

	if (ctx) {

		if (ctx->size == 0) {

			SPDK_ERRLOG("Context size can't be 0\n");

			return -EINVAL;

		}

		if (ctx->user_ctx &&

		    offsetof(struct spdk_memory_domain_ctx, user_ctx_size) + sizeof(ctx->user_ctx_size) <= ctx->size) {

			user_ctx_size = ctx->user_ctx_size;

		}

	}



	domain = calloc(1, sizeof(*domain));

	domain = calloc(1, sizeof(*domain) + user_ctx_size);

	if (!domain) {

		SPDK_ERRLOG("Failed to allocate memory");

		return -ENOMEM;
@@ -87,6 +95,12 @@ spdk_memory_domain_create(struct spdk_memory_domain **_domain, enum spdk_dma_dev 
		domain->ctx->size = ctx_size;

	}



	if (user_ctx_size) {

		assert(ctx);

		memcpy(domain->user_ctx, ctx->user_ctx, user_ctx_size);

		domain->user_ctx_size = user_ctx_size;

	}



	domain->type = type;



	pthread_mutex_lock(&g_dma_mutex);
@@ -160,6 +174,19 @@ spdk_memory_domain_get_context(struct spdk_memory_domain *domain) 
	return domain->ctx;

}



void *

spdk_memory_domain_get_user_context(struct spdk_memory_domain *domain, size_t *ctx_size)

{

	assert(domain);



	if (!domain->user_ctx_size) {

		return NULL;

	}



	*ctx_size = domain->user_ctx_size;

	return domain->user_ctx;

}



/* We have to use the typedef in the function declaration to appease astyle. */

typedef enum spdk_dma_device_type spdk_dma_device_type_t;

lib/dma/spdk_dma.map

+1 −0
Original line number Diff line number Diff line
@@ -10,6 +10,7 @@ 
	spdk_memory_domain_set_data_transfer;

	spdk_memory_domain_set_memzero;

	spdk_memory_domain_get_context;

	spdk_memory_domain_get_user_context;

	spdk_memory_domain_get_dma_device_type;

	spdk_memory_domain_get_dma_device_id;

	spdk_memory_domain_destroy;

lib/rdma_utils/rdma_utils.c

+2 −1
Original line number Diff line number Diff line
@@ -443,7 +443,7 @@ struct spdk_memory_domain * 
spdk_rdma_utils_get_memory_domain(struct ibv_pd *pd)

{

	struct rdma_utils_memory_domain *domain = NULL;

	struct spdk_memory_domain_ctx ctx;

	struct spdk_memory_domain_ctx ctx = {};

	int rc;



	pthread_mutex_lock(&g_memory_domains_lock);
@@ -467,6 +467,7 @@ spdk_rdma_utils_get_memory_domain(struct ibv_pd *pd) 
	domain->rdma_ctx.ibv_pd = pd;

	ctx.size = sizeof(ctx);

	ctx.user_ctx = &domain->rdma_ctx;

	ctx.user_ctx_size = domain->rdma_ctx.size;



	rc = spdk_memory_domain_create(&domain->domain, SPDK_DMA_DEVICE_TYPE_RDMA, &ctx,

				       SPDK_RDMA_DMA_DEVICE);