Commit 0a9c0239 authored by Alexey Marchuk's avatar Alexey Marchuk Committed by Tomasz Zawadzki
Browse files

lib/rdma_utils: Add API to get/put memory domain 



That allows to use common memory domain per PD
in different modules

Signed-off-by: default avatarAlexey Marchuk <alexeymar@nvidia.com>
Change-Id: I57ab7e2acd13529811f81a08211db2625ccd84cc
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/23094


Reviewed-by: default avatarBen Walker <ben@nvidia.com>
Reviewed-by: default avatarShuhei Matsumoto <smatsumoto@nvidia.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
parent 8ffb2c09

include/spdk_internal/rdma_utils.h

+19 −0
Original line number Diff line number Diff line
@@ -112,6 +112,25 @@ spdk_rdma_utils_get_pd(struct ibv_context *context); 
 */

void spdk_rdma_utils_put_pd(struct ibv_pd *pd);



/**

 * Get memory domain for the specified protection domain.

 *

 * If memory domain does not exist for the specified protection domain, it will be allocated.

 * If memory domain already exists, reference will be increased.

 *

 * \param pd Protection domain of memory domain

 * \return Pointer to memory domain or NULL;

 */

struct spdk_memory_domain *spdk_rdma_utils_get_memory_domain(struct ibv_pd *pd);



/**

 * Release a reference to a memory domain, which will be destroyed when reference becomes 0.

 *

 * \param _domain Pointer to memory domain

 * \return 0 on success, negated errno on failure

 */

int spdk_rdma_utils_put_memory_domain(struct spdk_memory_domain *_domain);



#ifdef __cplusplus

}

#endif

lib/rdma_utils/rdma_utils.c

+96 −0
Original line number Diff line number Diff line
@@ -31,6 +31,15 @@ struct spdk_rdma_utils_mem_map { 
	LIST_ENTRY(spdk_rdma_utils_mem_map)	link;

};



struct rdma_utils_memory_domain {

	TAILQ_ENTRY(rdma_utils_memory_domain) link;

	uint32_t ref;

	enum spdk_dma_device_type type;

	struct ibv_pd *pd;

	struct spdk_memory_domain *domain;

	struct spdk_memory_domain_rdma_ctx rdma_ctx;

};



static pthread_mutex_t g_dev_mutex = PTHREAD_MUTEX_INITIALIZER;

static struct ibv_context **g_ctx_list = NULL;

static TAILQ_HEAD(, rdma_utils_device) g_dev_list = TAILQ_HEAD_INITIALIZER(g_dev_list);
@@ -39,6 +48,10 @@ static LIST_HEAD(, spdk_rdma_utils_mem_map) g_rdma_utils_mr_maps = LIST_HEAD_INI 
			&g_rdma_utils_mr_maps);

static pthread_mutex_t g_rdma_mr_maps_mutex = PTHREAD_MUTEX_INITIALIZER;



static TAILQ_HEAD(, rdma_utils_memory_domain) g_memory_domains = TAILQ_HEAD_INITIALIZER(

			g_memory_domains);

static pthread_mutex_t g_memory_domains_lock = PTHREAD_MUTEX_INITIALIZER;



static int

rdma_utils_mem_notify(void *cb_ctx, struct spdk_mem_map *map,

		      enum spdk_mem_map_notify_action action,
@@ -423,3 +436,86 @@ _rdma_utils_fini(void) 
		g_ctx_list = NULL;

	}

}



struct spdk_memory_domain *

spdk_rdma_utils_get_memory_domain(struct ibv_pd *pd)

{

	struct rdma_utils_memory_domain *domain = NULL;

	struct spdk_memory_domain_ctx ctx;

	int rc;



	pthread_mutex_lock(&g_memory_domains_lock);



	TAILQ_FOREACH(domain, &g_memory_domains, link) {

		if (domain->pd == pd) {

			domain->ref++;

			pthread_mutex_unlock(&g_memory_domains_lock);

			return domain->domain;

		}

	}



	domain = calloc(1, sizeof(*domain));

	if (!domain) {

		SPDK_ERRLOG("Memory allocation failed\n");

		pthread_mutex_unlock(&g_memory_domains_lock);

		return NULL;

	}



	domain->rdma_ctx.size = sizeof(domain->rdma_ctx);

	domain->rdma_ctx.ibv_pd = pd;

	ctx.size = sizeof(ctx);

	ctx.user_ctx = &domain->rdma_ctx;



	rc = spdk_memory_domain_create(&domain->domain, SPDK_DMA_DEVICE_TYPE_RDMA, &ctx,

				       SPDK_RDMA_DMA_DEVICE);

	if (rc) {

		SPDK_ERRLOG("Failed to create memory domain\n");

		free(domain);

		pthread_mutex_unlock(&g_memory_domains_lock);

		return NULL;

	}



	domain->pd = pd;

	domain->ref = 1;

	TAILQ_INSERT_TAIL(&g_memory_domains, domain, link);



	pthread_mutex_unlock(&g_memory_domains_lock);



	return domain->domain;

}



int

spdk_rdma_utils_put_memory_domain(struct spdk_memory_domain *_domain)

{

	struct rdma_utils_memory_domain *domain = NULL;



	if (!_domain) {

		return 0;

	}



	pthread_mutex_lock(&g_memory_domains_lock);



	TAILQ_FOREACH(domain, &g_memory_domains, link) {

		if (domain->domain == _domain) {

			break;

		}

	}



	if (!domain) {

		pthread_mutex_unlock(&g_memory_domains_lock);

		return -ENODEV;

	}

	assert(domain->ref > 0);



	domain->ref--;



	if (domain->ref == 0) {

		spdk_memory_domain_destroy(domain->domain);

		TAILQ_REMOVE(&g_memory_domains, domain, link);

		free(domain);

	}



	pthread_mutex_unlock(&g_memory_domains_lock);



	return 0;

}

lib/rdma_utils/spdk_rdma_utils.map

+2 −0
Original line number Diff line number Diff line
@@ -7,6 +7,8 @@ 
	spdk_rdma_utils_get_translation;

	spdk_rdma_utils_get_pd;

	spdk_rdma_utils_put_pd;

	spdk_rdma_utils_get_memory_domain;

	spdk_rdma_utils_put_memory_domain;



	local: *;

};

mk/spdk.lib_deps.mk

+1 −1
Original line number Diff line number Diff line
@@ -36,7 +36,7 @@ endif 
DEPDIRS-conf := log util

DEPDIRS-json := log util

DEPDIRS-rdma_provider := log util

DEPDIRS-rdma_utils := log util

DEPDIRS-rdma_utils := dma log util

DEPDIRS-reduce := log util

DEPDIRS-thread := log util trace

DEPDIRS-keyring := log util $(JSON_LIBS)

test/unit/lib/rdma/common.c/common_ut.c

+4 −0
Original line number Diff line number Diff line
@@ -17,6 +17,10 @@ DEFINE_STUB(spdk_mem_map_clear_translation, int, (struct spdk_mem_map *map, uint 
		uint64_t size), 0);

DEFINE_STUB(spdk_mem_map_translate, uint64_t, (const struct spdk_mem_map *map, uint64_t vaddr,

		uint64_t *size), 0);

DEFINE_STUB(spdk_memory_domain_create, int, (struct spdk_memory_domain **_domain,

		enum spdk_dma_device_type type,

		struct spdk_memory_domain_ctx *ctx, const char *id), 0);

DEFINE_STUB_V(spdk_memory_domain_destroy, (struct spdk_memory_domain *domain));



struct ut_rdma_device {

	struct ibv_context		*context;