Commit 8a01b4d6 authored by Alexey Marchuk's avatar Alexey Marchuk Committed by Tomasz Zawadzki
Browse files

lib/rdma_utils: Intorduce generic rdma utils lib 



This library holds generic rdma functions from
rdma_provider. That is done to avoid cross link
reference in future patches. The library will be
extended with new functionality

Signed-off-by: default avatarAlexey Marchuk <alexeymar@nvidia.com>
Change-Id: If1918efb0fe6f0baa77cf20f992fbd6a97de4264
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/23072


Reviewed-by: default avatarShuhei Matsumoto <smatsumoto@nvidia.com>
Reviewed-by: default avatarBen Walker <ben@nvidia.com>
Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
parent cf151d60

include/spdk_internal/rdma_provider.h

+0 −105
Original line number Diff line number Diff line
@@ -10,9 +10,6 @@ 
#include <rdma/rdma_cma.h>

#include <rdma/rdma_verbs.h>



/* Contains hooks definition */

#include "spdk/nvme.h"



/* rxe driver vendor_id has been changed from 0 to 0XFFFFFF in 0184afd15a141d7ce24c32c0d86a1e3ba6bc0eb3 */

#define SPDK_RDMA_RXE_VENDOR_ID_OLD 0

#define SPDK_RDMA_RXE_VENDOR_ID_NEW 0XFFFFFF
@@ -58,22 +55,6 @@ struct spdk_rdma_provider_qp { 
	bool shared_stats;

};



struct spdk_rdma_mem_map;



union spdk_rdma_mr {

	struct ibv_mr	*mr;

	uint64_t	key;

};



enum SPDK_RDMA_TRANSLATION_TYPE {

	SPDK_RDMA_TRANSLATION_MR = 0,

	SPDK_RDMA_TRANSLATION_KEY

};



struct spdk_rdma_memory_translation {

	union spdk_rdma_mr mr_or_key;

	uint8_t translation_type;

};

struct spdk_rdma_provider_srq_init_attr {

	struct ibv_pd *pd;

	struct spdk_rdma_provider_wr_stats *stats;
@@ -87,11 +68,6 @@ struct spdk_rdma_provider_srq { 
	bool shared_stats;

};



enum spdk_rdma_memory_map_role {

	SPDK_RDMA_MEMORY_MAP_ROLE_TARGET,

	SPDK_RDMA_MEMORY_MAP_ROLE_INITIATOR

};



/**

 * Create RDMA SRQ

 *
@@ -215,85 +191,4 @@ bool spdk_rdma_provider_qp_queue_recv_wrs(struct spdk_rdma_provider_qp *spdk_rdm 
int spdk_rdma_provider_qp_flush_recv_wrs(struct spdk_rdma_provider_qp *spdk_rdma_qp,

		struct ibv_recv_wr **bad_wr);



/**

 * Create a memory map which is used to register Memory Regions and perform address -> memory

 * key translations

 *

 * \param pd Protection Domain which will be used to create Memory Regions

 * \param hooks Optional hooks which are used to create Protection Domain or ger RKey

 * \param role Specifies whether this map is used by RDMA target or initiator, determines access flags of registered MRs

 * \return Pointer to memory map or NULL on failure

 */

struct spdk_rdma_mem_map *

spdk_rdma_create_mem_map(struct ibv_pd *pd, struct spdk_nvme_rdma_hooks *hooks,

			 enum spdk_rdma_memory_map_role role);



/**

 * Free previously allocated memory map

 *

 * \param map Pointer to memory map to free

 */

void spdk_rdma_free_mem_map(struct spdk_rdma_mem_map **map);



/**

 * Get a translation for the given address and length.

 *

 * Note: the user of this function should use address returned in \b translation structure

 *

 * \param map Pointer to translation map

 * \param address Memory address for translation

 * \param length Length of the memory address

 * \param[in,out] translation Pointer to translation result to be filled by this function

 * \retval -EINVAL if translation is not found

 * \retval 0 translation succeed

 */

int spdk_rdma_get_translation(struct spdk_rdma_mem_map *map, void *address,

			      size_t length, struct spdk_rdma_memory_translation *translation);



/**

 * Helper function for retrieving Local Memory Key. Should be applied to a translation

 * returned by \b spdk_rdma_get_translation

 *

 * \param translation Memory translation

 * \return Local Memory Key

 */

static inline uint32_t

spdk_rdma_memory_translation_get_lkey(struct spdk_rdma_memory_translation

				      *translation)

{

	return translation->translation_type == SPDK_RDMA_TRANSLATION_MR ?

	       translation->mr_or_key.mr->lkey : (uint32_t)translation->mr_or_key.key;

}



/**

 * Helper function for retrieving Remote Memory Key. Should be applied to a translation

 * returned by \b spdk_rdma_get_translation

 *

 * \param translation Memory translation

 * \return Remote Memory Key

 */

static inline uint32_t

spdk_rdma_memory_translation_get_rkey(struct spdk_rdma_memory_translation

				      *translation)

{

	return translation->translation_type == SPDK_RDMA_TRANSLATION_MR ?

	       translation->mr_or_key.mr->rkey : (uint32_t)translation->mr_or_key.key;

}



/**

 * Get a Protection Domain for an RDMA device context.

 *

 * \param context RDMA device context

 * \return Pointer to the allocated Protection Domain

 */

struct ibv_pd *

spdk_rdma_get_pd(struct ibv_context *context);



/**

 * Return a Protection Domain.

 *

 * \param pd Pointer to the Protection Domain

 */

void spdk_rdma_put_pd(struct ibv_pd *pd);



#endif /* SPDK_RDMA_H */

include/spdk_internal/rdma_utils.h

0 → 100644
+124 −0
Original line number Diff line number Diff line 
/*   SPDX-License-Identifier: BSD-3-Clause

 *   Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.

 */



#ifndef SPDK_UTILS_RDMA_H

#define SPDK_UTILS_RDMA_H



#ifdef __cplusplus

extern "C" {

#endif



/* Contains hooks definition */

#include "spdk/nvme.h"



#include <infiniband/verbs.h>



union spdk_rdma_utils_mr {

	struct ibv_mr	*mr;

	uint64_t	key;

};



enum SPDK_RDMA_UTILS_TRANSLATION_TYPE {

	SPDK_RDMA_UTILS_TRANSLATION_MR = 0,

	SPDK_RDMA_UTILS_TRANSLATION_KEY

};



struct spdk_rdma_utils_memory_translation {

	union spdk_rdma_utils_mr mr_or_key;

	uint8_t translation_type;

};



enum spdk_rdma_utils_memory_map_role {

	SPDK_RDMA_UTILS_MEMORY_MAP_ROLE_TARGET,

	SPDK_RDMA_UTILS_MEMORY_MAP_ROLE_INITIATOR

};



struct spdk_rdma_utils_mem_map;



/**

 * Create a memory map which is used to register Memory Regions and perform address -> memory

 * key translations

 *

 * \param pd Protection Domain which will be used to create Memory Regions

 * \param hooks Optional hooks which are used to create Protection Domain or ger RKey

 * \param role Specifies whether this map is used by RDMA target or initiator, determines access flags of registered MRs

 * \return Pointer to memory map or NULL on failure

 */

struct spdk_rdma_utils_mem_map *

spdk_rdma_utils_create_mem_map(struct ibv_pd *pd, struct spdk_nvme_rdma_hooks *hooks,

			       enum spdk_rdma_utils_memory_map_role role);



/**

 * Free previously allocated memory map

 *

 * \param map Pointer to memory map to free

 */

void spdk_rdma_utils_free_mem_map(struct spdk_rdma_utils_mem_map **map);



/**

 * Get a translation for the given address and length.

 *

 * Note: the user of this function should use address returned in \b translation structure

 *

 * \param map Pointer to translation map

 * \param address Memory address for translation

 * \param length Length of the memory address

 * \param[in,out] translation Pointer to translation result to be filled by this function

 * \retval -EINVAL if translation is not found

 * \retval 0 translation succeed

 */

int spdk_rdma_utils_get_translation(struct spdk_rdma_utils_mem_map *map, void *address,

				    size_t length, struct spdk_rdma_utils_memory_translation *translation);



/**

 * Helper function for retrieving Local Memory Key. Should be applied to a translation

 * returned by \b spdk_rdma_utils_get_translation

 *

 * \param translation Memory translation

 * \return Local Memory Key

 */

static inline uint32_t

spdk_rdma_utils_memory_translation_get_lkey(

	struct spdk_rdma_utils_memory_translation *translation)

{

	return translation->translation_type == SPDK_RDMA_UTILS_TRANSLATION_MR ?

	       translation->mr_or_key.mr->lkey : (uint32_t)translation->mr_or_key.key;

}



/**

 * Helper function for retrieving Remote Memory Key. Should be applied to a translation

 * returned by \b spdk_rdma_utils_get_translation

 *

 * \param translation Memory translation

 * \return Remote Memory Key

 */

static inline uint32_t

spdk_rdma_utils_memory_translation_get_rkey(

	struct spdk_rdma_utils_memory_translation *translation)

{

	return translation->translation_type == SPDK_RDMA_UTILS_TRANSLATION_MR ?

	       translation->mr_or_key.mr->rkey : (uint32_t)translation->mr_or_key.key;

}



/**

 * Get a Protection Domain for an RDMA device context.

 *

 * \param context RDMA device context

 * \return Pointer to the allocated Protection Domain

 */

struct ibv_pd *

spdk_rdma_utils_get_pd(struct ibv_context *context);



/**

 * Return a Protection Domain.

 *

 * \param pd Pointer to the Protection Domain

 */

void spdk_rdma_utils_put_pd(struct ibv_pd *pd);



#ifdef __cplusplus

}

#endif



#endif /* SPDK_RDMA_UTILS_H */

lib/Makefile

+1 −0
Original line number Diff line number Diff line
@@ -30,6 +30,7 @@ DIRS-$(CONFIG_VHOST) += vhost 
DIRS-$(CONFIG_VIRTIO) += virtio

DIRS-$(CONFIG_VBDEV_COMPRESS) += reduce

DIRS-$(CONFIG_RDMA) += rdma_provider

DIRS-$(CONFIG_RDMA) += rdma_utils

DIRS-$(CONFIG_VFIO_USER) += vfu_tgt



ifeq ($(CONFIG_RDMA_PROV),mlx5_dv)

lib/mlx5/mlx5_crypto.c

+3 −3
Original line number Diff line number Diff line
@@ -12,7 +12,7 @@ 
#include "spdk/likely.h"

#include "spdk/util.h"

#include "spdk_internal/mlx5.h"

#include "spdk_internal/rdma_provider.h"

#include "spdk_internal/rdma_utils.h"



#define MLX5_VENDOR_ID_MELLANOX 0x2c9
@@ -144,7 +144,7 @@ spdk_mlx5_crypto_keytag_destroy(struct spdk_mlx5_crypto_keytag *keytag) 
			mlx5dv_dek_destroy(dek->dek_obj);

		}

		if (dek->pd) {

			spdk_rdma_put_pd(dek->pd);

			spdk_rdma_utils_put_pd(dek->pd);

		}

	}

	spdk_memset_s(keytag->keytag, sizeof(keytag->keytag), 0, sizeof(keytag->keytag));
@@ -223,7 +223,7 @@ spdk_mlx5_crypto_keytag_create(struct spdk_mlx5_crypto_dek_create_attr *attr, 
	for (i = 0; i < num_devs; i++) {

		keytag->deks_num++;

		dek = &keytag->deks[i];

		dek->pd = spdk_rdma_get_pd(devs[i]);

		dek->pd = spdk_rdma_utils_get_pd(devs[i]);

		if (!dek->pd) {

			SPDK_ERRLOG("Failed to get PD on device %s\n", devs[i]->device->name);

			rc = -EINVAL;

lib/nvme/nvme_rdma.c

+23 −22
Original line number Diff line number Diff line
@@ -24,6 +24,7 @@ 


#include "nvme_internal.h"

#include "spdk_internal/rdma_provider.h"

#include "spdk_internal/rdma_utils.h"



#define NVME_RDMA_TIME_OUT_IN_MS 2000

#define NVME_RDMA_RW_BUFFER_SIZE 131072
@@ -138,7 +139,7 @@ struct nvme_rdma_poller { 
	struct spdk_rdma_provider_srq	*srq;

	struct nvme_rdma_rsps		*rsps;

	struct ibv_pd			*pd;

	struct spdk_rdma_mem_map	*mr_map;

	struct spdk_rdma_utils_mem_map	*mr_map;

	uint32_t			refcnt;

	int				required_num_wc;

	int				current_num_wc;
@@ -175,7 +176,7 @@ struct nvme_rdma_rsp_opts { 
	uint16_t				num_entries;

	struct nvme_rdma_qpair			*rqpair;

	struct spdk_rdma_provider_srq		*srq;

	struct spdk_rdma_mem_map		*mr_map;

	struct spdk_rdma_utils_mem_map		*mr_map;

};



struct nvme_rdma_rsps {
@@ -221,7 +222,7 @@ struct nvme_rdma_qpair { 
	 */

	struct spdk_nvmf_cmd			*cmds;



	struct spdk_rdma_mem_map		*mr_map;

	struct spdk_rdma_utils_mem_map		*mr_map;



	TAILQ_HEAD(, spdk_nvme_rdma_req)	free_reqs;

	TAILQ_HEAD(, spdk_nvme_rdma_req)	outstanding_reqs;
@@ -784,7 +785,7 @@ nvme_rdma_qpair_init(struct nvme_rdma_qpair *rqpair) 
	if (g_nvme_hooks.get_ibv_pd) {

		attr.pd = g_nvme_hooks.get_ibv_pd(&rctrlr->ctrlr.trid, rqpair->cm_id->verbs);

	} else {

		attr.pd = spdk_rdma_get_pd(rqpair->cm_id->verbs);

		attr.pd = spdk_rdma_utils_get_pd(rqpair->cm_id->verbs);

	}



	attr.stats =		rqpair->poller ? &rqpair->poller->stats.rdma_stats : NULL;
@@ -913,7 +914,7 @@ static struct nvme_rdma_rsps * 
nvme_rdma_create_rsps(struct nvme_rdma_rsp_opts *opts)

{

	struct nvme_rdma_rsps *rsps;

	struct spdk_rdma_memory_translation translation;

	struct spdk_rdma_utils_memory_translation translation;

	uint16_t i;

	int rc;
@@ -954,11 +955,11 @@ nvme_rdma_create_rsps(struct nvme_rdma_rsp_opts *opts) 
		rsp->recv_wr = recv_wr;

		rsp_sgl->addr = (uint64_t)rsp;

		rsp_sgl->length = sizeof(struct spdk_nvme_cpl);

		rc = spdk_rdma_get_translation(opts->mr_map, rsp, sizeof(*rsp), &translation);

		rc = spdk_rdma_utils_get_translation(opts->mr_map, rsp, sizeof(*rsp), &translation);

		if (rc) {

			goto fail;

		}

		rsp_sgl->lkey = spdk_rdma_memory_translation_get_lkey(&translation);

		rsp_sgl->lkey = spdk_rdma_utils_memory_translation_get_lkey(&translation);



		recv_wr->wr_id = (uint64_t)&rsp->rdma_wr;

		recv_wr->next = NULL;
@@ -1000,7 +1001,7 @@ nvme_rdma_free_reqs(struct nvme_rdma_qpair *rqpair) 
static int

nvme_rdma_create_reqs(struct nvme_rdma_qpair *rqpair)

{

	struct spdk_rdma_memory_translation translation;

	struct spdk_rdma_utils_memory_translation translation;

	uint16_t i;

	int rc;
@@ -1032,11 +1033,11 @@ nvme_rdma_create_reqs(struct nvme_rdma_qpair *rqpair) 


		rdma_req->id = i;



		rc = spdk_rdma_get_translation(rqpair->mr_map, cmd, sizeof(*cmd), &translation);

		rc = spdk_rdma_utils_get_translation(rqpair->mr_map, cmd, sizeof(*cmd), &translation);

		if (rc) {

			goto fail;

		}

		rdma_req->send_sgl[0].lkey = spdk_rdma_memory_translation_get_lkey(&translation);

		rdma_req->send_sgl[0].lkey = spdk_rdma_utils_memory_translation_get_lkey(&translation);



		/* The first RDMA sgl element will always point

		 * at this data structure. Depending on whether
@@ -1169,8 +1170,8 @@ nvme_rdma_connect_established(struct nvme_rdma_qpair *rqpair, int ret) 
	}



	assert(!rqpair->mr_map);

	rqpair->mr_map = spdk_rdma_create_mem_map(rqpair->rdma_qp->qp->pd, &g_nvme_hooks,

			 SPDK_RDMA_MEMORY_MAP_ROLE_INITIATOR);

	rqpair->mr_map = spdk_rdma_utils_create_mem_map(rqpair->rdma_qp->qp->pd, &g_nvme_hooks,

			 SPDK_RDMA_UTILS_MEMORY_MAP_ROLE_INITIATOR);

	if (!rqpair->mr_map) {

		SPDK_ERRLOG("Unable to register RDMA memory translation map\n");

		return -1;
@@ -1428,7 +1429,7 @@ nvme_rdma_get_memory_translation(struct nvme_request *req, struct nvme_rdma_qpai 
{

	struct spdk_memory_domain_translation_ctx ctx;

	struct spdk_memory_domain_translation_result dma_translation = {.iov_count = 0};

	struct spdk_rdma_memory_translation rdma_translation;

	struct spdk_rdma_utils_memory_translation rdma_translation;

	int rc;



	assert(req);
@@ -1454,12 +1455,12 @@ nvme_rdma_get_memory_translation(struct nvme_request *req, struct nvme_rdma_qpai 
		_ctx->addr = dma_translation.iov.iov_base;

		_ctx->length = dma_translation.iov.iov_len;

	} else {

		rc = spdk_rdma_get_translation(rqpair->mr_map, _ctx->addr, _ctx->length, &rdma_translation);

		rc = spdk_rdma_utils_get_translation(rqpair->mr_map, _ctx->addr, _ctx->length, &rdma_translation);

		if (spdk_unlikely(rc)) {

			SPDK_ERRLOG("RDMA memory translation failed, rc %d\n", rc);

			return rc;

		}

		if (rdma_translation.translation_type == SPDK_RDMA_TRANSLATION_MR) {

		if (rdma_translation.translation_type == SPDK_RDMA_UTILS_TRANSLATION_MR) {

			_ctx->lkey = rdma_translation.mr_or_key.mr->lkey;

			_ctx->rkey = rdma_translation.mr_or_key.mr->rkey;

		} else {
@@ -1859,7 +1860,7 @@ nvme_rdma_qpair_destroy(struct nvme_rdma_qpair *rqpair) 
	struct nvme_rdma_ctrlr *rctrlr;

	struct nvme_rdma_cm_event_entry *entry, *tmp;



	spdk_rdma_free_mem_map(&rqpair->mr_map);

	spdk_rdma_utils_free_mem_map(&rqpair->mr_map);



	if (rqpair->evt) {

		rdma_ack_cm_event(rqpair->evt);
@@ -1883,7 +1884,7 @@ nvme_rdma_qpair_destroy(struct nvme_rdma_qpair *rqpair) 


	if (rqpair->cm_id) {

		if (rqpair->rdma_qp) {

			spdk_rdma_put_pd(rqpair->rdma_qp->qp->pd);

			spdk_rdma_utils_put_pd(rqpair->rdma_qp->qp->pd);

			spdk_rdma_provider_qp_destroy(rqpair->rdma_qp);

			rqpair->rdma_qp = NULL;

		}
@@ -2895,10 +2896,10 @@ nvme_rdma_poller_destroy(struct nvme_rdma_poller *poller) 
		spdk_rdma_provider_srq_destroy(poller->srq);

	}

	if (poller->mr_map) {

		spdk_rdma_free_mem_map(&poller->mr_map);

		spdk_rdma_utils_free_mem_map(&poller->mr_map);

	}

	if (poller->pd) {

		spdk_rdma_put_pd(poller->pd);

		spdk_rdma_utils_put_pd(poller->pd);

	}

	free(poller);

}
@@ -2929,14 +2930,14 @@ nvme_rdma_poller_create(struct nvme_rdma_poll_group *group, struct ibv_context * 
			goto fail;

		}



		poller->pd = spdk_rdma_get_pd(ctx);

		poller->pd = spdk_rdma_utils_get_pd(ctx);

		if (poller->pd == NULL) {

			SPDK_ERRLOG("Unable to get PD.\n");

			goto fail;

		}



		poller->mr_map = spdk_rdma_create_mem_map(poller->pd, &g_nvme_hooks,

				 SPDK_RDMA_MEMORY_MAP_ROLE_INITIATOR);

		poller->mr_map = spdk_rdma_utils_create_mem_map(poller->pd, &g_nvme_hooks,

				 SPDK_RDMA_UTILS_MEMORY_MAP_ROLE_INITIATOR);

		if (poller->mr_map == NULL) {

			SPDK_ERRLOG("Unable to create memory map.\n");

			goto fail;