Commit 7e14f724 authored by Alexey Marchuk's avatar Alexey Marchuk Committed by Tomasz Zawadzki
Browse files

rdma: Intorduce RDMA providers 



RDMA provider is used as an abstraction layer and allows
to add vendor specific verbs implementation. This patch adds
an interface to create and destroy qpairs and its implementation
using regular verbs API.
Future patches will add functions to queue/flush Work Requests,
enable usage of RDMA providers API in NVMEoF target/initiator and
add Mellanox Direct Verbs implementation. This abstraction layer
is flexible enough to add other vendor's implementations.
The choice of a specific RDMA provider will be made at
the configuration step, it doesn't add any runtime overhead.

Change-Id: If8eec5bd45c82c71f73a14f6e84121cd02a1d65d
Signed-off-by: default avatarAlexey Marchuk <alexeymar@mellanox.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/1655


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Community-CI: Broadcom CI
Reviewed-by: default avatarSeth Howell <seth.howell@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: default avatarJacek Kalwas <jacek.kalwas@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
parent 3ceeab7c

include/spdk_internal/rdma.h

0 → 100644
+70 −0
Original line number Diff line number Diff line 
/*-

 *   BSD LICENSE

 *

 *   Copyright (c) Intel Corporation. All rights reserved.

 *   Copyright (c) Mellanox Technologies LTD. All rights reserved.

 *

 *   Redistribution and use in source and binary forms, with or without

 *   modification, are permitted provided that the following conditions

 *   are met:

 *

 *     * Redistributions of source code must retain the above copyright

 *       notice, this list of conditions and the following disclaimer.

 *     * Redistributions in binary form must reproduce the above copyright

 *       notice, this list of conditions and the following disclaimer in

 *       the documentation and/or other materials provided with the

 *       distribution.

 *     * Neither the name of Intel Corporation nor the names of its

 *       contributors may be used to endorse or promote products derived

 *       from this software without specific prior written permission.

 *

 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS

 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT

 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR

 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT

 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,

 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY

 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 */



#ifndef SPDK_RDMA_H

#define SPDK_RDMA_H



#include <infiniband/verbs.h>

#include <rdma/rdma_cma.h>

#include <rdma/rdma_verbs.h>



struct spdk_rdma_qp_init_attr {

	void		       *qp_context;

	struct ibv_cq	       *send_cq;

	struct ibv_cq	       *recv_cq;

	struct ibv_srq	       *srq;

	struct ibv_qp_cap	cap;

	struct ibv_pd	       *pd;

};



struct spdk_rdma_qp {

	struct ibv_qp *qp;

	struct rdma_cm_id *cm_id;

};



/**

 * Create RDMA provider specific qpair

 * \param cm_id Pointer to RDMACM cm_id

 * \param qp_attr Pointer to qpair init attributes

 * \return Pointer to a newly created qpair on success or NULL on failure

 */

struct spdk_rdma_qp *spdk_rdma_qp_create(struct rdma_cm_id *cm_id,

		struct spdk_rdma_qp_init_attr *qp_attr);



/**

 * Destory RDMA provider specific qpair

 * \param spdk_rdma_qp Pointer to qpair to be destroyed

 */

void spdk_rdma_qp_destroy(struct spdk_rdma_qp *spdk_rdma_qp);



#endif /* SPDK_RDMA_H */

lib/Makefile

+1 −0
Original line number Diff line number Diff line
@@ -48,6 +48,7 @@ DIRS-$(CONFIG_VHOST) += vhost 
DIRS-$(CONFIG_VIRTIO) += virtio

DIRS-$(CONFIG_REDUCE) += reduce

DIRS-$(CONFIG_VHOST_INTERNAL_LIB) += rte_vhost

DIRS-$(CONFIG_RDMA) += rdma



# If CONFIG_ENV is pointing at a directory in lib, build it.

# Out-of-tree env implementations must be built separately by the user.

lib/rdma/Makefile

0 → 100644
+63 −0
Original line number Diff line number Diff line 
#

#  BSD LICENSE

#

#  Copyright (c) Intel Corporation. All rights reserved.

#  Copyright (c) Mellanox Technologies LTD. All rights reserved.

#

#  Redistribution and use in source and binary forms, with or without

#  modification, are permitted provided that the following conditions

#  are met:

#

#    * Redistributions of source code must retain the above copyright

#      notice, this list of conditions and the following disclaimer.

#    * Redistributions in binary form must reproduce the above copyright

#      notice, this list of conditions and the following disclaimer in

#      the documentation and/or other materials provided with the

#      distribution.

#    * Neither the name of Intel Corporation nor the names of its

#      contributors may be used to endorse or promote products derived

#      from this software without specific prior written permission.

#

#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS

#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT

#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR

#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT

#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,

#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY

#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#



SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)

include $(SPDK_ROOT_DIR)/mk/spdk.common.mk



SO_VER := 1

SO_MINOR := 0

SO_SUFFIX := $(SO_VER).$(SO_MINOR)



SPDK_MAP_FILE = $(abspath $(CURDIR)/spdk_rdma.map)



LIBNAME = rdma

C_SRCS = rdma_verbs.c



LOCAL_SYS_LIBS += -libverbs -lrdmacm

#Attach only if FreeBSD and RDMA is specified with configure

ifeq ($(OS),FreeBSD)

# Mellanox - MLX4 HBA Userspace Library

ifneq ("$(wildcard /usr/lib/libmlx4.*)","")

LOCAL_SYS_LIBS += -lmlx4

endif

# Mellanox - MLX5 HBA Userspace Library

ifneq ("$(wildcard /usr/lib/libmlx5.*)","")

LOCAL_SYS_LIBS += -lmlx5

endif

# Chelsio HBA Userspace Library

ifneq ("$(wildcard /usr/lib/libcxgb4.*)","")

LOCAL_SYS_LIBS += -lcxgb4

endif

endif



include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk

lib/rdma/rdma_verbs.c

0 → 100644
+87 −0
Original line number Diff line number Diff line 
/*-

 *   BSD LICENSE

 *

 *   Copyright (c) Intel Corporation. All rights reserved.

 *   Copyright (c) Mellanox Technologies LTD. All rights reserved.

 *

 *   Redistribution and use in source and binary forms, with or without

 *   modification, are permitted provided that the following conditions

 *   are met:

 *

 *     * Redistributions of source code must retain the above copyright

 *       notice, this list of conditions and the following disclaimer.

 *     * Redistributions in binary form must reproduce the above copyright

 *       notice, this list of conditions and the following disclaimer in

 *       the documentation and/or other materials provided with the

 *       distribution.

 *     * Neither the name of Intel Corporation nor the names of its

 *       contributors may be used to endorse or promote products derived

 *       from this software without specific prior written permission.

 *

 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS

 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT

 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR

 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT

 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,

 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY

 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 */



#include <rdma/rdma_cma.h>



#include "spdk/stdinc.h"

#include "spdk/string.h"

#include "spdk/likely.h"



#include "spdk_internal/rdma.h"

#include "spdk_internal/log.h"



struct spdk_rdma_qp *

spdk_rdma_qp_create(struct rdma_cm_id *cm_id, struct spdk_rdma_qp_init_attr *qp_attr)

{

	struct spdk_rdma_qp *spdk_rdma_qp;

	int rc;

	struct ibv_qp_init_attr attr = {

		.qp_context = qp_attr->qp_context,

		.send_cq = qp_attr->send_cq,

		.recv_cq = qp_attr->recv_cq,

		.srq = qp_attr->srq,

		.cap = qp_attr->cap,

		.qp_type = IBV_QPT_RC

	};



	spdk_rdma_qp = calloc(1, sizeof(*spdk_rdma_qp));

	if (!spdk_rdma_qp) {

		SPDK_ERRLOG("qp memory allocation failed\n");

		return NULL;

	}



	rc = rdma_create_qp(cm_id, qp_attr->pd, &attr);

	if (rc) {

		SPDK_ERRLOG("Failed to create qp, errno %s (%d)\n", spdk_strerror(errno), errno);

		free(spdk_rdma_qp);

		return NULL;

	}



	qp_attr->cap = attr.cap;

	spdk_rdma_qp->qp = cm_id->qp;

	spdk_rdma_qp->cm_id = cm_id;



	return spdk_rdma_qp;

}



void

spdk_rdma_qp_destroy(struct spdk_rdma_qp *spdk_rdma_qp)

{

	assert(spdk_rdma_qp != NULL);



	if (spdk_rdma_qp->qp) {

		rdma_destroy_qp(spdk_rdma_qp->cm_id);

	}



	free(spdk_rdma_qp);

}

lib/rdma/spdk_rdma.map

0 → 100644
+9 −0
Original line number Diff line number Diff line 
{

	global:



	# Public functions

	spdk_rdma_qp_create;

	spdk_rdma_qp_destroy;



	local: *;

};