Commit ea7a4f3c authored by Alexey Marchuk's avatar Alexey Marchuk Committed by Tomasz Zawadzki
Browse files

nvmf/rdma: Use RDMA provider API to create/destroy qpair 



This patch updates NVMF target to use RDMA provider API
to create and destroy qpairs.
Makefiles have been updated with new dependency on RDMA lib

Change-Id: Iae35aea601380f8d1a6453a7fd6115f781e126f5
Signed-off-by: default avatarAlexey Marchuk <alexeymar@mellanox.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/1656


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Community-CI: Broadcom CI
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: default avatarSeth Howell <seth.howell@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
parent 7e14f724

lib/nvmf/Makefile

+0 −18
Original line number Diff line number Diff line
@@ -44,24 +44,6 @@ C_SRCS = ctrlr.c ctrlr_discovery.c ctrlr_bdev.c \ 
C_SRCS-$(CONFIG_RDMA) += rdma.c

LIBNAME = nvmf

LOCAL_SYS_LIBS = -luuid

ifeq ($(CONFIG_RDMA),y)

LOCAL_SYS_LIBS += -libverbs -lrdmacm

#Attach only if FreeBSD and RDMA is specified with configure

ifeq ($(OS),FreeBSD)

# Mellanox - MLX4 HBA Userspace Library

ifneq ("$(wildcard /usr/lib/libmlx4.*)","")

LOCAL_SYS_LIBS += -lmlx4

endif

# Mellanox - MLX5 HBA Userspace Library

ifneq ("$(wildcard /usr/lib/libmlx5.*)","")

LOCAL_SYS_LIBS += -lmlx5

endif

# Chelsio HBA Userspace Library

ifneq ("$(wildcard /usr/lib/libcxgb4.*)","")

LOCAL_SYS_LIBS += -lcxgb4

endif

endif

endif



ifeq ($(CONFIG_FC),y)

C_SRCS += fc.c fc_ls.c

lib/nvmf/rdma.c

+29 −25
Original line number Diff line number Diff line
@@ -33,10 +33,6 @@ 


#include "spdk/stdinc.h"



#include <infiniband/verbs.h>

#include <rdma/rdma_cma.h>

#include <rdma/rdma_verbs.h>



#include "spdk/config.h"

#include "spdk/thread.h"

#include "spdk/likely.h"
@@ -47,6 +43,7 @@ 


#include "spdk_internal/assert.h"

#include "spdk_internal/log.h"

#include "spdk_internal/rdma.h"



struct spdk_nvme_rdma_hooks g_nvmf_hooks = {};

const struct spdk_nvmf_transport_ops spdk_nvmf_transport_rdma;
@@ -343,6 +340,7 @@ struct spdk_nvmf_rdma_qpair { 
	struct spdk_nvmf_rdma_device		*device;

	struct spdk_nvmf_rdma_poller		*poller;



	struct spdk_rdma_qp			*rdma_qp;

	struct rdma_cm_id			*cm_id;

	struct ibv_srq				*srq;

	struct rdma_cm_id			*listen_id;
@@ -896,8 +894,9 @@ nvmf_rdma_qpair_destroy(struct spdk_nvmf_rdma_qpair *rqpair) 
	}



	if (rqpair->cm_id) {

		if (rqpair->cm_id->qp != NULL) {

			rdma_destroy_qp(rqpair->cm_id);

		if (rqpair->rdma_qp != NULL) {

			spdk_rdma_qp_destroy(rqpair->rdma_qp);

			rqpair->rdma_qp = NULL;

		}

		rdma_destroy_id(rqpair->cm_id);
@@ -955,48 +954,45 @@ static int 
nvmf_rdma_qpair_initialize(struct spdk_nvmf_qpair *qpair)

{

	struct spdk_nvmf_rdma_qpair		*rqpair;

	int					rc;

	struct spdk_nvmf_rdma_transport		*rtransport;

	struct spdk_nvmf_transport		*transport;

	struct spdk_nvmf_rdma_resource_opts	opts;

	struct spdk_nvmf_rdma_device		*device;

	struct ibv_qp_init_attr			ibv_init_attr;

	struct spdk_rdma_qp_init_attr		qp_init_attr = {};



	rqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_rdma_qpair, qpair);

	device = rqpair->device;



	memset(&ibv_init_attr, 0, sizeof(struct ibv_qp_init_attr));

	ibv_init_attr.qp_context	= rqpair;

	ibv_init_attr.qp_type		= IBV_QPT_RC;

	ibv_init_attr.send_cq		= rqpair->poller->cq;

	ibv_init_attr.recv_cq		= rqpair->poller->cq;

	qp_init_attr.qp_context	= rqpair;

	qp_init_attr.pd		= device->pd;

	qp_init_attr.send_cq	= rqpair->poller->cq;

	qp_init_attr.recv_cq	= rqpair->poller->cq;



	if (rqpair->srq) {

		ibv_init_attr.srq		= rqpair->srq;

		qp_init_attr.srq		= rqpair->srq;

	} else {

		ibv_init_attr.cap.max_recv_wr	= rqpair->max_queue_depth;

		qp_init_attr.cap.max_recv_wr	= rqpair->max_queue_depth;

	}



	ibv_init_attr.cap.max_send_wr	= (uint32_t)rqpair->max_queue_depth *

					  2; /* SEND, READ or WRITE operations */

	ibv_init_attr.cap.max_send_sge	= spdk_min((uint32_t)device->attr.max_sge, NVMF_DEFAULT_TX_SGE);

	ibv_init_attr.cap.max_recv_sge	= spdk_min((uint32_t)device->attr.max_sge, NVMF_DEFAULT_RX_SGE);

	/* SEND, READ, and WRITE operations */

	qp_init_attr.cap.max_send_wr	= (uint32_t)rqpair->max_queue_depth * 2;

	qp_init_attr.cap.max_send_sge	= spdk_min((uint32_t)device->attr.max_sge, NVMF_DEFAULT_TX_SGE);

	qp_init_attr.cap.max_recv_sge	= spdk_min((uint32_t)device->attr.max_sge, NVMF_DEFAULT_RX_SGE);



	if (rqpair->srq == NULL && nvmf_rdma_resize_cq(rqpair, device) < 0) {

		SPDK_ERRLOG("Failed to resize the completion queue. Cannot initialize qpair.\n");

		goto error;

	}



	rc = rdma_create_qp(rqpair->cm_id, device->pd, &ibv_init_attr);

	if (rc) {

		SPDK_ERRLOG("rdma_create_qp failed: errno %d: %s\n", errno, spdk_strerror(errno));

	rqpair->rdma_qp = spdk_rdma_qp_create(rqpair->cm_id, &qp_init_attr);

	if (!rqpair->rdma_qp) {

		goto error;

	}



	rqpair->max_send_depth = spdk_min((uint32_t)(rqpair->max_queue_depth * 2),

					  ibv_init_attr.cap.max_send_wr);

	rqpair->max_send_sge = spdk_min(NVMF_DEFAULT_TX_SGE, ibv_init_attr.cap.max_send_sge);

	rqpair->max_recv_sge = spdk_min(NVMF_DEFAULT_RX_SGE, ibv_init_attr.cap.max_recv_sge);

					  qp_init_attr.cap.max_send_wr);

	rqpair->max_send_sge = spdk_min(NVMF_DEFAULT_TX_SGE, qp_init_attr.cap.max_send_sge);

	rqpair->max_recv_sge = spdk_min(NVMF_DEFAULT_RX_SGE, qp_init_attr.cap.max_recv_sge);

	spdk_trace_record(TRACE_RDMA_QP_CREATE, 0, 0, (uintptr_t)rqpair->cm_id, 0);

	SPDK_DEBUGLOG(SPDK_LOG_RDMA, "New RDMA Connection: %p\n", qpair);
@@ -1181,6 +1177,14 @@ nvmf_rdma_event_accept(struct rdma_cm_id *id, struct spdk_nvmf_rdma_qpair *rqpai 
		ctrlr_event_data.rnr_retry_count = 0x7;

	}



	/* When qpair is created without use of rdma cm API, an additional

	 * information must be provided to initiator in the connection response:

	 * whether qpair is using SRQ and its qp_num

	 * Fields below are ignored by rdma cm if qpair has been

	 * created using rdma cm API. */

	ctrlr_event_data.srq = rqpair->srq ? 1 : 0;

	ctrlr_event_data.qp_num = rqpair->rdma_qp->qp->qp_num;



	rc = rdma_accept(id, &ctrlr_event_data);

	if (rc) {

		SPDK_ERRLOG("Error %d on rdma_accept\n", errno);

mk/spdk.lib_deps.mk

+3 −0
Original line number Diff line number Diff line
@@ -77,6 +77,9 @@ DEPDIRS-event := log util conf thread $(JSON_LIBS) trace 
DEPDIRS-ftl := log util thread trace bdev

DEPDIRS-nbd := log util thread $(JSON_LIBS) bdev

DEPDIRS-nvmf := log sock util nvme thread $(JSON_LIBS) trace bdev

ifeq ($(CONFIG_RDMA),y)

DEPDIRS-nvmf += rdma

endif

DEPDIRS-scsi := log util thread $(JSON_LIBS) trace bdev



DEPDIRS-iscsi := log sock util conf thread $(JSON_LIBS) trace event scsi

mk/spdk.modules.mk

+1 −0
Original line number Diff line number Diff line
@@ -52,6 +52,7 @@ endif 


ifeq ($(CONFIG_RDMA),y)

SYS_LIBS += -libverbs -lrdmacm

BLOCKDEV_MODULES_LIST += rdma

endif



ifeq ($(OS),Linux)

test/common/lib/test_rdma.c

0 → 100644
+41 −0
Original line number Diff line number Diff line 
/*-

 *   BSD LICENSE

 *

 *   Copyright (c) Intel Corporation. All rights reserved.

 *   Copyright (c) 2020 Mellanox Technologies LTD. All rights reserved.

 *

 *   Redistribution and use in source and binary forms, with or without

 *   modification, are permitted provided that the following conditions

 *   are met:

 *

 *     * Redistributions of source code must retain the above copyright

 *       notice, this list of conditions and the following disclaimer.

 *     * Redistributions in binary form must reproduce the above copyright

 *       notice, this list of conditions and the following disclaimer in

 *       the documentation and/or other materials provided with the

 *       distribution.

 *     * Neither the name of Intel Corporation nor the names of its

 *       contributors may be used to endorse or promote products derived

 *       from this software without specific prior written permission.

 *

 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS

 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT

 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR

 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT

 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,

 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY

 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 */



#include "spdk/stdinc.h"



#include "spdk_internal/rdma.h"

#include "spdk_internal/mock.h"



DEFINE_STUB(spdk_rdma_qp_create, struct spdk_rdma_qp *, (struct rdma_cm_id *cm_id,

		struct spdk_rdma_qp_init_attr *qp_attr), NULL);

DEFINE_STUB_V(spdk_rdma_qp_destroy, (struct spdk_rdma_qp *spdk_rdma_qp));