Commit b4a9d7d3 authored by Alexey Marchuk's avatar Alexey Marchuk Committed by Tomasz Zawadzki
Browse files

nvme/rdma: Use RDMA provider API to create/destroy qpair 



This patch adds use of RDMA provider API to NVMEoF initiator.
Makefiles have been updated with new RDMA lib dependency

Change-Id: Ieaefeb12ee9681d3db2b618c5cf0c54dc52230af
Signed-off-by: default avatarAlexey Marchuk <alexeymar@mellanox.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/1657


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Community-CI: Broadcom CI
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
parent ea7a4f3c

app/iscsi_tgt/Makefile

+0 −1
Original line number Diff line number Diff line
@@ -52,7 +52,6 @@ ifeq ($(SPDK_ROOT_DIR)/lib/env_dpdk,$(CONFIG_ENV)) 
SPDK_LIB_LIST += env_dpdk_rpc

endif





ifeq ($(OS),Linux)

SPDK_LIB_LIST += event_nbd nbd

endif

lib/nvme/Makefile

+0 −18
Original line number Diff line number Diff line
@@ -45,24 +45,6 @@ C_SRCS-$(CONFIG_NVME_CUSE) += nvme_cuse.c 


LIBNAME = nvme

LOCAL_SYS_LIBS = -luuid

ifeq ($(CONFIG_RDMA),y)

LOCAL_SYS_LIBS += -libverbs -lrdmacm

#Attach only if FreeBSD and RDMA is specified with configure

ifeq ($(OS),FreeBSD)

# Mellanox - MLX4 HBA Userspace Library

ifneq ("$(wildcard /usr/lib/libmlx4.*)","")

LOCAL_SYS_LIBS += -lmlx4

endif

# Mellanox - MLX5 HBA Userspace Library

ifneq ("$(wildcard /usr/lib/libmlx5.*)","")

LOCAL_SYS_LIBS += -lmlx5

endif

# Chelsio HBA Userspace Library

ifneq ("$(wildcard /usr/lib/libcxgb4.*)","")

LOCAL_SYS_LIBS += -lcxgb4

endif

endif

endif



ifeq ($(CONFIG_NVME_CUSE),y)

# fuse requires to set _FILE_OFFSET_BITS to 64 bits even for 64 bit machines

lib/nvme/nvme_rdma.c

+9 −13
Original line number Diff line number Diff line
@@ -37,10 +37,6 @@ 


#include "spdk/stdinc.h"



#include <infiniband/verbs.h>

#include <rdma/rdma_cma.h>

#include <rdma/rdma_verbs.h>



#include "spdk/assert.h"

#include "spdk/log.h"

#include "spdk/trace.h"
@@ -54,6 +50,7 @@ 
#include "spdk/config.h"



#include "nvme_internal.h"

#include "spdk_internal/rdma.h"



#define NVME_RDMA_TIME_OUT_IN_MS 2000

#define NVME_RDMA_RW_BUFFER_SIZE 131072
@@ -146,8 +143,8 @@ struct spdk_nvme_recv_wr_list { 
struct nvme_rdma_qpair {

	struct spdk_nvme_qpair			qpair;



	struct spdk_rdma_qp			*rdma_qp;

	struct rdma_cm_id			*cm_id;



	struct ibv_cq				*cq;



	struct	spdk_nvme_rdma_req		*rdma_reqs;
@@ -503,7 +500,7 @@ static int 
nvme_rdma_qpair_init(struct nvme_rdma_qpair *rqpair)

{

	int			rc;

	struct ibv_qp_init_attr	attr;

	struct spdk_rdma_qp_init_attr	attr = {};

	struct ibv_device_attr	dev_attr;

	struct nvme_rdma_ctrlr	*rctrlr;
@@ -526,8 +523,7 @@ nvme_rdma_qpair_init(struct nvme_rdma_qpair *rqpair) 
		rctrlr->pd = NULL;

	}



	memset(&attr, 0, sizeof(struct ibv_qp_init_attr));

	attr.qp_type		= IBV_QPT_RC;

	attr.pd =		rctrlr->pd;

	attr.send_cq		= rqpair->cq;

	attr.recv_cq		= rqpair->cq;

	attr.cap.max_send_wr	= rqpair->num_entries; /* SEND operations */
@@ -535,10 +531,9 @@ nvme_rdma_qpair_init(struct nvme_rdma_qpair *rqpair) 
	attr.cap.max_send_sge	= spdk_min(NVME_RDMA_DEFAULT_TX_SGE, dev_attr.max_sge);

	attr.cap.max_recv_sge	= spdk_min(NVME_RDMA_DEFAULT_RX_SGE, dev_attr.max_sge);



	rc = rdma_create_qp(rqpair->cm_id, rctrlr->pd, &attr);

	rqpair->rdma_qp = spdk_rdma_qp_create(rqpair->cm_id, &attr);



	if (rc) {

		SPDK_ERRLOG("rdma_create_qp failed\n");

	if (!rqpair->rdma_qp) {

		return -1;

	}
@@ -1684,8 +1679,9 @@ nvme_rdma_ctrlr_disconnect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme 
			}

		}



		if (rqpair->cm_id->qp) {

			rdma_destroy_qp(rqpair->cm_id);

		if (rqpair->rdma_qp) {

			spdk_rdma_qp_destroy(rqpair->rdma_qp);

			rqpair->rdma_qp = NULL;

		}

		rdma_destroy_id(rqpair->cm_id);

		rqpair->cm_id = NULL;

mk/nvme.libtest.mk

+1 −1
Original line number Diff line number Diff line
@@ -43,7 +43,7 @@ SPDK_LIB_LIST = $(filter-out sock_vpp,$(SOCK_MODULES_LIST)) 
SPDK_LIB_LIST += nvme thread util log sock vmd



ifeq ($(CONFIG_RDMA),y)

SYS_LIBS += -libverbs -lrdmacm

SPDK_LIB_LIST += rdma

endif



include $(SPDK_ROOT_DIR)/mk/spdk.app.mk

mk/spdk.lib_deps.mk

+6 −2
Original line number Diff line number Diff line
@@ -52,10 +52,14 @@ DEPDIRS-vmd := log 


DEPDIRS-conf := log util

DEPDIRS-json := log util

DEPDIRS-nvme := log sock util

DEPDIRS-rdma := log util

DEPDIRS-reduce := log util

DEPDIRS-thread := log util

DEPDIRS-rdma := log util



DEPDIRS-nvme := log sock util

ifeq ($(CONFIG_RDMA),y)

DEPDIRS-nvme += rdma

endif



DEPDIRS-blob := log util thread

DEPDIRS-accel := log util thread