Commit 268aacb2 authored by Alexey Marchuk's avatar Alexey Marchuk Committed by Tomasz Zawadzki
Browse files

rdma: Add new API spdk_rdma_qp_accept 



This API is a wrapper for rdma_accept which allows
to remove spdk_rdma_qp_init_attr::initiator_side.

Change-Id: Iba2be5e74e537c498fb11c939c922b2bbda95309
Signed-off-by: default avatarAlexey Marchuk <alexeymar@mellanox.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/2908


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Broadcom CI
Community-CI: Mellanox Build Bot
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
parent 5f6ef934

include/spdk_internal/rdma.h

+10 −3
Original line number Diff line number Diff line
@@ -45,7 +45,6 @@ struct spdk_rdma_qp_init_attr { 
	struct ibv_srq	       *srq;

	struct ibv_qp_cap	cap;

	struct ibv_pd	       *pd;

	bool			initiator_side;

};



struct spdk_rdma_send_wr_list {
@@ -68,11 +67,19 @@ struct spdk_rdma_qp { 
struct spdk_rdma_qp *spdk_rdma_qp_create(struct rdma_cm_id *cm_id,

		struct spdk_rdma_qp_init_attr *qp_attr);



/**

 * Accept a connection request. Called by the passive side (NVMEoF target)

 * \param spdk_rdma_qp Pointer to a qpair

 * \param conn_param Optional information needed to establish the connection

 * \return 0 on success, errno on failure

 */

int spdk_rdma_qp_accept(struct spdk_rdma_qp *spdk_rdma_qp, struct rdma_conn_param *conn_param);



/**

 * Complete the connection process, must be called by the active

 * side (NVMEoF initiator) upon receipt RDMA_CM_EVENT_CONNECT_RESPONSE

 * @param spdk_rdma_qp pointer to a qpair

 * @return 0 on success, errno on failure

 * \param spdk_rdma_qp Pointer to a qpair

 * \return 0 on success, errno on failure

 */

int spdk_rdma_qp_complete_connect(struct spdk_rdma_qp *spdk_rdma_qp);

lib/nvme/nvme_rdma.c

+0 −1
Original line number Diff line number Diff line
@@ -619,7 +619,6 @@ nvme_rdma_qpair_init(struct nvme_rdma_qpair *rqpair) 
	attr.cap.max_recv_wr	= rqpair->num_entries; /* RECV operations */

	attr.cap.max_send_sge	= spdk_min(NVME_RDMA_DEFAULT_TX_SGE, dev_attr.max_sge);

	attr.cap.max_recv_sge	= spdk_min(NVME_RDMA_DEFAULT_RX_SGE, dev_attr.max_sge);

	attr.initiator_side	= true;



	rqpair->rdma_qp = spdk_rdma_qp_create(rqpair->cm_id, &attr);

lib/nvmf/rdma.c

+2 −3
Original line number Diff line number Diff line
@@ -975,7 +975,6 @@ nvmf_rdma_qpair_initialize(struct spdk_nvmf_qpair *qpair) 
	qp_init_attr.cap.max_send_wr	= (uint32_t)rqpair->max_queue_depth * 2;

	qp_init_attr.cap.max_send_sge	= spdk_min((uint32_t)device->attr.max_sge, NVMF_DEFAULT_TX_SGE);

	qp_init_attr.cap.max_recv_sge	= spdk_min((uint32_t)device->attr.max_sge, NVMF_DEFAULT_RX_SGE);

	qp_init_attr.initiator_side	= false;



	if (rqpair->srq == NULL && nvmf_rdma_resize_cq(rqpair, device) < 0) {

		SPDK_ERRLOG("Failed to resize the completion queue. Cannot initialize qpair.\n");
@@ -1168,9 +1167,9 @@ nvmf_rdma_event_accept(struct rdma_cm_id *id, struct spdk_nvmf_rdma_qpair *rqpai 
	ctrlr_event_data.srq = rqpair->srq ? 1 : 0;

	ctrlr_event_data.qp_num = rqpair->rdma_qp->qp->qp_num;



	rc = rdma_accept(id, &ctrlr_event_data);

	rc = spdk_rdma_qp_accept(rqpair->rdma_qp, &ctrlr_event_data);

	if (rc) {

		SPDK_ERRLOG("Error %d on rdma_accept\n", errno);

		SPDK_ERRLOG("Error %d on spdk_rdma_qp_accept\n", errno);

	} else {

		SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Sent back the accept\n");

	}

lib/rdma/rdma_mlx5_dv.c

+20 −11
Original line number Diff line number Diff line
@@ -44,7 +44,6 @@ 
struct spdk_rdma_mlx5_dv_qp {

	struct spdk_rdma_qp common;

	struct ibv_qp_ex *qpex;

	bool initiator_side;

};



static int
@@ -132,23 +131,36 @@ spdk_rdma_qp_create(struct rdma_cm_id *cm_id, struct spdk_rdma_qp_init_attr *qp_ 
	mlx5_qp->common.qp = qp;

	mlx5_qp->common.cm_id = cm_id;

	mlx5_qp->qpex = ibv_qp_to_qp_ex(qp);

	mlx5_qp->initiator_side = qp_attr->initiator_side;



	if (!mlx5_qp->qpex) {

		spdk_rdma_qp_destroy(&mlx5_qp->common);

		return NULL;

	}



	qp_attr->cap = dv_qp_attr.cap;



	return &mlx5_qp->common;

}



int

spdk_rdma_qp_accept(struct spdk_rdma_qp *spdk_rdma_qp, struct rdma_conn_param *conn_param)

{

	struct spdk_rdma_mlx5_dv_qp *mlx5_qp;



	assert(spdk_rdma_qp != NULL);

	assert(spdk_rdma_qp->cm_id != NULL);



	mlx5_qp = SPDK_CONTAINEROF(spdk_rdma_qp, struct spdk_rdma_mlx5_dv_qp, common);



	/* NVMEoF target must move qpair to RTS state */

	if (!mlx5_qp->initiator_side && rdma_mlx5_dv_init_qpair(mlx5_qp) != 0) {

	if (rdma_mlx5_dv_init_qpair(mlx5_qp) != 0) {

		SPDK_ERRLOG("Failed to initialize qpair\n");

		spdk_rdma_qp_destroy(&mlx5_qp->common);

		return NULL;

		/* Set errno to be compliant with rdma_accept behaviour */

		errno = ECONNABORTED;

		return -1;

	}



	qp_attr->cap = dv_qp_attr.cap;



	return &mlx5_qp->common;

	return rdma_accept(spdk_rdma_qp->cm_id, conn_param);

}



int
@@ -160,9 +172,6 @@ spdk_rdma_qp_complete_connect(struct spdk_rdma_qp *spdk_rdma_qp) 
	assert(spdk_rdma_qp);



	mlx5_qp = SPDK_CONTAINEROF(spdk_rdma_qp, struct spdk_rdma_mlx5_dv_qp, common);

	if (!mlx5_qp->initiator_side) {

		return 0;

	}



	rc = rdma_mlx5_dv_init_qpair(mlx5_qp);

	if (rc) {

lib/rdma/rdma_verbs.c

+9 −0
Original line number Diff line number Diff line
@@ -74,6 +74,15 @@ spdk_rdma_qp_create(struct rdma_cm_id *cm_id, struct spdk_rdma_qp_init_attr *qp_ 
	return spdk_rdma_qp;

}



int

spdk_rdma_qp_accept(struct spdk_rdma_qp *spdk_rdma_qp, struct rdma_conn_param *conn_param)

{

	assert(spdk_rdma_qp != NULL);

	assert(spdk_rdma_qp->cm_id != NULL);



	return rdma_accept(spdk_rdma_qp->cm_id, conn_param);

}



int

spdk_rdma_qp_complete_connect(struct spdk_rdma_qp *spdk_rdma_qp)

{