Commit 696e8580 authored by Alexey Marchuk's avatar Alexey Marchuk Committed by Tomasz Zawadzki
Browse files

nvmf/rdma: Use RDMA provider SRQ API 



Change-Id: I9401efe73f7d0b39d0449b304ea2c211efb3a9f2
Signed-off-by: default avatarAlexey Marchuk <alexeymar@mellanox.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/6291


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
parent d0b2448b

lib/nvmf/rdma.c

+43 −32
Original line number Diff line number Diff line
@@ -334,7 +334,7 @@ struct spdk_nvmf_rdma_qpair { 


	struct spdk_rdma_qp			*rdma_qp;

	struct rdma_cm_id			*cm_id;

	struct ibv_srq				*srq;

	struct spdk_rdma_srq			*srq;

	struct rdma_cm_id			*listen_id;



	/* The maximum number of I/O outstanding on this connection at one time */
@@ -422,7 +422,7 @@ struct spdk_nvmf_rdma_poller { 
	uint16_t				max_srq_depth;



	/* Shared receive queue */

	struct ibv_srq				*srq;

	struct spdk_rdma_srq			*srq;



	struct spdk_nvmf_rdma_resources		*resources;

	struct spdk_nvmf_rdma_poller_stat	stat;
@@ -687,10 +687,11 @@ nvmf_rdma_resources_create(struct spdk_nvmf_rdma_resource_opts *opts) 
	struct spdk_nvmf_rdma_resources	*resources;

	struct spdk_nvmf_rdma_request	*rdma_req;

	struct spdk_nvmf_rdma_recv	*rdma_recv;

	struct ibv_qp			*qp;

	struct ibv_srq			*srq;

	struct ibv_qp			*qp = NULL;

	struct spdk_rdma_srq		*srq = NULL;

	struct ibv_recv_wr		*bad_wr = NULL;

	uint32_t			i;

	int				rc;

	int				rc = 0;



	resources = calloc(1, sizeof(struct spdk_nvmf_rdma_resources));

	if (!resources) {
@@ -752,9 +753,13 @@ nvmf_rdma_resources_create(struct spdk_nvmf_rdma_resource_opts *opts) 
	STAILQ_INIT(&resources->incoming_queue);

	STAILQ_INIT(&resources->free_queue);



	for (i = 0; i < opts->max_queue_depth; i++) {

		struct ibv_recv_wr *bad_wr = NULL;

	if (opts->shared) {

		srq = (struct spdk_rdma_srq *)opts->qp;

	} else {

		qp = (struct ibv_qp *)opts->qp;

	}



	for (i = 0; i < opts->max_queue_depth; i++) {

		rdma_recv = &resources->recvs[i];

		rdma_recv->qpair = opts->qpair;
@@ -781,10 +786,8 @@ nvmf_rdma_resources_create(struct spdk_nvmf_rdma_resource_opts *opts) 
		rdma_recv->wr.wr_id = (uintptr_t)&rdma_recv->rdma_wr;

		rdma_recv->wr.sg_list = rdma_recv->sgl;

		if (opts->shared) {

			srq = (struct ibv_srq *)opts->qp;

			rc = ibv_post_srq_recv(srq, &rdma_recv->wr, &bad_wr);

			spdk_rdma_srq_queue_recv_wrs(srq, &rdma_recv->wr);

		} else {

			qp = (struct ibv_qp *)opts->qp;

			rc = ibv_post_recv(qp, &rdma_recv->wr, &bad_wr);

		}

		if (rc) {
@@ -830,6 +833,13 @@ nvmf_rdma_resources_create(struct spdk_nvmf_rdma_resource_opts *opts) 
		STAILQ_INSERT_TAIL(&resources->free_queue, rdma_req, state_link);

	}



	if (opts->shared) {

		rc = spdk_rdma_srq_flush_recv_wrs(srq, &bad_wr);

		if (rc) {

			goto cleanup;

		}

	}



	return resources;



cleanup:
@@ -893,7 +903,8 @@ nvmf_rdma_qpair_destroy(struct spdk_nvmf_rdma_qpair *rqpair) 
			STAILQ_FOREACH_SAFE(rdma_recv, &rqpair->resources->incoming_queue, link, recv_tmp) {

				if (rqpair == rdma_recv->qpair) {

					STAILQ_REMOVE(&rqpair->resources->incoming_queue, rdma_recv, spdk_nvmf_rdma_recv, link);

					rc = ibv_post_srq_recv(rqpair->srq, &rdma_recv->wr, &bad_recv_wr);

					spdk_rdma_srq_queue_recv_wrs(rqpair->srq, &rdma_recv->wr);

					rc = spdk_rdma_srq_flush_recv_wrs(rqpair->srq, &bad_recv_wr);

					if (rc) {

						SPDK_ERRLOG("Unable to re-post rx descriptor\n");

					}
@@ -988,7 +999,7 @@ nvmf_rdma_qpair_initialize(struct spdk_nvmf_qpair *qpair) 
	qp_init_attr.recv_cq	= rqpair->poller->cq;



	if (rqpair->srq) {

		qp_init_attr.srq		= rqpair->srq;

		qp_init_attr.srq		= rqpair->srq->srq;

	} else {

		qp_init_attr.cap.max_recv_wr	= rqpair->max_queue_depth;

	}
@@ -1058,6 +1069,11 @@ nvmf_rdma_qpair_queue_recv_wrs(struct spdk_nvmf_rdma_qpair *rqpair, struct ibv_r 
	struct spdk_nvmf_rdma_transport *rtransport = SPDK_CONTAINEROF(rqpair->qpair.transport,

			struct spdk_nvmf_rdma_transport, transport);



	if (rqpair->srq != NULL) {

		spdk_rdma_srq_queue_recv_wrs(rqpair->srq, first);

		return;

	}



	last = first;

	while (last->next != NULL) {

		last = last->next;
@@ -1066,9 +1082,7 @@ nvmf_rdma_qpair_queue_recv_wrs(struct spdk_nvmf_rdma_qpair *rqpair, struct ibv_r 
	if (rqpair->resources->recvs_to_post.first == NULL) {

		rqpair->resources->recvs_to_post.first = first;

		rqpair->resources->recvs_to_post.last = last;

		if (rqpair->srq == NULL) {

		STAILQ_INSERT_TAIL(&rqpair->poller->qpairs_pending_recv, rqpair, recv_link);

		}

	} else {

		rqpair->resources->recvs_to_post.last->next = first;

		rqpair->resources->recvs_to_post.last = last;
@@ -3214,7 +3228,7 @@ nvmf_rdma_poll_group_create(struct spdk_nvmf_transport *transport) 
	struct spdk_nvmf_rdma_poll_group	*rgroup;

	struct spdk_nvmf_rdma_poller		*poller;

	struct spdk_nvmf_rdma_device		*device;

	struct ibv_srq_init_attr		srq_init_attr;

	struct spdk_rdma_srq_init_attr		srq_init_attr;

	struct spdk_nvmf_rdma_resource_opts	opts;

	int					num_cqe;
@@ -3249,10 +3263,11 @@ nvmf_rdma_poll_group_create(struct spdk_nvmf_transport *transport) 
			poller->max_srq_depth = rtransport->rdma_opts.max_srq_depth;



			device->num_srq++;

			memset(&srq_init_attr, 0, sizeof(struct ibv_srq_init_attr));

			srq_init_attr.attr.max_wr = poller->max_srq_depth;

			srq_init_attr.attr.max_sge = spdk_min(device->attr.max_sge, NVMF_DEFAULT_RX_SGE);

			poller->srq = ibv_create_srq(device->pd, &srq_init_attr);

			memset(&srq_init_attr, 0, sizeof(srq_init_attr));

			srq_init_attr.pd = device->pd;

			srq_init_attr.srq_init_attr.attr.max_wr = poller->max_srq_depth;

			srq_init_attr.srq_init_attr.attr.max_sge = spdk_min(device->attr.max_sge, NVMF_DEFAULT_RX_SGE);

			poller->srq = spdk_rdma_srq_create(&srq_init_attr);

			if (!poller->srq) {

				SPDK_ERRLOG("Unable to create shared receive queue, errno %d\n", errno);

				nvmf_rdma_poll_group_destroy(&rgroup->group);
@@ -3368,7 +3383,7 @@ nvmf_rdma_poll_group_destroy(struct spdk_nvmf_transport_poll_group *group) 
			if (poller->resources) {

				nvmf_rdma_resources_destroy(poller->resources);

			}

			ibv_destroy_srq(poller->srq);

			spdk_rdma_srq_destroy(poller->srq);

			SPDK_DEBUGLOG(rdma, "Destroyed RDMA shared queue %p\n", poller->srq);

		}
@@ -3503,7 +3518,8 @@ nvmf_rdma_request_free(struct spdk_nvmf_request *req) 
		int rc;

		struct ibv_recv_wr *bad_recv_wr;



		rc = ibv_post_srq_recv(rqpair->srq, &rdma_req->recv->wr, &bad_recv_wr);

		spdk_rdma_srq_queue_recv_wrs(rqpair->srq, &rdma_req->recv->wr);

		rc = spdk_rdma_srq_flush_recv_wrs(rqpair->srq, &bad_recv_wr);

		if (rc) {

			SPDK_ERRLOG("Unable to re-post rx descriptor\n");

		}
@@ -3628,14 +3644,10 @@ _poller_submit_recvs(struct spdk_nvmf_rdma_transport *rtransport, 
	int				rc;



	if (rpoller->srq) {

		if (rpoller->resources->recvs_to_post.first != NULL) {

			rc = ibv_post_srq_recv(rpoller->srq, rpoller->resources->recvs_to_post.first, &bad_recv_wr);

		rc = spdk_rdma_srq_flush_recv_wrs(rpoller->srq, &bad_recv_wr);

		if (rc) {

			_poller_reset_failed_recvs(rpoller, bad_recv_wr, rc);

		}

			rpoller->resources->recvs_to_post.first = NULL;

			rpoller->resources->recvs_to_post.last = NULL;

		}

	} else {

		while (!STAILQ_EMPTY(&rpoller->qpairs_pending_recv)) {

			rqpair = STAILQ_FIRST(&rpoller->qpairs_pending_recv);
@@ -3828,9 +3840,8 @@ nvmf_rdma_poller_poll(struct spdk_nvmf_rdma_transport *rtransport, 
					int rc;



					rdma_recv->wr.next = NULL;

					rc = ibv_post_srq_recv(rpoller->srq,

							       &rdma_recv->wr,

							       &bad_wr);

					spdk_rdma_srq_queue_recv_wrs(rpoller->srq, &rdma_recv->wr);

					rc = spdk_rdma_srq_flush_recv_wrs(rpoller->srq, &bad_wr);

					if (rc) {

						SPDK_ERRLOG("Failed to re-post recv WR to SRQ, err %d\n", rc);

					}

test/common/lib/test_rdma.c

+7 −0
Original line number Diff line number Diff line
@@ -50,6 +50,13 @@ DEFINE_STUB(spdk_rdma_qp_queue_send_wrs, bool, (struct spdk_rdma_qp *spdk_rdma_q 
		struct ibv_send_wr *first), true);

DEFINE_STUB(spdk_rdma_qp_flush_send_wrs, int, (struct spdk_rdma_qp *spdk_rdma_qp,

		struct ibv_send_wr **bad_wr), 0);

DEFINE_STUB(spdk_rdma_srq_create, struct spdk_rdma_srq *,

	    (struct spdk_rdma_srq_init_attr *init_attr), NULL);

DEFINE_STUB(spdk_rdma_srq_destroy, int, (struct spdk_rdma_srq *rdma_srq), 0);

DEFINE_STUB(spdk_rdma_srq_queue_recv_wrs, bool, (struct spdk_rdma_srq *rdma_srq,

		struct ibv_recv_wr *first), true);

DEFINE_STUB(spdk_rdma_srq_flush_recv_wrs, int, (struct spdk_rdma_srq *rdma_srq,

		struct ibv_recv_wr **bad_wr), 0);

DEFINE_STUB(spdk_rdma_create_mem_map, struct spdk_rdma_mem_map *, (struct ibv_pd *pd,

		struct spdk_nvme_rdma_hooks *hooks), NULL);

DEFINE_STUB_V(spdk_rdma_free_mem_map, (struct spdk_rdma_mem_map **map));