Commit fa79f64a authored by Seth Howell's avatar Seth Howell Committed by Jim Harris
Browse files

rdma: Keep a pointer to the SRQ in the qpair 



Change-Id: Id173038b6ad6b1564acf5d6886814f7d310964c7
Signed-off-by: default avatarSeth Howell <seth.howell@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/447471


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
parent 01201d3e

lib/nvmf/rdma.c

+11 −9
Original line number Diff line number Diff line
@@ -308,6 +308,7 @@ struct spdk_nvmf_rdma_qpair { 
	struct spdk_nvmf_rdma_poller		*poller;



	struct rdma_cm_id			*cm_id;

	struct ibv_srq				*srq;

	struct rdma_cm_id			*listen_id;



	/* The maximum number of I/O outstanding on this connection at one time */
@@ -798,7 +799,7 @@ spdk_nvmf_rdma_qpair_destroy(struct spdk_nvmf_rdma_qpair *rqpair) 
	spdk_poller_unregister(&rqpair->destruct_poller);



	if (rqpair->qd != 0) {

		if (rqpair->poller->srq == NULL) {

		if (rqpair->srq == NULL) {

			nvmf_rdma_dump_qpair_contents(rqpair);

		}

		SPDK_WARNLOG("Destroying qpair when queue depth is %d\n", rqpair->qd);
@@ -807,12 +808,12 @@ spdk_nvmf_rdma_qpair_destroy(struct spdk_nvmf_rdma_qpair *rqpair) 
	if (rqpair->poller) {

		TAILQ_REMOVE(&rqpair->poller->qpairs, rqpair, link);



		if (rqpair->poller->srq != NULL) {

		if (rqpair->srq != NULL) {

			/* Drop all received but unprocessed commands for this queue and return them to SRQ */

			STAILQ_FOREACH_SAFE(rdma_recv, &rqpair->resources->incoming_queue, link, recv_tmp) {

				if (rqpair == rdma_recv->qpair) {

					STAILQ_REMOVE_HEAD(&rqpair->resources->incoming_queue, link);

					rc = ibv_post_srq_recv(rqpair->poller->srq, &rdma_recv->wr, &bad_recv_wr);

					rc = ibv_post_srq_recv(rqpair->srq, &rdma_recv->wr, &bad_recv_wr);

					if (rc) {

						SPDK_ERRLOG("Unable to re-post rx descriptor\n");

					}
@@ -830,7 +831,7 @@ spdk_nvmf_rdma_qpair_destroy(struct spdk_nvmf_rdma_qpair *rqpair) 
		}

	}



	if (rqpair->poller != NULL && rqpair->poller->srq == NULL) {

	if (rqpair->srq == NULL) {

		nvmf_rdma_resources_destroy(rqpair->resources);

	}
@@ -858,8 +859,8 @@ spdk_nvmf_rdma_qpair_initialize(struct spdk_nvmf_qpair *qpair) 
	ibv_init_attr.send_cq		= rqpair->poller->cq;

	ibv_init_attr.recv_cq		= rqpair->poller->cq;



	if (rqpair->poller->srq) {

		ibv_init_attr.srq		= rqpair->poller->srq;

	if (rqpair->srq) {

		ibv_init_attr.srq		= rqpair->srq;

	} else {

		ibv_init_attr.cap.max_recv_wr	= rqpair->max_queue_depth +

						  1; /* RECV operations + dummy drain WR */
@@ -1008,11 +1009,11 @@ request_transfer_out(struct spdk_nvmf_request *req, int *data_posted) 
	assert(rdma_req->recv != NULL);

	SPDK_DEBUGLOG(SPDK_LOG_RDMA, "RDMA RECV POSTED. Recv: %p Connection: %p\n", rdma_req->recv,

		      rqpair);

	if (rqpair->poller->srq == NULL) {

	if (rqpair->srq == NULL) {

		rc = ibv_post_recv(rqpair->cm_id->qp, &rdma_req->recv->wr, &bad_recv_wr);

	} else {

		rdma_req->recv->qpair = NULL;

		rc = ibv_post_srq_recv(rqpair->poller->srq, &rdma_req->recv->wr, &bad_recv_wr);

		rc = ibv_post_srq_recv(rqpair->srq, &rdma_req->recv->wr, &bad_recv_wr);

	}



	if (rc) {
@@ -2262,7 +2263,7 @@ spdk_nvmf_rdma_qpair_process_pending(struct spdk_nvmf_rdma_transport *rtransport 
		rdma_req->recv = STAILQ_FIRST(&resources->incoming_queue);

		STAILQ_REMOVE_HEAD(&resources->incoming_queue, link);



		if (rqpair->poller->srq != NULL) {

		if (rqpair->srq != NULL) {

			rdma_req->req.qpair = &rdma_req->recv->qpair->qpair;

			rdma_req->recv->qpair->qd++;

		} else {
@@ -2735,6 +2736,7 @@ spdk_nvmf_rdma_poll_group_add(struct spdk_nvmf_transport_poll_group *group, 


	TAILQ_INSERT_TAIL(&poller->qpairs, rqpair, link);

	rqpair->poller = poller;

	rqpair->srq = rqpair->poller->srq;



	rc = spdk_nvmf_rdma_qpair_initialize(qpair);

	if (rc < 0) {