Commit efe7ec16 authored by Aleksey Marchuk's avatar Aleksey Marchuk Committed by Jim Harris
Browse files

nvmf/rdma: List of active qpairs 



A poller maintains a list of qpairs which qdepth is > 0,
this list is used to process pending events instead of
iterating entire tree which might be suboptimal at high scale.

Change-Id: I547029e1a91638a23a6c188f596112aa8bc77497
Signed-off-by: default avatarAleksey Marchuk <alexeymar@nvidia.com>
Reviewed-on: https://review.spdk.io/c/spdk/spdk/+/26349


Reviewed-by: default avatarKonrad Sztyber <ksztyber@nvidia.com>
Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK Automated Test System <spdkbot@gmail.com>
Reviewed-by: default avatarJim Harris <jim.harris@nvidia.com>
parent fc0defc0

lib/nvmf/rdma.c

+15 −7
Original line number Diff line number Diff line
@@ -362,6 +362,8 @@ struct spdk_nvmf_rdma_qpair { 


	RB_ENTRY(spdk_nvmf_rdma_qpair)		node;



	TAILQ_ENTRY(spdk_nvmf_rdma_qpair)	active_link;



	STAILQ_ENTRY(spdk_nvmf_rdma_qpair)	recv_link;



	STAILQ_ENTRY(spdk_nvmf_rdma_qpair)	send_link;
@@ -423,6 +425,8 @@ struct spdk_nvmf_rdma_poller { 


	RB_HEAD(qpairs_tree, spdk_nvmf_rdma_qpair) qpairs;



	TAILQ_HEAD(, spdk_nvmf_rdma_qpair)	active_qpairs;



	STAILQ_HEAD(, spdk_nvmf_rdma_qpair)	qpairs_pending_recv;



	STAILQ_HEAD(, spdk_nvmf_rdma_qpair)	qpairs_pending_send;
@@ -882,6 +886,7 @@ nvmf_rdma_qpair_destroy(struct spdk_nvmf_rdma_qpair *rqpair) 


	if (rqpair->poller) {

		RB_REMOVE(qpairs_tree, &rqpair->poller->qpairs, rqpair);

		assert(TAILQ_ENTRY_NOT_ENQUEUED(rqpair, active_link));



		if (rqpair->srq != NULL && rqpair->resources != NULL) {

			/* Drop all received but unprocessed commands for this queue and return them to SRQ */
@@ -2004,6 +2009,10 @@ _nvmf_rdma_request_free(struct spdk_nvmf_rdma_request *rdma_req, 
	STAILQ_INSERT_HEAD(&rqpair->resources->free_queue, rdma_req, state_link);

	rqpair->qpair.queue_depth--;

	rdma_req->state = RDMA_REQUEST_STATE_FREE;

	if (rqpair->qpair.queue_depth == 0) {

		assert(TAILQ_ENTRY_ENQUEUED(rqpair, active_link));

		TAILQ_REMOVE_CLEAR(&rqpair->poller->active_qpairs, rqpair, active_link);

	}

}



static void
@@ -3463,13 +3472,7 @@ nvmf_rdma_poller_process_pending_qpairs(struct spdk_nvmf_rdma_transport *rtransp 
{

	struct spdk_nvmf_rdma_qpair *rqpair, *tmp;



	/* TODO: Here we iterate all qpairs, active and not active and touch at least 2 cache lines per

	 * qpair. On high scale with small number of active qpairs we may observe higher rate of L2 cache

	 * misses. To solve this problem we need to maintain a dedicated list of active qpairs */

	RB_FOREACH_SAFE(rqpair, qpairs_tree, &rpoller->qpairs, tmp) {

		if (rqpair->qpair.queue_depth == 0) {

			continue;

		}

	TAILQ_FOREACH_SAFE(rqpair, &rpoller->active_qpairs, active_link, tmp) {

		nvmf_rdma_qpair_process_pending(rtransport, rqpair, false);

	}

}
@@ -4070,6 +4073,7 @@ nvmf_rdma_poller_create(struct spdk_nvmf_rdma_transport *rtransport, 
	RB_INIT(&poller->qpairs);

	STAILQ_INIT(&poller->qpairs_pending_send);

	STAILQ_INIT(&poller->qpairs_pending_recv);

	TAILQ_INIT(&poller->active_qpairs);



	TAILQ_INSERT_TAIL(&rgroup->pollers, poller, link);

	SPDK_DEBUGLOG(rdma, "Create poller %p on device %p in poll group %p.\n", poller, device, rgroup);
@@ -4787,6 +4791,10 @@ nvmf_rdma_poller_poll(struct spdk_nvmf_rdma_transport *rtransport, 
			rdma_recv->receive_tsc = poll_tsc;

			rpoller->stat.requests++;

			STAILQ_INSERT_TAIL(&rqpair->resources->incoming_queue, rdma_recv, link);

			if (rqpair->qpair.queue_depth == 0) {

				assert(TAILQ_ENTRY_NOT_ENQUEUED(rqpair, active_link));

				TAILQ_INSERT_TAIL(&rpoller->active_qpairs, rqpair, active_link);

			}

			rqpair->qpair.queue_depth++;

			break;

		case RDMA_WR_TYPE_DATA: