Commit 10392543 authored by Seth Howell's avatar Seth Howell Committed by Tomasz Zawadzki
Browse files

nvme/rdma: add cq resizing. 



Signed-off-by: default avatarSeth Howell <seth.howell@intel.com>
Change-Id: I6350d76b8c1e778c18e693b2dfbb10dd36b3e3d8
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/1927


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: default avatarAleksey Marchuk <alexeymar@mellanox.com>
parent 67b0dcfe

lib/nvme/nvme_rdma.c

+35 −0
Original line number Diff line number Diff line
@@ -95,6 +95,8 @@ 
 */

#define NVME_RDMA_DESTROYED_QPAIR_EXPIRATION_CYCLES	50



#define WC_PER_QPAIR(queue_depth)	(queue_depth * 2)



enum nvme_rdma_wr_type {

	RDMA_WR_TYPE_RECV,

	RDMA_WR_TYPE_SEND,
@@ -152,6 +154,8 @@ struct nvme_rdma_destroyed_qpair { 
struct nvme_rdma_poller {

	struct ibv_context		*device;

	struct ibv_cq			*cq;

	int				required_num_wc;

	int				current_num_wc;

	STAILQ_ENTRY(nvme_rdma_poller)	link;

};
@@ -2427,6 +2431,8 @@ nvme_rdma_poller_create(struct nvme_rdma_poll_group *group, struct ibv_context * 


	STAILQ_INSERT_HEAD(&group->pollers, poller, link);

	group->num_pollers++;

	poller->current_num_wc = DEFAULT_NVME_RDMA_CQ_SIZE;

	poller->required_num_wc = 0;

	return 0;

}
@@ -2512,6 +2518,32 @@ nvme_rdma_poll_group_get_qpair_by_id(struct nvme_rdma_poll_group *group, uint32_ 
	return NULL;

}



static int

nvme_rdma_resize_cq(struct nvme_rdma_qpair *rqpair, struct nvme_rdma_poller *poller)

{

	int	current_num_wc, required_num_wc;



	required_num_wc = poller->required_num_wc + WC_PER_QPAIR(rqpair->num_entries);

	current_num_wc = poller->current_num_wc;

	if (current_num_wc < required_num_wc) {

		current_num_wc = spdk_max(current_num_wc * 2, required_num_wc);

	}



	if (poller->current_num_wc != current_num_wc) {

		SPDK_DEBUGLOG(SPDK_LOG_NVME, "Resize RDMA CQ from %d to %d\n", poller->current_num_wc,

			      current_num_wc);

		if (ibv_resize_cq(poller->cq, current_num_wc)) {

			SPDK_ERRLOG("RDMA CQ resize failed: errno %d: %s\n", errno, spdk_strerror(errno));

			return -1;

		}



		poller->current_num_wc = current_num_wc;

	}



	poller->required_num_wc = required_num_wc;

	return 0;

}



static int

nvme_rdma_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair)

{
@@ -2523,6 +2555,9 @@ nvme_rdma_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair) 


	STAILQ_FOREACH(poller, &group->pollers, link) {

		if (poller->device == rqpair->cm_id->verbs) {

			if (nvme_rdma_resize_cq(rqpair, poller)) {

				return -EPROTO;

			}

			rqpair->cq = poller->cq;

			break;

		}