Commit d1357fd8 authored by Shuhei Matsumoto's avatar Shuhei Matsumoto Committed by Tomasz Zawadzki
Browse files

nvme_rdma: Add option to limit indefinite growth of CQ size 



CQ size had grown indefinitely every when an I/O qpair was added to a
poller. This was not desirable. To limit the CQ size, add a transport
option, rdma_max_cq_size. It is zero, which means unlimited, by default.

Signed-off-by: default avatarShuhei Matsumoto <smatsumoto@nvidia.com>
Change-Id: I8867ad4600ce85cc13c3c8b7129e39563e2cbfc9
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/18840


Reviewed-by: default avatarBen Walker <ben@nvidia.com>
Reviewed-by: default avatarAleksey Marchuk <alexeymar@nvidia.com>
Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
parent c71f33ee

CHANGELOG.md

+4 −0
Original line number Diff line number Diff line
@@ -16,6 +16,10 @@ live recovery feature of vhost-scsi target. 


Added support for `SBC WRITE SAME 10` and `SBC WRITE SAME 16`.



### nvme



A new transport option `rdma_max_cq_size` was added to limit indefinite growth of CQ size.



## v23.09



### accel

include/spdk/nvme.h

+9 −1
Original line number Diff line number Diff line
@@ -4270,8 +4270,16 @@ struct spdk_nvme_transport_opts { 
	 * structure are valid. And the library will populate any remaining fields with default values.

	 */

	size_t opts_size;



	/**

	 * It is used for RDMA transport.

	 *

	 * The maximum queue depth of a rdma completion queue.

	 * It is zero, which means unlimited, by default.

	 */

	uint32_t rdma_max_cq_size;

};

SPDK_STATIC_ASSERT(sizeof(struct spdk_nvme_transport_opts) == 16, "Incorrect size");

SPDK_STATIC_ASSERT(sizeof(struct spdk_nvme_transport_opts) == 24, "Incorrect size");



/**

 * Get the current NVMe transport options.

lib/nvme/Makefile

+2 −2
Original line number Diff line number Diff line
@@ -6,8 +6,8 @@ 
SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)

include $(SPDK_ROOT_DIR)/mk/spdk.common.mk



SO_VER := 11

SO_MINOR := 1

SO_VER := 12

SO_MINOR := 0



C_SRCS = nvme_ctrlr_cmd.c nvme_ctrlr.c nvme_fabric.c nvme_ns_cmd.c \

	nvme_ns.c nvme_pcie_common.c nvme_pcie.c nvme_qpair.c nvme.c \

lib/nvme/nvme_rdma.c

+19 −2
Original line number Diff line number Diff line
@@ -679,6 +679,7 @@ static int 
nvme_rdma_resize_cq(struct nvme_rdma_qpair *rqpair, struct nvme_rdma_poller *poller)

{

	int	current_num_wc, required_num_wc;

	int	max_cq_size;



	required_num_wc = poller->required_num_wc + WC_PER_QPAIR(rqpair->num_entries);

	current_num_wc = poller->current_num_wc;
@@ -686,6 +687,11 @@ nvme_rdma_resize_cq(struct nvme_rdma_qpair *rqpair, struct nvme_rdma_poller *pol 
		current_num_wc = spdk_max(current_num_wc * 2, required_num_wc);

	}



	max_cq_size = g_spdk_nvme_transport_opts.rdma_max_cq_size;

	if (max_cq_size != 0 && current_num_wc > max_cq_size) {

		current_num_wc = max_cq_size;

	}



	if (poller->current_num_wc != current_num_wc) {

		SPDK_DEBUGLOG(nvme, "Resize RDMA CQ from %d to %d\n", poller->current_num_wc,

			      current_num_wc);
@@ -739,6 +745,7 @@ nvme_rdma_qpair_init(struct nvme_rdma_qpair *rqpair) 
	struct spdk_rdma_qp_init_attr	attr = {};

	struct ibv_device_attr	dev_attr;

	struct nvme_rdma_ctrlr	*rctrlr;

	uint32_t num_cqe, max_num_cqe;



	rc = ibv_query_device(rqpair->cm_id->verbs, &dev_attr);

	if (rc != 0) {
@@ -755,7 +762,12 @@ nvme_rdma_qpair_init(struct nvme_rdma_qpair *rqpair) 
		}

		assert(rqpair->cq);

	} else {

		rqpair->cq = ibv_create_cq(rqpair->cm_id->verbs, rqpair->num_entries * 2, rqpair, NULL, 0);

		num_cqe = rqpair->num_entries * 2;

		max_num_cqe = g_spdk_nvme_transport_opts.rdma_max_cq_size;

		if (max_num_cqe != 0 && num_cqe > max_num_cqe) {

			num_cqe = max_num_cqe;

		}

		rqpair->cq = ibv_create_cq(rqpair->cm_id->verbs, num_cqe, rqpair, NULL, 0);

		if (!rqpair->cq) {

			SPDK_ERRLOG("Unable to create completion queue: errno %d: %s\n", errno, spdk_strerror(errno));

			return -1;
@@ -2868,7 +2880,7 @@ nvme_rdma_poller_create(struct nvme_rdma_poll_group *group, struct ibv_context * 
	struct ibv_device_attr dev_attr;

	struct spdk_rdma_srq_init_attr srq_init_attr = {};

	struct nvme_rdma_rsp_opts opts;

	int num_cqe;

	int num_cqe, max_num_cqe;

	int rc;



	poller = calloc(1, sizeof(*poller));
@@ -2940,6 +2952,11 @@ nvme_rdma_poller_create(struct nvme_rdma_poll_group *group, struct ibv_context * 
		num_cqe = DEFAULT_NVME_RDMA_CQ_SIZE;

	}



	max_num_cqe = g_spdk_nvme_transport_opts.rdma_max_cq_size;

	if (max_num_cqe != 0 && num_cqe > max_num_cqe) {

		num_cqe = max_num_cqe;

	}



	poller->cq = ibv_create_cq(poller->device, num_cqe, group, NULL, 0);



	if (poller->cq == NULL) {

lib/nvme/nvme_transport.c

+4 −1
Original line number Diff line number Diff line
@@ -27,6 +27,7 @@ int g_current_transport_index = 0; 


struct spdk_nvme_transport_opts g_spdk_nvme_transport_opts = {

	.rdma_srq_size = 0,

	.rdma_max_cq_size = 0,

};



const struct spdk_nvme_transport *
@@ -818,10 +819,11 @@ spdk_nvme_transport_get_opts(struct spdk_nvme_transport_opts *opts, size_t opts_ 
	} \



	SET_FIELD(rdma_srq_size);

	SET_FIELD(rdma_max_cq_size);



	/* Do not remove this statement, you should always update this statement when you adding a new field,

	 * and do not forget to add the SET_FIELD statement for your added field. */

	SPDK_STATIC_ASSERT(sizeof(struct spdk_nvme_transport_opts) == 16, "Incorrect size");

	SPDK_STATIC_ASSERT(sizeof(struct spdk_nvme_transport_opts) == 24, "Incorrect size");



#undef SET_FIELD

}
@@ -845,6 +847,7 @@ spdk_nvme_transport_set_opts(const struct spdk_nvme_transport_opts *opts, size_t 
	} \



	SET_FIELD(rdma_srq_size);

	SET_FIELD(rdma_max_cq_size);



	g_spdk_nvme_transport_opts.opts_size = opts->opts_size;