Commit daa8f941 authored by Jacek Kalwas's avatar Jacek Kalwas Committed by Tomasz Zawadzki
Browse files

nvme: extend ctrlr opts with admin queue size 



Align rdma and tcp to respect opts. Reduce default number of entries
for admin queue so it becomes memory optimization.

Linux driver by default creates admin queue with 32 depth, there is no
good reason to enlarge that queue by default within SPDK NVMe driver.

Signed-off-by: default avatarJacek Kalwas <jacek.kalwas@intel.com>
Change-Id: I97ceea8f350c52313021a63190fb0980f604c48e
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/1110


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarAleksey Marchuk <alexeymar@mellanox.com>
Reviewed-by: default avatarChangpeng Liu <changpeng.liu@intel.com>
parent ea863bb0

include/spdk/nvme.h

+5 −0
Original line number Diff line number Diff line
@@ -234,6 +234,11 @@ struct spdk_nvme_ctrlr_opts { 
	 * 4.096 * 2^(transport_ack_timeout) usec.

	 */

	uint8_t transport_ack_timeout;



	/**

	 * The queue depth of NVMe Admin queue.

	 */

	uint16_t admin_queue_size;

};



/**

lib/nvme/nvme_ctrlr.c

+16 −0
Original line number Diff line number Diff line
@@ -178,6 +178,10 @@ spdk_nvme_ctrlr_get_default_ctrlr_opts(struct spdk_nvme_ctrlr_opts *opts, size_t 
	if (FIELD_OK(transport_ack_timeout)) {

		opts->transport_ack_timeout = SPDK_NVME_DEFAULT_TRANSPORT_ACK_TIMEOUT;

	}



	if (FIELD_OK(admin_queue_size)) {

		opts->admin_queue_size = DEFAULT_ADMIN_QUEUE_SIZE;

	}

#undef FIELD_OK

}
@@ -2576,6 +2580,18 @@ nvme_ctrlr_construct(struct spdk_nvme_ctrlr *ctrlr) 
		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT, NVME_TIMEOUT_INFINITE);

	}



	if (ctrlr->opts.admin_queue_size > SPDK_NVME_ADMIN_QUEUE_MAX_ENTRIES) {

		SPDK_ERRLOG("admin_queue_size %u exceeds max defined by NVMe spec, use max value\n",

			    ctrlr->opts.admin_queue_size);

		ctrlr->opts.admin_queue_size = SPDK_NVME_ADMIN_QUEUE_MAX_ENTRIES;

	}



	if (ctrlr->opts.admin_queue_size < SPDK_NVME_ADMIN_QUEUE_MIN_ENTRIES) {

		SPDK_ERRLOG("admin_queue_size %u is less than minimum defined by NVMe spec, use min value\n",

			    ctrlr->opts.admin_queue_size);

		ctrlr->opts.admin_queue_size = SPDK_NVME_ADMIN_QUEUE_MIN_ENTRIES;

	}



	ctrlr->flags = 0;

	ctrlr->free_io_qids = NULL;

	ctrlr->is_resetting = false;

lib/nvme/nvme_internal.h

+1 −1
Original line number Diff line number Diff line
@@ -151,10 +151,10 @@ extern pid_t g_spdk_nvme_pid; 
 *  try to configure, if available.

 */

#define DEFAULT_MAX_IO_QUEUES		(1024)

#define DEFAULT_ADMIN_QUEUE_SIZE	(32)

#define DEFAULT_IO_QUEUE_SIZE		(256)

#define DEFAULT_IO_QUEUE_SIZE_FOR_QUIRK	(1024) /* Matches Linux kernel driver */



#define DEFAULT_ADMIN_QUEUE_REQUESTS	(32)

#define DEFAULT_IO_QUEUE_REQUESTS	(512)



#define SPDK_NVME_DEFAULT_RETRY_COUNT	(4)

lib/nvme/nvme_pcie.c

+4 −6
Original line number Diff line number Diff line
@@ -50,8 +50,6 @@ 
#define NVME_MIN_COMPLETIONS	(1)

#define NVME_MAX_COMPLETIONS	(128)



#define NVME_ADMIN_ENTRIES	(128)



/*

 * NVME_MAX_SGL_DESCRIPTORS defines the maximum number of descriptors in one SGL

 *  segment.
@@ -673,7 +671,7 @@ nvme_pcie_ctrlr_free_bars(struct nvme_pcie_ctrlr *pctrlr) 
}



static int

nvme_pcie_ctrlr_construct_admin_qpair(struct spdk_nvme_ctrlr *ctrlr)

nvme_pcie_ctrlr_construct_admin_qpair(struct spdk_nvme_ctrlr *ctrlr, uint16_t num_entries)

{

	struct nvme_pcie_qpair *pqpair;

	int rc;
@@ -683,7 +681,7 @@ nvme_pcie_ctrlr_construct_admin_qpair(struct spdk_nvme_ctrlr *ctrlr) 
		return -ENOMEM;

	}



	pqpair->num_entries = NVME_ADMIN_ENTRIES;

	pqpair->num_entries = num_entries;

	pqpair->flags.delay_cmd_submit = 0;



	ctrlr->adminq = &pqpair->qpair;
@@ -692,7 +690,7 @@ nvme_pcie_ctrlr_construct_admin_qpair(struct spdk_nvme_ctrlr *ctrlr) 
			     0, /* qpair ID */

			     ctrlr,

			     SPDK_NVME_QPRIO_URGENT,

			     NVME_ADMIN_ENTRIES);

			     num_entries);

	if (rc != 0) {

		return rc;

	}
@@ -856,7 +854,7 @@ static struct spdk_nvme_ctrlr *nvme_pcie_ctrlr_construct(const struct spdk_nvme_ 
	pci_id = spdk_pci_device_get_id(pci_dev);

	pctrlr->ctrlr.quirks = nvme_get_quirks(&pci_id);



	rc = nvme_pcie_ctrlr_construct_admin_qpair(&pctrlr->ctrlr);

	rc = nvme_pcie_ctrlr_construct_admin_qpair(&pctrlr->ctrlr, pctrlr->ctrlr.opts.admin_queue_size);

	if (rc != 0) {

		nvme_ctrlr_destruct(&pctrlr->ctrlr);

		return NULL;

lib/nvme/nvme_rdma.c

+2 −2
Original line number Diff line number Diff line
@@ -1816,8 +1816,8 @@ static struct spdk_nvme_ctrlr *nvme_rdma_ctrlr_construct(const struct spdk_nvme_ 
	}



	rctrlr->ctrlr.adminq = nvme_rdma_ctrlr_create_qpair(&rctrlr->ctrlr, 0,

			       SPDK_NVMF_MIN_ADMIN_QUEUE_ENTRIES, 0, SPDK_NVMF_MIN_ADMIN_QUEUE_ENTRIES,

			       false);

			       rctrlr->ctrlr.opts.admin_queue_size, 0,

			       rctrlr->ctrlr.opts.admin_queue_size, false);

	if (!rctrlr->ctrlr.adminq) {

		SPDK_ERRLOG("failed to create admin qpair\n");

		nvme_rdma_ctrlr_destruct(&rctrlr->ctrlr);