Commit 5c80b1e5 authored by Evgeniy Kochetov's avatar Evgeniy Kochetov Committed by Tomasz Zawadzki
Browse files

nvme/rdma: Limit max_sges by command capsule size 



According to NVMe over Fabrics spec number of SGLs supported by the
controller is reported in MSDBD. But it is also implicitly limited by
command capsule size (IOCCSZ) since SGL are passed in capsule.

This patch adjusts max_sges to capsule size if required. Adjustment to
MSDBD is also moved to transport layer because it is fabrics specific
parameter and is not valid for PCIe transport.

Signed-off-by: default avatarEvgeniy Kochetov <evgeniik@nvidia.com>
Change-Id: I44918eb949345c61242ca50a524d21d04b6ac058
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/11669


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Community-CI: Mellanox Build Bot
Reviewed-by: default avatarAleksey Marchuk <alexeymar@mellanox.com>
Reviewed-by: default avatarShuhei Matsumoto <smatsumoto@nvidia.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
parent 742d818e

lib/nvme/nvme_ctrlr.c

+1 −9
Original line number Diff line number Diff line
@@ -1986,16 +1986,8 @@ nvme_ctrlr_identify_done(void *arg, const struct spdk_nvme_cpl *cpl) 
		if (ctrlr->cdata.sgls.supported == 0x2) {

			ctrlr->flags |= SPDK_NVME_CTRLR_SGL_REQUIRES_DWORD_ALIGNMENT;

		}

		/*

		 * Use MSDBD to ensure our max_sges doesn't exceed what the

		 *  controller supports.

		 */



		ctrlr->max_sges = nvme_transport_ctrlr_get_max_sges(ctrlr);

		if (ctrlr->cdata.nvmf_specific.msdbd != 0) {

			ctrlr->max_sges = spdk_min(ctrlr->cdata.nvmf_specific.msdbd, ctrlr->max_sges);

		} else {

			/* A value 0 indicates no limit. */

		}

		NVME_CTRLR_DEBUGLOG(ctrlr, "transport max_sges %u\n", ctrlr->max_sges);

	}

lib/nvme/nvme_rdma.c

+14 −1
Original line number Diff line number Diff line
@@ -2434,8 +2434,21 @@ static uint16_t 
nvme_rdma_ctrlr_get_max_sges(struct spdk_nvme_ctrlr *ctrlr)

{

	struct nvme_rdma_ctrlr *rctrlr = nvme_rdma_ctrlr(ctrlr);

	uint32_t max_sge = rctrlr->max_sge;

	uint32_t max_in_capsule_sge = (ctrlr->cdata.nvmf_specific.ioccsz * 16 -

				       sizeof(struct spdk_nvme_cmd)) /

				      sizeof(struct spdk_nvme_sgl_descriptor);



	return rctrlr->max_sge;

	/* Max SGE is limited by capsule size */

	max_sge = spdk_min(max_sge, max_in_capsule_sge);

	/* Max SGE may be limited by MSDBD */

	if (ctrlr->cdata.nvmf_specific.msdbd != 0) {

		max_sge = spdk_min(max_sge, ctrlr->cdata.nvmf_specific.msdbd);

	}



	/* Max SGE can't be less than 1 */

	max_sge = spdk_max(1, max_sge);

	return max_sge;

}



static int

test/unit/lib/nvme/nvme_rdma.c/nvme_rdma_ut.c

+29 −0
Original line number Diff line number Diff line
@@ -1370,6 +1370,34 @@ test_nvme_rdma_poll_group_get_qpair_by_id(void) 
	STAILQ_REMOVE_HEAD(&group.destroyed_qpairs, link);

}



static void

test_nvme_rdma_ctrlr_get_max_sges(void)

{

	struct nvme_rdma_ctrlr	rctrlr = {};



	rctrlr.ctrlr.trid.trtype = SPDK_NVME_TRANSPORT_RDMA;

	rctrlr.max_sge = NVME_RDMA_MAX_SGL_DESCRIPTORS;

	rctrlr.ctrlr.cdata.nvmf_specific.msdbd = 16;

	rctrlr.ctrlr.cdata.nvmf_specific.ioccsz = 4096;

	CU_ASSERT(nvme_rdma_ctrlr_get_max_sges(&rctrlr.ctrlr) == 16);



	rctrlr.ctrlr.cdata.nvmf_specific.msdbd = 32;

	rctrlr.ctrlr.cdata.nvmf_specific.ioccsz = 4096;

	CU_ASSERT(nvme_rdma_ctrlr_get_max_sges(&rctrlr.ctrlr) == 16);



	rctrlr.ctrlr.cdata.nvmf_specific.msdbd = 8;

	rctrlr.ctrlr.cdata.nvmf_specific.ioccsz = 4096;

	CU_ASSERT(nvme_rdma_ctrlr_get_max_sges(&rctrlr.ctrlr) == 8);



	rctrlr.ctrlr.cdata.nvmf_specific.msdbd = 16;

	rctrlr.ctrlr.cdata.nvmf_specific.ioccsz = 4;

	CU_ASSERT(nvme_rdma_ctrlr_get_max_sges(&rctrlr.ctrlr) == 1);



	rctrlr.ctrlr.cdata.nvmf_specific.msdbd = 16;

	rctrlr.ctrlr.cdata.nvmf_specific.ioccsz = 6;

	CU_ASSERT(nvme_rdma_ctrlr_get_max_sges(&rctrlr.ctrlr) == 2);

}



int main(int argc, char **argv)

{

	CU_pSuite	suite = NULL;
@@ -1402,6 +1430,7 @@ int main(int argc, char **argv) 
	CU_ADD_TEST(suite, test_rdma_ctrlr_get_memory_domains);

	CU_ADD_TEST(suite, test_rdma_get_memory_translation);

	CU_ADD_TEST(suite, test_nvme_rdma_poll_group_get_qpair_by_id);

	CU_ADD_TEST(suite, test_nvme_rdma_ctrlr_get_max_sges);



	CU_basic_set_mode(CU_BRM_VERBOSE);

	CU_basic_run_tests();