Commit 8b539eb5 authored by Shuhei Matsumoto's avatar Shuhei Matsumoto Committed by Changpeng Liu
Browse files

nvme: Set appropriate value to max_xfer_size and max_sge 



SPDK NVMe-oF initiator driver could not transfer IO whose size is
more than 128KiB even if NVMe-oF target allows IO whose size is
more than 128KiB both for RDMA and TCP transport.

Some use cases need to transfer IO larger than 128KiB.

For RDMA transport, max_mr_size by ibv_query_device of RDMA devices
indicates the maximum size of a single memory region and is independent
from the actual I/O size, and is very likely to be larger than 2 MiB
which is the granularity we currently register memory regions.

Actually some RDMA NICs return UINT64_MAX for max_mr_size by ibv_query_device.

Hence use UINT32_MAX and let the generic layer use the controller data
to moderate this value.

On the other hand, for TCP transport, there is no limit for maximum IO
size and hence use UINT32_MAX.

Besides, for RDMA transport, max_sges should be the minimum of
max_sge got by querying RDMA devices and NVME_RDMA_MAX_SGL_DESCRIPTORS.
Hence do this change together in this patch.

Signed-off-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Change-Id: Idc813afd3e525bf5f370c0fcd2623f9c146a5528
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/459218


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarSeth Howell <seth.howell5141@gmail.com>
Reviewed-by: default avatarZiye Yang <ziye.yang@intel.com>
Reviewed-by: default avatarChangpeng Liu <changpeng.liu@intel.com>
parent cf3c54bc

lib/nvme/nvme_rdma.c

+40 −4
Original line number Diff line number Diff line
@@ -86,6 +86,8 @@ struct nvme_rdma_ctrlr { 
	struct spdk_nvme_ctrlr			ctrlr;



	struct ibv_pd				*pd;



	uint16_t				max_sge;

};



/* NVMe RDMA qpair extensions for spdk_nvme_qpair */
@@ -1461,7 +1463,9 @@ struct spdk_nvme_ctrlr *nvme_rdma_ctrlr_construct(const struct spdk_nvme_transpo 
	struct nvme_rdma_ctrlr *rctrlr;

	union spdk_nvme_cap_register cap;

	union spdk_nvme_vs_register vs;

	int rc;

	struct ibv_context **contexts;

	struct ibv_device_attr dev_attr;

	int i, rc;



	rctrlr = calloc(1, sizeof(struct nvme_rdma_ctrlr));

	if (rctrlr == NULL) {
@@ -1473,6 +1477,30 @@ struct spdk_nvme_ctrlr *nvme_rdma_ctrlr_construct(const struct spdk_nvme_transpo 
	rctrlr->ctrlr.opts = *opts;

	memcpy(&rctrlr->ctrlr.trid, trid, sizeof(rctrlr->ctrlr.trid));



	contexts = rdma_get_devices(NULL);

	if (contexts == NULL) {

		SPDK_ERRLOG("rdma_get_devices() failed: %s (%d)\n", spdk_strerror(errno), errno);

		free(rctrlr);

		return NULL;

	}



	i = 0;

	rctrlr->max_sge = NVME_RDMA_MAX_SGL_DESCRIPTORS;



	while (contexts[i] != NULL) {

		rc = ibv_query_device(contexts[i], &dev_attr);

		if (rc < 0) {

			SPDK_ERRLOG("Failed to query RDMA device attributes.\n");

			rdma_free_devices(contexts);

			free(rctrlr);

			return NULL;

		}

		rctrlr->max_sge = spdk_min(rctrlr->max_sge, (uint16_t)dev_attr.max_sge);

		i++;

	}



	rdma_free_devices(contexts);



	rc = nvme_ctrlr_construct(&rctrlr->ctrlr);

	if (rc != 0) {

		free(rctrlr);
@@ -1763,14 +1791,22 @@ nvme_rdma_qpair_process_completions(struct spdk_nvme_qpair *qpair, 
uint32_t

nvme_rdma_ctrlr_get_max_xfer_size(struct spdk_nvme_ctrlr *ctrlr)

{

	/* Todo, which should get from the NVMF target */

	return NVME_RDMA_RW_BUFFER_SIZE;

	/* max_mr_size by ibv_query_device indicates the largest value that we can

	 * set for a registered memory region.  It is independent from the actual

	 * I/O size and is very likely to be larger than 2 MiB which is the

	 * granularity we currently register memory regions.  Hence return

	 * UINT32_MAX here and let the generic layer use the controller data to

	 * moderate this value.

	 */

	return UINT32_MAX;

}



uint16_t

nvme_rdma_ctrlr_get_max_sges(struct spdk_nvme_ctrlr *ctrlr)

{

	return NVME_RDMA_MAX_SGL_DESCRIPTORS;

	struct nvme_rdma_ctrlr *rctrlr = nvme_rdma_ctrlr(ctrlr);



	return rctrlr->max_sge;

}



volatile struct spdk_nvme_registers *

lib/nvme/nvme_tcp.c

+2 −1
Original line number Diff line number Diff line
@@ -1836,7 +1836,8 @@ struct spdk_nvme_ctrlr *nvme_tcp_ctrlr_construct(const struct spdk_nvme_transpor 
uint32_t

nvme_tcp_ctrlr_get_max_xfer_size(struct spdk_nvme_ctrlr *ctrlr)

{

	return NVME_TCP_RW_BUFFER_SIZE;

	/* TCP transport doens't limit maximum IO transfer size. */

	return UINT32_MAX;

}



uint16_t