Commit 1d9be84b authored by Seth Howell's avatar Seth Howell Committed by Changpeng Liu
Browse files

nvmf/rdma: change the default buffer size. 



Having the buffers be the same size as the maximum xfer size doesn't do
us any favors. Make these buffers a ratio of the maximum transfer size
and the number of supported nvmf SGLs.

Also configure the number of nvmf request iovs to correspond with this
new ratio.

Change-Id: I3147dcd86b599c74521ebfdf3bcdbcdee8871a3a
Signed-off-by: default avatarSeth Howell <seth.howell@intel.com>
Reviewed-on: https://review.gerrithub.io/428747


Chandler-Test-Pool: SPDK Automated Test System <sys_sgsw@intel.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarChangpeng Liu <changpeng.liu@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
parent 6134d778

lib/nvmf/nvmf_internal.h

+1 −1
Original line number Diff line number Diff line
@@ -151,7 +151,7 @@ struct spdk_nvmf_request { 
	void				*data;

	union nvmf_h2c_msg		*cmd;

	union nvmf_c2h_msg		*rsp;

	struct iovec			iov[SPDK_NVMF_MAX_SGL_ENTRIES];

	struct iovec			iov[SPDK_NVMF_MAX_SGL_ENTRIES * 2];

	uint32_t			iovcnt;

	struct spdk_bdev_io_wait_entry	bdev_io_wait;

lib/nvmf/rdma.c

+9 −6
Original line number Diff line number Diff line
@@ -1499,7 +1499,8 @@ spdk_nvmf_rdma_request_process(struct spdk_nvmf_rdma_transport *rtransport, 
#define SPDK_NVMF_RDMA_DEFAULT_MAX_QPAIRS_PER_CTRLR 64

#define SPDK_NVMF_RDMA_DEFAULT_IN_CAPSULE_DATA_SIZE 4096

#define SPDK_NVMF_RDMA_DEFAULT_MAX_IO_SIZE 131072

#define SPDK_NVMF_RDMA_DEFAULT_IO_BUFFER_SIZE 131072

#define SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE 4096

#define SPDK_NVMF_RDMA_DEFAULT_IO_BUFFER_SIZE (SPDK_NVMF_RDMA_DEFAULT_MAX_IO_SIZE / SPDK_NVMF_MAX_SGL_ENTRIES)



static void

spdk_nvmf_rdma_opts_init(struct spdk_nvmf_transport_opts *opts)
@@ -1508,7 +1509,8 @@ spdk_nvmf_rdma_opts_init(struct spdk_nvmf_transport_opts *opts) 
	opts->max_qpairs_per_ctrlr = SPDK_NVMF_RDMA_DEFAULT_MAX_QPAIRS_PER_CTRLR;

	opts->in_capsule_data_size = SPDK_NVMF_RDMA_DEFAULT_IN_CAPSULE_DATA_SIZE;

	opts->max_io_size =          SPDK_NVMF_RDMA_DEFAULT_MAX_IO_SIZE;

	opts->io_unit_size =         SPDK_NVMF_RDMA_DEFAULT_IO_BUFFER_SIZE;

	opts->io_unit_size =         spdk_max(SPDK_NVMF_RDMA_DEFAULT_IO_BUFFER_SIZE,

					      SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE);

	opts->max_aq_depth =         SPDK_NVMF_RDMA_DEFAULT_AQ_DEPTH;

}
@@ -1589,9 +1591,10 @@ spdk_nvmf_rdma_create(struct spdk_nvmf_transport_opts *opts) 
		return NULL;

	}



	/* The maximum number of buffers we will need for a given request is equal to just less than double the number of SGL elements */

	rtransport->data_buf_pool = spdk_mempool_create("spdk_nvmf_rdma",

				    opts->max_queue_depth * 4, /* The 4 is arbitrarily chosen. Needs to be configurable. */

				    opts->max_io_size + NVMF_DATA_BUFFER_ALIGNMENT,

				    opts->max_queue_depth * (SPDK_NVMF_MAX_SGL_ENTRIES * 2) * 4,

				    opts->io_unit_size + NVMF_DATA_BUFFER_ALIGNMENT,

				    SPDK_MEMPOOL_DEFAULT_CACHE_SIZE,

				    SPDK_ENV_SOCKET_ID_ANY);

	if (!rtransport->data_buf_pool) {
@@ -1740,10 +1743,10 @@ spdk_nvmf_rdma_destroy(struct spdk_nvmf_transport *transport) 


	if (rtransport->data_buf_pool != NULL) {

		if (spdk_mempool_count(rtransport->data_buf_pool) !=

		    (transport->opts.max_queue_depth * 4)) {

		    (transport->opts.max_queue_depth * (SPDK_NVMF_MAX_SGL_ENTRIES * 2) * 4)) {

			SPDK_ERRLOG("transport buffer pool count is %zu but should be %u\n",

				    spdk_mempool_count(rtransport->data_buf_pool),

				    transport->opts.max_queue_depth * 4);

				    transport->opts.max_queue_depth * (SPDK_NVMF_MAX_SGL_ENTRIES * 2) * 4);

		}

	}