Commit 350e429a authored by Seth Howell's avatar Seth Howell Committed by Ben Walker
Browse files

rdma: add a flag for disabling srq. 



There are cases where srq can be a detriment. Add a flag to allow users
to disable srq even if they have a piece of hardware that supports it.

Change-Id: Ia3be8e8c8e8463964e6ff1c02b07afbf4c3cc8f7
Signed-off-by: default avatarSeth Howell <seth.howell@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/452271


Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
parent 9d0e11d4

CHANGELOG.md

+6 −0
Original line number Diff line number Diff line
@@ -2,6 +2,12 @@ 


## v19.07: (Upcoming Release)



### NVMe-oF Target



Shared receive queue can now be disabled even for NICs that support it using the

`nvmf_create_transport` RPC method parameter `no_srq`. The actual use of a shared

receive queue is predicated on hardware support when this flag is not used.



## v19.04:



### nvme

doc/jsonrpc.md

+1 −0
Original line number Diff line number Diff line
@@ -3486,6 +3486,7 @@ max_aq_depth | Optional | number | Max number of admin cmds per 
num_shared_buffers          | Optional | number  | The number of pooled data buffers available to the transport

buf_cache_size              | Optional | number  | The number of shared buffers to reserve for each poll group

max_srq_depth               | Optional | number  | The number of elements in a per-thread shared receive queue (RDMA only)

no_srq                      | Optional | boolean | Disable shared receive queue even for devices that support it. (RDMA only)



### Example:

include/spdk/nvmf.h

+10 −9
Original line number Diff line number Diff line
@@ -73,6 +73,7 @@ struct spdk_nvmf_transport_opts { 
	uint32_t	num_shared_buffers;

	uint32_t	buf_cache_size;

	uint32_t	max_srq_depth;

	bool		no_srq;

};



/**

lib/event/subsystems/nvmf/nvmf_rpc.c

+4 −0
Original line number Diff line number Diff line
@@ -1452,6 +1452,10 @@ static const struct spdk_json_object_decoder nvmf_rpc_create_transport_decoder[] 
		"max_srq_depth", offsetof(struct nvmf_rpc_create_transport_ctx, opts.max_srq_depth),

		spdk_json_decode_uint32, true

	},

	{

		"no_srq", offsetof(struct nvmf_rpc_create_transport_ctx, opts.no_srq),

		spdk_json_decode_bool, true

	},

};



static void

lib/nvmf/rdma.c

+6 −3
Original line number Diff line number Diff line
@@ -2043,6 +2043,7 @@ spdk_nvmf_rdma_request_process(struct spdk_nvmf_rdma_transport *rtransport, 
#define SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE (SPDK_NVMF_RDMA_DEFAULT_MAX_IO_SIZE / SPDK_NVMF_MAX_SGL_ENTRIES)

#define SPDK_NVMF_RDMA_DEFAULT_NUM_SHARED_BUFFERS 4095

#define SPDK_NVMF_RDMA_DEFAULT_BUFFER_CACHE_SIZE 32

#define SPDK_NVMF_RDMA_DEFAULT_NO_SRQ false;



static void

spdk_nvmf_rdma_opts_init(struct spdk_nvmf_transport_opts *opts)
@@ -2056,6 +2057,7 @@ spdk_nvmf_rdma_opts_init(struct spdk_nvmf_transport_opts *opts) 
	opts->num_shared_buffers =	SPDK_NVMF_RDMA_DEFAULT_NUM_SHARED_BUFFERS;

	opts->buf_cache_size =		SPDK_NVMF_RDMA_DEFAULT_BUFFER_CACHE_SIZE;

	opts->max_srq_depth =		SPDK_NVMF_RDMA_DEFAULT_SRQ_DEPTH;

	opts->no_srq =			SPDK_NVMF_RDMA_DEFAULT_NO_SRQ

}



const struct spdk_mem_map_ops g_nvmf_rdma_map_ops = {
@@ -2099,7 +2101,7 @@ spdk_nvmf_rdma_create(struct spdk_nvmf_transport_opts *opts) 
		     "  Transport opts:  max_ioq_depth=%d, max_io_size=%d,\n"

		     "  max_qpairs_per_ctrlr=%d, io_unit_size=%d,\n"

		     "  in_capsule_data_size=%d, max_aq_depth=%d,\n"

		     "  num_shared_buffers=%d, max_srq_depth=%d\n",

		     "  num_shared_buffers=%d, max_srq_depth=%d, no_srq=%d\n",

		     opts->max_queue_depth,

		     opts->max_io_size,

		     opts->max_qpairs_per_ctrlr,
@@ -2107,7 +2109,8 @@ spdk_nvmf_rdma_create(struct spdk_nvmf_transport_opts *opts) 
		     opts->in_capsule_data_size,

		     opts->max_aq_depth,

		     opts->num_shared_buffers,

		     opts->max_srq_depth);

		     opts->max_srq_depth,

		     opts->no_srq);



	/* I/O unit size cannot be larger than max I/O size */

	if (opts->io_unit_size > opts->max_io_size) {
@@ -2936,7 +2939,7 @@ spdk_nvmf_rdma_poll_group_create(struct spdk_nvmf_transport *transport) 
		TAILQ_INIT(&poller->qpairs);



		TAILQ_INSERT_TAIL(&rgroup->pollers, poller, link);

		if (device->attr.max_srq != 0) {

		if (transport->opts.no_srq == false && device->attr.max_srq != 0) {

			poller->max_srq_depth = transport->opts.max_srq_depth;



			memset(&srq_init_attr, 0, sizeof(struct ibv_srq_init_attr));