Commit 6ad6a113 authored by Ziye Yang's avatar Ziye Yang Committed by Changpeng Liu
Browse files

nvmf/tcp: Add a feature to allow set the sock priority of the connection. 



This priority is used to differentiate the sock priority on the TCP connections
between  NVMe-oF TCP target and other TCP based applications.

Signed-off-by: default avatarZiye Yang <ziye.yang@intel.com>
Change-Id: I6ee294e647420b56d1d91a07c2e37bf34ce24e03
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/461801


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarChangpeng Liu <changpeng.liu@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
parent 96b75929

CHANGELOG.md

+5 −0
Original line number Diff line number Diff line
@@ -35,6 +35,11 @@ for reads is turned on by default. A config knob was added to allow disabling 
the optimization. This will mostly be used for integration testing with 5.0.x kernels

while some compatibility fixes make their way down the pipeline for 5.1.x kernels.



The sock priority setting of the TCP connection owned by the tcp transport is added. It is

used to optimize the TCP connection performance under designated traffic classes. And the

priority is used to differeniate the sock priority between SPDK NVMe-oF TCP target application

and other TCP based applications.



Shared receive queue can now be disabled even for NICs that support it using the

`nvmf_create_transport` RPC method parameter `no_srq`. The actual use of a shared

receive queue is predicated on hardware support when this flag is not used.

doc/jsonrpc.md

+1 −0
Original line number Diff line number Diff line
@@ -3616,6 +3616,7 @@ max_srq_depth | Optional | number | The number of elements in a p 
no_srq                      | Optional | boolean | Disable shared receive queue even for devices that support it. (RDMA only)

c2h_success                 | Optional | boolean | Disable C2H success optimization (TCP only)

dif_insert_or_strip         | Optional | boolean | Enable DIF insert for write I/O and DIF strip for read I/O DIF (TCP only)

sock_priority               | Optional | number  | The socket priority of the connection owned by this transport (TCP only)



### Example:

include/spdk/nvmf.h

+1 −0
Original line number Diff line number Diff line
@@ -76,6 +76,7 @@ struct spdk_nvmf_transport_opts { 
	bool		no_srq;

	bool		c2h_success;

	bool		dif_insert_or_strip;

	uint32_t	sock_priority;

};



struct spdk_nvmf_poll_group_stat {

lib/event/subsystems/nvmf/nvmf_rpc.c

+5 −0
Original line number Diff line number Diff line
@@ -1458,6 +1458,10 @@ static const struct spdk_json_object_decoder nvmf_rpc_create_transport_decoder[] 
		"dif_insert_or_strip", offsetof(struct nvmf_rpc_create_transport_ctx, opts.dif_insert_or_strip),

		spdk_json_decode_bool, true

	},

	{

		"sock_priority", offsetof(struct nvmf_rpc_create_transport_ctx, opts.sock_priority),

		spdk_json_decode_uint32, true

	},

};



static void
@@ -1597,6 +1601,7 @@ dump_nvmf_transport(struct spdk_json_write_ctx *w, struct spdk_nvmf_transport *t 
	} else if (type == SPDK_NVME_TRANSPORT_TCP) {

		spdk_json_write_named_bool(w, "c2h_success", opts->c2h_success);

		spdk_json_write_named_bool(w, "dif_insert_or_strip", opts->dif_insert_or_strip);

		spdk_json_write_named_uint32(w, "sock_priority", opts->sock_priority);

	}



	spdk_json_write_object_end(w);

lib/nvmf/tcp.c

+23 −2
Original line number Diff line number Diff line
@@ -54,6 +54,7 @@ 
#define NVMF_TCP_PDU_MAX_H2C_DATA_SIZE	131072

#define NVMF_TCP_PDU_MAX_C2H_DATA_SIZE	131072

#define NVMF_TCP_QPAIR_MAX_C2H_PDU_NUM  64  /* Maximal c2h_data pdu number for ecah tqpair */

#define SPDK_NVMF_TCP_DEFAULT_MAX_SOCK_PRIORITY 6



/* spdk nvmf related structure */

enum spdk_nvmf_tcp_req_state {
@@ -540,7 +541,7 @@ spdk_nvmf_tcp_create(struct spdk_nvmf_transport_opts *opts) 
		     "  max_qpairs_per_ctrlr=%d, io_unit_size=%d,\n"

		     "  in_capsule_data_size=%d, max_aq_depth=%d\n"

		     "  num_shared_buffers=%d, c2h_success=%d,\n"

		     "  dif_insert_or_strip=%d\n",

		     "  dif_insert_or_strip=%d, sock_priority=%d\n",

		     opts->max_queue_depth,

		     opts->max_io_size,

		     opts->max_qpairs_per_ctrlr,
@@ -549,7 +550,16 @@ spdk_nvmf_tcp_create(struct spdk_nvmf_transport_opts *opts) 
		     opts->max_aq_depth,

		     opts->num_shared_buffers,

		     opts->c2h_success,

		     opts->dif_insert_or_strip);

		     opts->dif_insert_or_strip,

		     opts->sock_priority);



	if (opts->sock_priority > SPDK_NVMF_TCP_DEFAULT_MAX_SOCK_PRIORITY) {

		SPDK_ERRLOG("Unsupported socket_priority=%d, the current range is: 0 to %d\n"

			    "you can use man 7 socket to view the range of priority under SO_PRIORITY item\n",

			    opts->sock_priority, SPDK_NVMF_TCP_DEFAULT_MAX_SOCK_PRIORITY);

		free(ttransport);

		return NULL;

	}



	/* I/O unit size cannot be larger than max I/O size */

	if (opts->io_unit_size > opts->max_io_size) {
@@ -1096,6 +1106,15 @@ _spdk_nvmf_tcp_handle_connect(struct spdk_nvmf_transport *transport, 
	SPDK_DEBUGLOG(SPDK_LOG_NVMF_TCP, "New connection accepted on %s port %s\n",

		      port->trid.traddr, port->trid.trsvcid);



	if (transport->opts.sock_priority) {

		rc = spdk_sock_set_priority(sock, transport->opts.sock_priority);

		if (rc) {

			SPDK_ERRLOG("Failed to set the priority of the socket\n");

			spdk_sock_close(&sock);

			return;

		}

	}



	tqpair = calloc(1, sizeof(struct spdk_nvmf_tcp_qpair));

	if (tqpair == NULL) {

		SPDK_ERRLOG("Could not allocate new connection.\n");
@@ -2866,6 +2885,7 @@ spdk_nvmf_tcp_qpair_set_sq_size(struct spdk_nvmf_qpair *qpair) 
#define SPDK_NVMF_TCP_DEFAULT_BUFFER_CACHE_SIZE 32

#define SPDK_NVMF_TCP_DEFAULT_SUCCESS_OPTIMIZATION true

#define SPDK_NVMF_TCP_DEFAULT_DIF_INSERT_OR_STRIP false

#define SPDK_NVMF_TCP_DEFAULT_SOCK_PRIORITY 0



static void

spdk_nvmf_tcp_opts_init(struct spdk_nvmf_transport_opts *opts)
@@ -2880,6 +2900,7 @@ spdk_nvmf_tcp_opts_init(struct spdk_nvmf_transport_opts *opts) 
	opts->buf_cache_size =		SPDK_NVMF_TCP_DEFAULT_BUFFER_CACHE_SIZE;

	opts->c2h_success =		SPDK_NVMF_TCP_DEFAULT_SUCCESS_OPTIMIZATION;

	opts->dif_insert_or_strip =	SPDK_NVMF_TCP_DEFAULT_DIF_INSERT_OR_STRIP;

	opts->sock_priority =		SPDK_NVMF_TCP_DEFAULT_SOCK_PRIORITY;

}



const struct spdk_nvmf_transport_ops spdk_nvmf_transport_tcp = {