Commit 750a4213 authored by Ziye Yang's avatar Ziye Yang Committed by Changpeng Liu
Browse files

nvmf: add spdk_nvmf_get_optimal_poll_group 



This patch is used to do the following work:

1 It is optimized for NVMe/TCP transport. If the qpair's
socket has same NAPI_ID, then the qpair will be handled
by the same polling group.

2. We add a new connection scheduling strategy, named as
ConnectionScheduler in the configuration file. It will be
used to input different scheduler according to the customers'
input.

Signed-off-by: default avatarZiye Yang <ziye.yang@intel.com>
Change-Id: Ifc9246eece0da69bdd39fd63bfdefff18be64132
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/454550


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: default avatarChangpeng Liu <changpeng.liu@intel.com>
parent 960460f0

CHANGELOG.md

+5 −0
Original line number Diff line number Diff line
@@ -25,6 +25,11 @@ Shared receive queue can now be disabled even for NICs that support it using the 
`nvmf_create_transport` RPC method parameter `no_srq`. The actual use of a shared

receive queue is predicated on hardware support when this flag is not used.



spdk_nvmf_get_optimal_poll_group was added, which is used to return the optimal

poll group for the qpair. And `ConnectionScheduler` configuration is added into the

[Nvmf] section in etc/spdk/nvmf.conf.in to demonstrate how to configure the connection

scheduling strategy among different spdk threads.



### notify



The function `spdk_notify_get_types()` and `spdk_notify_get_events()` were

etc/spdk/nvmf.conf.in

+9 −0
Original line number Diff line number Diff line
@@ -65,6 +65,15 @@ 
  # poll. Units in microseconds.

  AcceptorPollRate 10000



  # Set how the connection is scheduled among multiple threads, current supported string value are

  # "RoundRobin", "Host", "Transport".

  # RoundRobin: Schedule the connection with roundrobin manner.

  # Host: Schedule the connection according to host IP.

  # Transport: Schedule the connection according to the transport characteristics.

  #  For example, for  TCP transport, we can schedule the connection according to socket NAPI_ID info.

  #  The connection which has the same socket NAPI_ID info will be grouped in the same polling group.

  ConnectionScheduler RoundRobin



# One valid transport type must be set in each [Transport].

# The first is the case of RDMA transport and the second is the case of TCP transport.

[Transport]

include/spdk/nvmf.h

+9 −0
Original line number Diff line number Diff line
@@ -163,6 +163,15 @@ void spdk_nvmf_tgt_accept(struct spdk_nvmf_tgt *tgt, new_qpair_fn cb_fn); 
 */

struct spdk_nvmf_poll_group *spdk_nvmf_poll_group_create(struct spdk_nvmf_tgt *tgt);



/**

 * Get optimal nvmf poll group for the qpair.

 *

 * \param qpair Requested qpair

 *

 * \return a poll group on success, or NULL on failure.

 */

struct spdk_nvmf_poll_group *spdk_nvmf_get_optimal_poll_group(struct spdk_nvmf_qpair *qpair);



/**

 * Destroy a poll group.

 *

lib/event/subsystems/nvmf/conf.c

+33 −2
Original line number Diff line number Diff line
@@ -101,16 +101,42 @@ spdk_nvmf_read_config_file_tgt_max_subsystems(struct spdk_conf_section *sp, 
	}

}



static void

static int

spdk_nvmf_read_config_file_tgt_conf(struct spdk_conf_section *sp,

				    struct spdk_nvmf_tgt_conf *conf)

{

	int acceptor_poll_rate;

	const char *conn_scheduler;

	int rc = 0;



	acceptor_poll_rate = spdk_conf_section_get_intval(sp, "AcceptorPollRate");

	if (acceptor_poll_rate >= 0) {

		conf->acceptor_poll_rate = acceptor_poll_rate;

	}



	conn_scheduler = spdk_conf_section_get_val(sp, "ConnectionScheduler");



	if (conn_scheduler) {

		if (strcasecmp(conn_scheduler, "RoundRobin") == 0) {

			conf->conn_sched = CONNECT_SCHED_ROUND_ROBIN;

		} else if (strcasecmp(conn_scheduler, "Host") == 0) {

			conf->conn_sched = CONNECT_SCHED_HOST_IP;

		} else if (strcasecmp(conn_scheduler, "Transport") == 0) {

			conf->conn_sched = CONNECT_SCHED_TRANSPORT_OPTIMAL_GROUP;

		} else {

			SPDK_ERRLOG("The valid value of ConnectionScheduler should be:\n"

				    "\t RoundRobin\n"

				    "\t Host\n"

				    "\t Transport\n");

			rc = -1;

		}



	} else {

		SPDK_NOTICELOG("The value of ConnectionScheduler is not configured,\n"

			       "we will use RoundRobin as the default scheduler\n");

	}



	return rc;

}



static int
@@ -132,6 +158,7 @@ spdk_nvmf_parse_tgt_conf(void) 
{

	struct spdk_nvmf_tgt_conf *conf;

	struct spdk_conf_section *sp;

	int rc;



	conf = calloc(1, sizeof(*conf));

	if (!conf) {
@@ -144,7 +171,11 @@ spdk_nvmf_parse_tgt_conf(void) 


	sp = spdk_conf_find_section(NULL, "Nvmf");

	if (sp != NULL) {

		spdk_nvmf_read_config_file_tgt_conf(sp, conf);

		rc = spdk_nvmf_read_config_file_tgt_conf(sp, conf);

		if (rc) {

			free(conf);

			return NULL;

		}

	}



	return conf;

lib/event/subsystems/nvmf/event_nvmf.h

+1 −0
Original line number Diff line number Diff line
@@ -48,6 +48,7 @@ 
enum spdk_nvmf_connect_sched {

	CONNECT_SCHED_ROUND_ROBIN = 0,

	CONNECT_SCHED_HOST_IP,

	CONNECT_SCHED_TRANSPORT_OPTIMAL_GROUP,

};



struct spdk_nvmf_tgt_conf {