Commit 0e983c56 authored by Krzysztof Goreczny's avatar Krzysztof Goreczny Committed by Tomasz Zawadzki
Browse files

nvmf/tcp: use sock group polling for the listening sockets 



Sock groups are more efficient in polling when number of sockets becomes
high. This patch laverages that for the TCP listening ports.

Change-Id: I33231fdc10f76c0f9b3ee6187240e2c83b0d02af
Signed-off-by: default avatarKrzysztof Goreczny <krzysztof.goreczny@dell.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/23213


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <jim.harris@samsung.com>
Community-CI: Mellanox Build Bot
Reviewed-by: default avatarBoris Glimcher <Boris.Glimcher@emc.com>
Reviewed-by: default avatarKonrad Sztyber <konrad.sztyber@intel.com>
parent cff94374

lib/nvmf/tcp.c

+36 −5
Original line number Diff line number Diff line
@@ -367,6 +367,7 @@ struct spdk_nvmf_tcp_transport { 
	struct spdk_nvmf_tcp_poll_group		*next_pg;



	struct spdk_poller			*accept_poller;

	struct spdk_sock_group			*listen_sock_group;



	TAILQ_HEAD(, spdk_nvmf_tcp_port)	ports;

	TAILQ_HEAD(, spdk_nvmf_tcp_poll_group)	poll_groups;
@@ -630,6 +631,7 @@ nvmf_tcp_destroy(struct spdk_nvmf_transport *transport, 
	}



	spdk_poller_unregister(&ttransport->accept_poller);

	spdk_sock_group_close(&ttransport->listen_sock_group);

	free(ttransport);



	if (cb_fn) {
@@ -640,6 +642,8 @@ nvmf_tcp_destroy(struct spdk_nvmf_transport *transport, 


static int nvmf_tcp_accept(void *ctx);



static void nvmf_tcp_accept_cb(void *ctx, struct spdk_sock_group *group, struct spdk_sock *sock);



static struct spdk_nvmf_transport *

nvmf_tcp_create(struct spdk_nvmf_transport_opts *opts)

{
@@ -768,6 +772,14 @@ nvmf_tcp_create(struct spdk_nvmf_transport_opts *opts) 
		return NULL;

	}



	ttransport->listen_sock_group = spdk_sock_group_create(NULL);

	if (ttransport->listen_sock_group == NULL) {

		SPDK_ERRLOG("Failed to create socket group for listen sockets\n");

		spdk_poller_unregister(&ttransport->accept_poller);

		free(ttransport);

		return NULL;

	}



	return &ttransport->transport;

}
@@ -896,6 +908,7 @@ nvmf_tcp_listen(struct spdk_nvmf_transport *transport, const struct spdk_nvme_tr 
	struct spdk_sock_impl_opts impl_opts;

	size_t impl_opts_size = sizeof(impl_opts);

	struct spdk_sock_opts opts;

	int rc;



	if (!strlen(trid->trsvcid)) {

		SPDK_ERRLOG("Service id is required\n");
@@ -979,6 +992,15 @@ nvmf_tcp_listen(struct spdk_nvmf_transport *transport, const struct spdk_nvme_tr 
		return -EINVAL;

	}



	rc = spdk_sock_group_add_sock(ttransport->listen_sock_group, port->listen_sock, nvmf_tcp_accept_cb,

				      port);

	if (rc < 0) {

		SPDK_ERRLOG("Failed to add socket to the listen socket group\n");

		spdk_sock_close(&port->listen_sock);

		free(port);

		return -errno;

	}



	port->transport = transport;



	SPDK_NOTICELOG("*** NVMe/TCP Target Listening on %s port %s ***\n",
@@ -1002,6 +1024,7 @@ nvmf_tcp_stop_listen(struct spdk_nvmf_transport *transport, 


	port = nvmf_tcp_find_port(ttransport, trid);

	if (port) {

		spdk_sock_group_remove_sock(ttransport->listen_sock_group, port->listen_sock);

		TAILQ_REMOVE(&ttransport->ports, port, link);

		spdk_sock_close(&port->listen_sock);

		free(port);
@@ -1402,16 +1425,24 @@ nvmf_tcp_accept(void *ctx) 
{

	struct spdk_nvmf_transport *transport = ctx;

	struct spdk_nvmf_tcp_transport *ttransport;

	struct spdk_nvmf_tcp_port *port;

	uint32_t count = 0;

	int count;



	ttransport = SPDK_CONTAINEROF(transport, struct spdk_nvmf_tcp_transport, transport);



	TAILQ_FOREACH(port, &ttransport->ports, link) {

		count += nvmf_tcp_port_accept(port);

	count = spdk_sock_group_poll(ttransport->listen_sock_group);

	if (count < 0) {

		SPDK_ERRLOG("Fail in TCP listen socket group poll\n");

	}



	return count > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE;

	return count != 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE;

}



static void

nvmf_tcp_accept_cb(void *ctx, struct spdk_sock_group *group, struct spdk_sock *sock)

{

	struct spdk_nvmf_tcp_port *port = ctx;



	nvmf_tcp_port_accept(port);

}



static void

test/unit/lib/nvmf/tcp.c/tcp_ut.c

+13 −0
Original line number Diff line number Diff line
@@ -380,11 +380,14 @@ test_nvmf_tcp_create(void) 
	struct spdk_nvmf_transport *transport;

	struct spdk_nvmf_tcp_transport *ttransport;

	struct spdk_nvmf_transport_opts opts;

	struct spdk_sock_group grp = {};



	thread = spdk_thread_create(NULL, NULL);

	SPDK_CU_ASSERT_FATAL(thread != NULL);

	spdk_set_thread(thread);



	MOCK_SET(spdk_sock_group_create, &grp);



	/* case 1 */

	memset(&opts, 0, sizeof(opts));

	opts.max_queue_depth = UT_MAX_QUEUE_DEPTH;
@@ -441,6 +444,8 @@ test_nvmf_tcp_create(void) 
	transport = nvmf_tcp_create(&opts);

	CU_ASSERT_PTR_NULL(transport);



	MOCK_CLEAR_P(spdk_sock_group_create);



	spdk_thread_exit(thread);

	while (!spdk_thread_is_exited(thread)) {

		spdk_thread_poll(thread, 0, 0);
@@ -454,6 +459,7 @@ test_nvmf_tcp_destroy(void) 
	struct spdk_thread *thread;

	struct spdk_nvmf_transport *transport;

	struct spdk_nvmf_transport_opts opts;

	struct spdk_sock_group grp = {};



	thread = spdk_thread_create(NULL, NULL);

	SPDK_CU_ASSERT_FATAL(thread != NULL);
@@ -468,7 +474,9 @@ test_nvmf_tcp_destroy(void) 
	opts.io_unit_size = UT_IO_UNIT_SIZE;

	opts.max_aq_depth = UT_MAX_AQ_DEPTH;

	opts.num_shared_buffers = UT_NUM_SHARED_BUFFERS;

	MOCK_SET(spdk_sock_group_create, &grp);

	transport = nvmf_tcp_create(&opts);

	MOCK_CLEAR_P(spdk_sock_group_create);

	CU_ASSERT_PTR_NOT_NULL(transport);

	transport->opts = opts;

	/* destroy transport */
@@ -518,7 +526,9 @@ test_nvmf_tcp_poll_group_create(void) 
	opts.io_unit_size = UT_IO_UNIT_SIZE;

	opts.max_aq_depth = UT_MAX_AQ_DEPTH;

	opts.num_shared_buffers = UT_NUM_SHARED_BUFFERS;

	MOCK_SET(spdk_sock_group_create, &grp);

	transport = nvmf_tcp_create(&opts);

	MOCK_CLEAR_P(spdk_sock_group_create);

	CU_ASSERT_PTR_NOT_NULL(transport);

	transport->opts = opts;

	MOCK_SET(spdk_sock_group_create, &grp);
@@ -1287,6 +1297,7 @@ test_nvmf_tcp_tls_add_remove_credentials(void) 
	struct spdk_nvmf_transport_opts opts;

	struct spdk_nvmf_subsystem subsystem;

	struct tcp_psk_entry *entry;

	struct spdk_sock_group grp = {};

	const char subnqn[] = {"nqn.2016-06.io.spdk:cnode1"};

	const char hostnqn[] = {"nqn.2016-06.io.spdk:host1"};

	const char *psk = "NVMeTLSkey-1:01:VRLbtnN9AQb2WXW3c9+wEf/DRLz0QuLdbYvEhwtdWwNf9LrZ:";
@@ -1307,7 +1318,9 @@ test_nvmf_tcp_tls_add_remove_credentials(void) 
	opts.io_unit_size = UT_IO_UNIT_SIZE;

	opts.max_aq_depth = UT_MAX_AQ_DEPTH;

	opts.num_shared_buffers = UT_NUM_SHARED_BUFFERS;

	MOCK_SET(spdk_sock_group_create, &grp);

	transport = nvmf_tcp_create(&opts);

	MOCK_CLEAR_P(spdk_sock_group_create);



	memset(&subsystem, 0, sizeof(subsystem));

	snprintf(subsystem.subnqn, sizeof(subsystem.subnqn), "%s", subnqn);