Commit de5f1ea0 authored by Shuhei Matsumoto's avatar Shuhei Matsumoto Committed by Tomasz Zawadzki
Browse files

lib/iscsi: Create and terminate poll group threads explicitly 



Login acceptor still runs on one of the default reactor threads,
but we move iSCSI poll group from the default reactor threads to
dedicated threads created at startup.

At startup, use reference count to detect completion and
move to the next step. By returning completion message to the init
thread, we can avoid using any atomic operation.

At shutdown, we can use spdk_for_each_channel() conveniently. Put
voluntary spdk_thread_exit() calls into the callback to
spdk_put_io_channel().

Moving login acceptor to a dedicated thread is another task.

To maintain the original behavior, number of threads created is
the number of cores that SPDK app uses.

Change-Id: Ifd1de9343ac0183254ca608d1fd8faa94acc254e
Signed-off-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Signed-off-by: default avatarVitaliy Mysak <vitaliy.mysak@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/492


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
parent 7d26adb6

lib/iscsi/iscsi.h

+1 −0
Original line number Original line Diff line number Diff line
@@ -342,6 +342,7 @@ struct spdk_iscsi_globals { 
	char *authfile;

	char *authfile;

	char *nodebase;

	char *nodebase;

	pthread_mutex_t mutex;

	pthread_mutex_t mutex;

	uint32_t refcnt;

	TAILQ_HEAD(, spdk_iscsi_portal)		portal_head;

	TAILQ_HEAD(, spdk_iscsi_portal)		portal_head;

	TAILQ_HEAD(, spdk_iscsi_portal_grp)	pg_head;

	TAILQ_HEAD(, spdk_iscsi_portal_grp)	pg_head;

	TAILQ_HEAD(, spdk_iscsi_init_grp)	ig_head;

	TAILQ_HEAD(, spdk_iscsi_init_grp)	ig_head;

lib/iscsi/iscsi_subsystem.c

+46 −6
Original line number Original line Diff line number Diff line
@@ -50,6 +50,7 @@ 




struct spdk_iscsi_opts *g_spdk_iscsi_opts = NULL;

struct spdk_iscsi_opts *g_spdk_iscsi_opts = NULL;





static struct spdk_thread *g_init_thread = NULL;

static spdk_iscsi_init_cb g_init_cb_fn = NULL;

static spdk_iscsi_init_cb g_init_cb_fn = NULL;

static void *g_init_cb_arg = NULL;

static void *g_init_cb_arg = NULL;
@@ -1143,7 +1144,7 @@ iscsi_init_complete(int rc) 
}

}





static void

static void

iscsi_parse_configuration(void *ctx)

iscsi_parse_configuration(void)

{

{

	int rc;

	int rc;
@@ -1238,6 +1239,8 @@ static void 
iscsi_poll_group_destroy(void *io_device, void *ctx_buf)

iscsi_poll_group_destroy(void *io_device, void *ctx_buf)

{

{

	struct spdk_iscsi_poll_group *pg = ctx_buf;

	struct spdk_iscsi_poll_group *pg = ctx_buf;

	struct spdk_io_channel *ch;

	struct spdk_thread *thread;





	assert(pg->poller != NULL);

	assert(pg->poller != NULL);

	assert(pg->sock_group != NULL);

	assert(pg->sock_group != NULL);
@@ -1245,6 +1248,24 @@ iscsi_poll_group_destroy(void *io_device, void *ctx_buf) 
	spdk_sock_group_close(&pg->sock_group);

	spdk_sock_group_close(&pg->sock_group);

	spdk_poller_unregister(&pg->poller);

	spdk_poller_unregister(&pg->poller);

	spdk_poller_unregister(&pg->nop_poller);

	spdk_poller_unregister(&pg->nop_poller);



	ch = spdk_io_channel_from_ctx(pg);

	thread = spdk_io_channel_get_thread(ch);



	assert(thread == spdk_get_thread());



	spdk_thread_exit(thread);

}



static void

_iscsi_init_thread_done(void *ctx)

{

	struct spdk_iscsi_poll_group *pg = ctx;



	TAILQ_INSERT_TAIL(&g_spdk_iscsi.poll_group_head, pg, link);

	if (--g_spdk_iscsi.refcnt == 0) {

		iscsi_parse_configuration();

	}

}

}





static void

static void
@@ -1256,19 +1277,38 @@ _iscsi_init_thread(void *ctx) 
	ch = spdk_get_io_channel(&g_spdk_iscsi);

	ch = spdk_get_io_channel(&g_spdk_iscsi);

	pg = spdk_io_channel_get_ctx(ch);

	pg = spdk_io_channel_get_ctx(ch);





	pthread_mutex_lock(&g_spdk_iscsi.mutex);

	spdk_thread_send_msg(g_init_thread, _iscsi_init_thread_done, pg);

	TAILQ_INSERT_TAIL(&g_spdk_iscsi.poll_group_head, pg, link);

	pthread_mutex_unlock(&g_spdk_iscsi.mutex);

}

}





static void

static void

initialize_iscsi_poll_group(void)

initialize_iscsi_poll_group(void)

{

{

	struct spdk_cpuset tmp_cpumask = {};

	uint32_t i;

	char thread_name[32];

	struct spdk_thread *thread;



	spdk_io_device_register(&g_spdk_iscsi, iscsi_poll_group_create, iscsi_poll_group_destroy,

	spdk_io_device_register(&g_spdk_iscsi, iscsi_poll_group_create, iscsi_poll_group_destroy,

				sizeof(struct spdk_iscsi_poll_group), "iscsi_tgt");

				sizeof(struct spdk_iscsi_poll_group), "iscsi_tgt");





	/* Send a message to each thread and create a poll group */

	/* Create threads for CPU cores active for this application, and send a

	spdk_for_each_thread(_iscsi_init_thread, NULL, iscsi_parse_configuration);

	 * message to each thread to create a poll group on it.

	 */

	g_init_thread = spdk_get_thread();

	assert(g_init_thread != NULL);

	assert(g_spdk_iscsi.refcnt == 0);



	SPDK_ENV_FOREACH_CORE(i) {

		spdk_cpuset_zero(&tmp_cpumask);

		spdk_cpuset_set_cpu(&tmp_cpumask, i, true);

		snprintf(thread_name, sizeof(thread_name), "iscsi_poll_group_%u", i);



		thread = spdk_thread_create(thread_name, &tmp_cpumask);

		assert(thread != NULL);



		g_spdk_iscsi.refcnt++;

		spdk_thread_send_msg(thread, _iscsi_init_thread, NULL);

	}

}

}





static int

static int