Commit 4e236535 authored by Shuhei Matsumoto's avatar Shuhei Matsumoto Committed by Tomasz Zawadzki
Browse files

nvmf_tgt: Detect completion of poll group destroy at shutdown 



Previously NVMe-oF target had requested each thread to destroy
its poll group, but had moved to the next state without waiting
for completion.

To create and destroy poll group threads dynamically, NVMe-oF
target have to know the completion of poll group destroy.

The reason is as follows.

spdk_thread_exit() requires that each poll group thread to unregister
its pollers, but poller unregistration is done asynchronously, and
releaes its I/O channels, but I/O channel release is done
asynchronously.

This patch does the following.

To avoid using mutex, add the init thread.

spdk_for_each_thread() requires completion callback but the
completion callback is usable only if each message handler is
synchronous. spdk_nvmf_poll_group_destroy() is asynchronous now.
So replace spdk_for_each_thread() by parsing g_poll_groups and
sending message to each corresonding thread.

Then add comletion callback nvmf_tgt_destroy_poll_group_done() to
spdk_nvmf_poll_group_destroy().

nvmf_tgt_destroy_poll_group_done() sends message to the init
thread.

The init thread executes _nvmf_tgt_destroy_poll_group_done().
_nvmf_tgt_destroy_poll_group_done() decrements g_num_poll_groups
and move to the next state if it becomes zero.

Signed-off-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Change-Id: I97eb90b9dbff29d7702b20cf7ac233d39d7216b6
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/496


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarAleksey Marchuk <alexeymar@mellanox.com>
parent 2fa51eeb

module/event/subsystems/nvmf/nvmf_tgt.c

+32 −17
Original line number Diff line number Diff line
@@ -77,6 +77,8 @@ struct spdk_nvmf_tgt *g_spdk_nvmf_tgt = NULL; 


static enum nvmf_tgt_state g_tgt_state;



static struct spdk_thread *g_tgt_fini_thread = NULL;



/* Round-Robin/IP-based tracking of threads to poll group assignment */

static struct nvmf_tgt_poll_group *g_next_poll_group = NULL;
@@ -306,30 +308,45 @@ acceptor_poll(void *arg) 
}



static void

nvmf_tgt_destroy_poll_group_done(void *ctx)

_nvmf_tgt_destroy_poll_group_done(void *ctx)

{

	assert(g_num_poll_groups > 0);



	if (--g_num_poll_groups == 0) {

		g_tgt_state = NVMF_TGT_FINI_STOP_ACCEPTOR;

	assert(g_num_poll_groups == 0);

		nvmf_tgt_advance_state();

	}

}



static void

nvmf_tgt_destroy_poll_group_done(void *cb_arg, int status)

{

	struct nvmf_tgt_poll_group *pg = cb_arg;



	free(pg);



	spdk_thread_send_msg(g_tgt_fini_thread, _nvmf_tgt_destroy_poll_group_done, NULL);

}



static void

nvmf_tgt_destroy_poll_group(void *ctx)

{

	struct nvmf_tgt_poll_group *pg = ctx;



	spdk_nvmf_poll_group_destroy(pg->group, nvmf_tgt_destroy_poll_group_done, pg);

}



static void

nvmf_tgt_destroy_poll_groups(void)

{

	struct nvmf_tgt_poll_group *pg, *tpg;

	struct spdk_thread *thread;



	thread = spdk_get_thread();

	g_tgt_fini_thread = spdk_get_thread();

	assert(g_tgt_fini_thread != NULL);



	TAILQ_FOREACH_SAFE(pg, &g_poll_groups, link, tpg) {

		if (pg->thread == thread) {

		TAILQ_REMOVE(&g_poll_groups, pg, link);

			spdk_nvmf_poll_group_destroy(pg->group, NULL, NULL);

			free(pg);

			assert(g_num_poll_groups > 0);

			g_num_poll_groups--;

			return;

		}

		spdk_thread_send_msg(pg->thread, nvmf_tgt_destroy_poll_group, pg);

	}

}
@@ -567,9 +584,7 @@ nvmf_tgt_advance_state(void) 
		}

		case NVMF_TGT_FINI_DESTROY_POLL_GROUPS:

			/* Send a message to each thread and destroy the poll group */

			spdk_for_each_thread(nvmf_tgt_destroy_poll_group,

					     NULL,

					     nvmf_tgt_destroy_poll_group_done);

			nvmf_tgt_destroy_poll_groups();

			break;

		case NVMF_TGT_FINI_STOP_ACCEPTOR:

			spdk_poller_unregister(&g_acceptor_poller);