Commit 991a56f8 authored by Ben Walker's avatar Ben Walker Committed by Tomasz Zawadzki
Browse files

fio: Continue polling spdk_threads on the init thread after shutdown 



When fio wants to shut down a thread, it stops polling it immediately.
Some SPDK threads need to continue to be polled for a period of time
to allow the message passing to all complete, so this behavior in fio
could lead to a hang. Instead, when fio wants to shut down a thread,
place that thread onto a global list that's polled by the init thread
which lives for the duration of the program. The init thread will
continue polling the spdk_thread until it has exited.

Change-Id: I07292d5ba389167c45a14aa8ffbe72b49e88d17d
Signed-off-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/3682


Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: default avatarGangCao <gang.cao@intel.com>
Reviewed-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
parent 4696163e

examples/bdev/fio_plugin/fio_plugin.c

+56 −21
Original line number Diff line number Diff line
@@ -87,6 +87,8 @@ struct spdk_fio_thread { 
	struct io_u		**iocq;		/* io completion queue */

	unsigned int		iocq_count;	/* number of iocq entries filled by last getevents */

	unsigned int		iocq_size;	/* number of iocq entries allocated */



	TAILQ_ENTRY(spdk_fio_thread)	link;

};



static bool g_spdk_env_initialized = false;
@@ -96,6 +98,12 @@ static int spdk_fio_init(struct thread_data *td); 
static void spdk_fio_cleanup(struct thread_data *td);

static size_t spdk_fio_poll_thread(struct spdk_fio_thread *fio_thread);



static pthread_t g_init_thread_id = 0;

static pthread_mutex_t g_init_mtx = PTHREAD_MUTEX_INITIALIZER;

static pthread_cond_t g_init_cond;

static bool g_poll_loop = true;

static TAILQ_HEAD(, spdk_fio_thread) g_threads = TAILQ_HEAD_INITIALIZER(g_threads);



/* Default polling timeout (ns) */

#define SPDK_FIO_POLLING_TIMEOUT 1000000000ULL
@@ -149,19 +157,9 @@ spdk_fio_cleanup_thread(struct spdk_fio_thread *fio_thread) 
{

	spdk_thread_send_msg(fio_thread->thread, spdk_fio_bdev_close_targets, fio_thread);



	while (!spdk_thread_is_idle(fio_thread->thread)) {

		spdk_fio_poll_thread(fio_thread);

	}



	spdk_set_thread(fio_thread->thread);



	spdk_thread_exit(fio_thread->thread);

	while (!spdk_thread_is_exited(fio_thread->thread)) {

		spdk_thread_poll(fio_thread->thread, 0, 0);

	}

	spdk_thread_destroy(fio_thread->thread);

	free(fio_thread->iocq);

	free(fio_thread);

	pthread_mutex_lock(&g_init_mtx);

	TAILQ_INSERT_TAIL(&g_threads, fio_thread, link);

	pthread_mutex_unlock(&g_init_mtx);

}



static void
@@ -189,11 +187,6 @@ spdk_fio_calc_timeout(struct spdk_fio_thread *fio_thread, struct timespec *ts) 
	}

}



static pthread_t g_init_thread_id = 0;

static pthread_mutex_t g_init_mtx = PTHREAD_MUTEX_INITIALIZER;

static pthread_cond_t g_init_cond;

static bool g_poll_loop = true;



static void

spdk_fio_bdev_init_done(int rc, void *cb_arg)

{
@@ -232,6 +225,7 @@ spdk_init_thread_poll(void *arg) 
{

	struct spdk_fio_options		*eo = arg;

	struct spdk_fio_thread		*fio_thread;

	struct spdk_fio_thread		*thread, *tmp;

	struct spdk_conf		*config = NULL;

	struct spdk_env_opts		opts;

	bool				done;
@@ -325,19 +319,37 @@ spdk_init_thread_poll(void *arg) 
	pthread_mutex_lock(&g_init_mtx);

	pthread_cond_signal(&g_init_cond);



	pthread_mutex_unlock(&g_init_mtx);



	while (g_poll_loop) {

		spdk_fio_poll_thread(fio_thread);



		pthread_mutex_lock(&g_init_mtx);

		if (!TAILQ_EMPTY(&g_threads)) {

			TAILQ_FOREACH_SAFE(thread, &g_threads, link, tmp) {

				spdk_fio_poll_thread(thread);

			}



			/* If there are exiting threads to poll, don't sleep. */

			pthread_mutex_unlock(&g_init_mtx);

			continue;

		}



		/* Figure out how long to sleep. */

		clock_gettime(CLOCK_MONOTONIC, &ts);

		spdk_fio_calc_timeout(fio_thread, &ts);



		rc = pthread_cond_timedwait(&g_init_cond, &g_init_mtx, &ts);

		pthread_mutex_unlock(&g_init_mtx);



		if (rc != ETIMEDOUT) {

			break;

		}





	}



	pthread_mutex_unlock(&g_init_mtx);

	spdk_fio_cleanup_thread(fio_thread);



	/* Finalize the bdev layer */

	done = false;
@@ -345,9 +357,32 @@ spdk_init_thread_poll(void *arg) 


	do {

		spdk_fio_poll_thread(fio_thread);

	} while (!done && !spdk_thread_is_idle(fio_thread->thread));



	spdk_fio_cleanup_thread(fio_thread);

		TAILQ_FOREACH_SAFE(thread, &g_threads, link, tmp) {

			spdk_fio_poll_thread(thread);

		}

	} while (!done);



	/* Now exit all the threads */

	TAILQ_FOREACH(thread, &g_threads, link) {

		spdk_set_thread(thread->thread);

		spdk_thread_exit(thread->thread);

		spdk_set_thread(NULL);

	}



	/* And wait for them to gracefully exit */

	while (!TAILQ_EMPTY(&g_threads)) {

		TAILQ_FOREACH_SAFE(thread, &g_threads, link, tmp) {

			if (spdk_thread_is_exited(thread->thread)) {

				TAILQ_REMOVE(&g_threads, thread, link);

				spdk_thread_destroy(thread->thread);

				free(thread->iocq);

				free(thread);

			} else {

				spdk_thread_poll(thread->thread, 0, 0);

			}

		}

	}



	pthread_exit(NULL);