Commit f50d3d77 authored by Darek Stojaczyk's avatar Darek Stojaczyk Committed by Ben Walker
Browse files

reactor: pin spdk threads to their cores 



We're about to drop legacy event messages from SPDK libs
and for that we'll replace various lcore numbers with
spdk_thread objects. For now SPDK libs can't spawn their
own threads, so they must use spdk_for_each_thread() and
spdk_get_thread() to retrieve different thread pointers.

The vhost library offers API to construct a vhost device
to be polled on specified cores, and in order to keep that
functionality we'll need to know which core each thread
is polling on. We would like to achieve that with
spdk_thread_get_cpumask(), but right now it always returns
all cores, so this patch changes it to return just
a single cpu on which the thread is actually pinned. It's
only a stop gap, eventually the vhost library will spawn
its own threads with custom cpumasks.

Change-Id: I15947727123c51b23f63727d52079770bfb2e07b
Signed-off-by: default avatarDarek Stojaczyk <dariusz.stojaczyk@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/452204


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
parent 45f0c800

lib/event/app.c

+12 −1
Original line number Diff line number Diff line
@@ -573,6 +573,7 @@ spdk_app_start(struct spdk_app_opts *opts, spdk_msg_fn start_fn, 
	struct spdk_conf	*config = NULL;

	int			rc;

	char			*tty;

	struct spdk_cpuset	*tmp_cpumask;



	if (!opts) {

		SPDK_ERRLOG("opts should not be NULL\n");
@@ -636,9 +637,19 @@ spdk_app_start(struct spdk_app_opts *opts, spdk_msg_fn start_fn, 
		goto app_start_log_close_err;

	}



	tmp_cpumask = spdk_cpuset_alloc();

	if (tmp_cpumask == NULL) {

		SPDK_ERRLOG("spdk_cpuset_alloc() failed\n");

		goto app_start_log_close_err;

	}



	spdk_cpuset_zero(tmp_cpumask);

	spdk_cpuset_set_cpu(tmp_cpumask, spdk_env_get_current_core(), true);



	/* Now that the reactors have been initialized, we can create an

	 * initialization thread. */

	g_app_thread = spdk_thread_create("app_thread", NULL);

	g_app_thread = spdk_thread_create("app_thread", tmp_cpumask);

	spdk_cpuset_free(tmp_cpumask);

	if (!g_app_thread) {

		SPDK_ERRLOG("Unable to create an spdk_thread for initialization\n");

		goto app_start_log_close_err;

lib/event/reactor.c

+15 −1
Original line number Diff line number Diff line
@@ -330,10 +330,18 @@ void 
spdk_reactors_start(void)

{

	struct spdk_reactor *reactor;

	struct spdk_cpuset *tmp_cpumask;

	uint32_t i, current_core;

	int rc;

	char thread_name[32];



	tmp_cpumask = spdk_cpuset_alloc();

	if (tmp_cpumask == NULL) {

		SPDK_ERRLOG("spdk_cpuset_alloc() failed\n");

		assert(false);

		return;

	}



	g_reactor_state = SPDK_REACTOR_STATE_RUNNING;

	g_spdk_app_core_mask = spdk_cpuset_alloc();
@@ -350,11 +358,17 @@ spdk_reactors_start(void) 


			/* For now, for each reactor spawn one thread. */

			snprintf(thread_name, sizeof(thread_name), "reactor_%u", reactor->lcore);

			spdk_thread_create(thread_name, NULL);



			spdk_cpuset_zero(tmp_cpumask);

			spdk_cpuset_set_cpu(tmp_cpumask, i, true);



			spdk_thread_create(thread_name, tmp_cpumask);

		}

		spdk_cpuset_set_cpu(g_spdk_app_core_mask, i, true);

	}



	spdk_cpuset_free(tmp_cpumask);



	/* Start the master reactor */

	reactor = spdk_reactor_get(current_core);

	_spdk_reactor_run(reactor);