Commit 227c8b81 authored by Liu Xiaodong's avatar Liu Xiaodong Committed by Tomasz Zawadzki
Browse files

reactor: avoid schedule thread to reactor in intr 



Currently, spdk_thread can't get executed on reactor
which is in interrupt mode but interrupt of spdk_thread
is not enabled. So avoid schedule spdk_thread on it.

Change-Id: Ib37d1585f173ed3be8def83cd18aef65e287361c
Signed-off-by: default avatarLiu Xiaodong <xiaodong.liu@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/5852


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
parent eff5b149

lib/event/app_rpc.c

+22 −0
Original line number Diff line number Diff line
@@ -600,6 +600,28 @@ rpc_thread_set_cpumask(struct spdk_jsonrpc_request *request, 
		goto err;

	}



	/* There may be any reactors running in interrupt mode. But currently,

	 * when interrupt ability of the spdk_thread is not enabled,

	 * spdk_thread can't get executed on reactor which runs in interrupt.

	 * Exclude the situation that reactors specified by the cpumask are

	 * all in interrupt mode.

	 */

	if (!spdk_interrupt_mode_is_enabled()) {

		struct spdk_reactor *local_reactor = spdk_reactor_get(spdk_env_get_current_core());

		struct spdk_cpuset tmp_cpuset;



		/* Masking off reactors which are in interrupt mode */

		spdk_cpuset_copy(&tmp_cpuset, &local_reactor->notify_cpuset);

		spdk_cpuset_negate(&tmp_cpuset);

		spdk_cpuset_and(&tmp_cpuset, &ctx->cpumask);

		if (spdk_cpuset_count(&tmp_cpuset) == 0) {

			spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR,

							     "cpumask %s are all in interrupt mode, and can't be scheduled yet\n",

							     req.cpumask);

			goto err;

		}

	}



	ctx->request = request;

	ctx->orig_thread = spdk_get_thread();

lib/event/reactor.c

+44 −0
Original line number Diff line number Diff line
@@ -957,6 +957,10 @@ _reactor_schedule_thread(struct spdk_thread *thread) 
	struct spdk_event *evt = NULL;

	struct spdk_cpuset *cpumask;

	uint32_t i;

	struct spdk_reactor *local_reactor = NULL;

	uint32_t current_lcore = spdk_env_get_current_core();

	struct spdk_cpuset polling_cpumask;

	struct spdk_cpuset valid_cpumask;



	cpumask = spdk_thread_get_cpumask(thread);
@@ -967,6 +971,46 @@ _reactor_schedule_thread(struct spdk_thread *thread) 
	memset(lw_thread, 0, sizeof(*lw_thread));

	lw_thread->last_stats = last_stats;



	if (current_lcore != SPDK_ENV_LCORE_ID_ANY) {

		local_reactor = spdk_reactor_get(current_lcore);

		assert(local_reactor);

	}



	/* When interrupt ability of spdk_thread is not enabled and the current

	 * reactor runs on DPDK thread, skip reactors which are in interrupt mode.

	 */

	if (!spdk_interrupt_mode_is_enabled() && local_reactor != NULL) {

		/* Get the cpumask of all reactors in polling */

		spdk_cpuset_zero(&polling_cpumask);

		SPDK_ENV_FOREACH_CORE(i) {

			spdk_cpuset_set_cpu(&polling_cpumask, i, true);

		}

		spdk_cpuset_xor(&polling_cpumask, &local_reactor->notify_cpuset);



		if (core == SPDK_ENV_LCORE_ID_ANY) {

			/* Get the cpumask of all valid reactors which are suggested and also in polling */

			spdk_cpuset_copy(&valid_cpumask, &polling_cpumask);

			spdk_cpuset_and(&valid_cpumask, spdk_thread_get_cpumask(thread));



			/* If there are any valid reactors, spdk_thread should be scheduled

			 * into one of the valid reactors.

			 * If there is no valid reactors, spdk_thread should be scheduled

			 * into one of the polling reactors.

			 */

			if (spdk_cpuset_count(&valid_cpumask) != 0) {

				cpumask = &valid_cpumask;

			} else {

				cpumask = &polling_cpumask;

			}

		} else if (!spdk_cpuset_get_cpu(&polling_cpumask, core)) {

			/* If specified reactor is not in polling, spdk_thread should be scheduled

			 * into one of the polling reactors.

			 */

			core = SPDK_ENV_LCORE_ID_ANY;

			cpumask = &polling_cpumask;

		}

	}



	pthread_mutex_lock(&g_scheduler_mtx);

	if (core == SPDK_ENV_LCORE_ID_ANY) {

		for (i = 0; i < spdk_env_get_core_count(); i++) {