Commit bfca56ff authored by Maciej Szwed's avatar Maciej Szwed Committed by Tomasz Zawadzki
Browse files

scheduler: Track threads count per core during rebalancing 



This will be helpful in upcoming patches for changing
interrupt mode (when reactor have no threads).

Signed-off-by: default avatarMaciej Szwed <maciej.szwed@intel.com>
Change-Id: I4902d0bd26f27db2b98dc4ca4fd4df934f59d9a3
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/5450


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: default avatarPaul Luse <paul.e.luse@intel.com>
parent ad15f441

include/spdk_internal/event.h

+1 −0
Original line number Diff line number Diff line
@@ -270,6 +270,7 @@ struct spdk_scheduler_core_info { 
	uint64_t core_busy_tsc;

	uint32_t lcore;

	uint32_t threads_count;

	uint32_t pending_threads_count;

	struct spdk_lw_thread **threads;

};

lib/event/scheduler_dynamic.c

+9 −0
Original line number Diff line number Diff line
@@ -127,9 +127,14 @@ balance(struct spdk_scheduler_core_info *cores_info, int cores_count, 
	g_last_main_core_busy = cores_info[g_main_lcore].core_busy_tsc;

	g_last_main_core_idle = cores_info[g_main_lcore].core_idle_tsc;



	SPDK_ENV_FOREACH_CORE(i) {

		cores_info[i].pending_threads_count = cores_info[i].threads_count;

	}



	/* Distribute active threads across all cores and move idle threads to main core */

	SPDK_ENV_FOREACH_CORE(i) {

		core = &cores_info[i];



		for (j = 0; j < core->threads_count; j++) {

			lw_thread = core->threads[j];

			lw_thread->new_lcore = lw_thread->lcore;
@@ -168,6 +173,8 @@ balance(struct spdk_scheduler_core_info *cores_info, int cores_count, 


					if (spdk_cpuset_get_cpu(cpumask, target_lcore)) {

						lw_thread->new_lcore = target_lcore;

						cores_info[target_lcore].pending_threads_count++;

						core->pending_threads_count--;



						if (target_lcore != g_main_lcore) {

							busy_threads_present = true;
@@ -181,6 +188,8 @@ balance(struct spdk_scheduler_core_info *cores_info, int cores_count, 
			} else if (i != g_main_lcore && load < SCHEDULER_LOAD_LIMIT) {

				/* This thread is idle but not on the main core, so we need to move it to the main core */

				lw_thread->new_lcore = g_main_lcore;

				cores_info[g_main_lcore].pending_threads_count++;

				core->pending_threads_count--;



				main_core_busy += spdk_min(UINT64_MAX - main_core_busy, thread_busy);

				main_core_idle -= spdk_min(main_core_idle, thread_busy);