Commit d841e24b authored by Michal Berger's avatar Michal Berger Committed by Tomasz Zawadzki
Browse files

test/scheduler: Add tests covering new load balancing functionality 



Signed-off-by: default avatarMichal Berger <michalx.berger@intel.com>
Signed-off-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
Change-Id: Ia7b075711192189918d9fb1b186b907814a12430
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/7450


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarKonrad Sztyber <konrad.sztyber@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
parent d47b0d2d

test/scheduler/common.sh

+22 −0
Original line number Diff line number Diff line
@@ -339,6 +339,9 @@ set_cpufreq_governor() { 
}



exec_under_dynamic_scheduler() {

	if [[ -e /proc/$spdk_pid/status ]]; then

		killprocess "$spdk_pid"

	fi

	"$@" --wait-for-rpc &

	spdk_pid=$!

	# Give some time for the app to init itself
@@ -515,3 +518,22 @@ collect_cpu_idle() { 
		fi

	done

}



update_thread_cpus_map() {

	local cpu

	local -g thread_cpus=()

	local reactor_framework



	((${#cpus[@]} > 0)) || return 1



	get_thread_stats



	reactor_framework=$(rpc_cmd framework_get_reactors | jq -r '.reactors[]')

	for cpu in "${cpus[@]}"; do

		for thread in $(jq -r "select(.lcore == $cpu) | .lw_threads[].id" <<< "$reactor_framework"); do

			printf '* Thread %u (%s) on cpu%u\n' "$thread" "${thread_map[thread]}" "$cpu"

			thread_cpus[thread]=$cpu

		done

	done

	((${#thread_cpus[@]} > 0))

}

test/scheduler/load_balancing.sh

+91 −0
Original line number Diff line number Diff line
@@ -134,7 +134,98 @@ balanced() { 
	done

}



core_load() {

	local sched_period=1 # default, 1s

	local thread

	local on_main_core=0 on_next_core=0



	# Re-exec the scheduler app to make sure rr balancer won't affect threads without

	# configured cpumask from the previous test suites.



	exec_under_dynamic_scheduler "$scheduler" -m "$spdk_cpusmask" --main-core "$spdk_main_core"



	# Create thread0 with 90% activity no cpumask, expecting it to remain on main cpu

	thread0=$(create_thread -n "thread0" -a 90)



	sleep $((2 * sched_period))

	update_thread_cpus_map



	((thread_cpus[thread0] == spdk_main_core))



	# Create thread1 with 90% activity. Expecting one of the threads to be moved to next

	# cpu and the other remain on main cpu. Verifying that threads are spread out when core

	# load is over 95% limit.

	thread1=$(create_thread -n "thread1" -a 90)



	# Three iterations are needed, as both active threads first are moved out of main core.

	# During next scheduling period one of them is moved back to the main core.

	sleep $((3 * sched_period))

	update_thread_cpus_map



	((thread_cpus[thread0] == spdk_main_core || thread_cpus[thread1] == spdk_main_core))

	((thread_cpus[thread0] != thread_cpus[thread1]))



	# Create thread2 with 10% activity. Expecting the idle thread2 to be placed on main cpu and two

	# other active threads on next cpus. Verifying the condition where core load over 95% moves threads

	# away from main cpu.

	thread2=$(create_thread -n "thread2" -a 10)



	sleep $((2 * sched_period))

	update_thread_cpus_map



	((thread_cpus[thread2] == spdk_main_core))

	((thread_cpus[thread1] != spdk_main_core))

	((thread_cpus[thread0] != spdk_main_core))

	((thread_cpus[thread0] != thread_cpus[thread1]))



	# Change all threads activity to 10%. Expecting all threads to be placed on main cpu.

	# Verifying the condition where core load less than 95% is grouping multiple threads.

	active_thread "$thread0" 10

	active_thread "$thread1" 10

	active_thread "$thread2" 10



	sleep $((2 * sched_period))

	update_thread_cpus_map



	for thread in \

		"$thread0" \

		"$thread1" \

		"$thread2"; do

		((thread_cpus[thread] == spdk_main_core))

	done



	# Create thread3, thread4 and thread 5 with 25% activity. Expecting one of the threads on next cpu

	# and rest on main cpu. Total load on main cpu will be (10*3+25*2) 80%, and next cpu 25%.

	thread3=$(create_thread -n "thread3" -a 25)

	thread4=$(create_thread -n "thread4" -a 25)

	thread5=$(create_thread -n "thread5" -a 25)



	# Three iterations are needed, as all threads look active on first iteration since they are on the main core.

	# Second iteraion will have them spread out over cores and only third will collapse to the expected scenario.

	sleep $((3 * sched_period))

	update_thread_cpus_map



	for thread in \

		"$thread0" \

		"$thread1" \

		"$thread2" \

		"$thread3" \

		"$thread4" \

		"$thread5"; do

		if ((thread_cpus[thread] == spdk_main_core)); then

			((++on_main_core))

		else

			((++on_next_core))

		fi



		destroy_thread "$thread"

	done



	((on_main_core == 5 && on_next_core == 1))

}



exec_under_dynamic_scheduler "$scheduler" -m "$spdk_cpusmask" --main-core "$spdk_main_core"



run_test "busy" busy

run_test "balanced" balanced

run_test "core_load" core_load