Commit 7832b34f authored by Michal Berger's avatar Michal Berger Committed by Konrad Sztyber
Browse files

test/scheduler: Trigger actual freq drop 



In order to do so we need to make sure that freq is lowered for
all thread siblings of a given core. Since DPDK and/or dynamic
scheduler do not take that into the account we need to do this
on our own.

Find thread sibling of the main cpu and imitate the DPDK's governor
work by adjusting its freq settings.

Signed-off-by: default avatarMichal Berger <michal.berger@intel.com>
Change-Id: I154a2a789903b66c2722160d7e252221083f5e3c
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/16930


Reviewed-by: default avatarKonrad Sztyber <konrad.sztyber@intel.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
parent 6984eff3

test/scheduler/common.sh

+7 −3
Original line number Diff line number Diff line
@@ -64,6 +64,9 @@ map_cpus_node() { 
			core_idx=$(< "$sysfs_cpu/cpu$cpu_idx/topology/core_id")

			local -n _cpu_core_map=node_${node_idx}_core_${core_idx}

			_cpu_core_map+=("$cpu_idx") cpu_core_map[cpu_idx]=$core_idx

			local -n _cpu_siblings=node_${node_idx}_core_${core_idx}_thread_${cpu_idx}

			_cpu_siblings=($(parse_cpu_list "$sysfs_cpu/cpu$cpu_idx/topology/thread_siblings_list"))

			cpu_siblings[cpu_idx]="node_${node_idx}_core_${core_idx}_thread_${cpu_idx}[@]"

		fi

		_cpu_node_map[cpu_idx]=$cpu_idx cpu_node_map[cpu_idx]=$node_idx

		cpus+=("$cpu_idx")
@@ -74,6 +77,7 @@ map_cpus_node() { 


map_cpus() {

	local -g cpus=()

	local -g cpu_siblings=()

	local -g nodes=()

	local -g cpu_node_map=()

	local -g cpu_core_map=()
@@ -354,12 +358,12 @@ set_cpufreq() { 
			echo "$min_freq" > "$cpufreq/scaling_setspeed"

			;;

		intel_pstate | intel_cpufreq)

			if ((min_freq <= cpufreq_max_freqs[cpu])); then

				echo "$min_freq" > "$cpufreq/scaling_min_freq"

			fi

			if [[ -n $max_freq ]] && ((max_freq >= min_freq)); then

				echo "$max_freq" > "$cpufreq/scaling_max_freq"

			fi

			if ((min_freq <= cpufreq_max_freqs[cpu])); then

				echo "$min_freq" > "$cpufreq/scaling_min_freq"

			fi

			;;

	esac

}

test/scheduler/governor.sh

+25 −9
Original line number Diff line number Diff line
@@ -43,11 +43,23 @@ update_main_core_cpufreq() { 
		intel_pstate | intel_cpufreq) main_core_setspeed=$main_core_set_max_freq ;;

		cppc_cpufreq) main_core_setspeed=${cpufreq_setspeed[spdk_main_core]} ;;

	esac



	local thread

	for thread in "${!cpu_siblings[spdk_main_core]}"; do

		((thread == spdk_main_core)) && continue # handled by DPDK/scheduler

		# While assigning cpus to SPDK, we took every thread from given core,

		# hence the cpufreq governor should already be properly set by the

		# DPDK. So the only thing we need to take care of is {max,min} freq.

		# The expectation here is to see actual freq drop on the main cpu in

		# next iterations. Both, max and min, should be set to the same value.

		set_cpufreq "$thread" "$main_core_set_min_freq" "$main_core_set_max_freq"

	done

}



verify_dpdk_governor() {

	xtrace_disable



	map_cpus

	# Run the app and see if DPDK's PM subsystem is doing the proper thing. Aim at the main core.

	# What we expect to happen is based mainly on what cpufreq driver is in use. The main assumption

	# is that with all SPDK threads being idle the governor should start lowering the frequency for
@@ -77,6 +89,7 @@ verify_dpdk_governor() { 
	unset -v "cpus[spdk_main_core]"



	local samples=0 all_set=0 dir=-1 old_main_core_setspeed=0

	local old_main_core_set_cur_freq=0 first_main_core_set_cur_freq=0



	exec_under_dynamic_scheduler "${SPDK_APP[@]}" -m "$spdk_cpumask" --main-core "$spdk_main_core"
@@ -94,9 +107,13 @@ verify_dpdk_governor() { 
		elif ((main_core_setspeed < old_main_core_setspeed)); then

			dir=0

		elif ((main_core_setspeed == old_main_core_setspeed)); then

			# Frequency didn't change, skip and wait for a bit

			# Frequency didn't change, wait for a bit, but then fall to the main check to

			# see if cur freq actually changed or not.

			sleep 0.5s

			continue

		fi



		if ((first_main_core_set_cur_freq == 0)); then

			first_main_core_set_cur_freq=$main_core_set_cur_freq

		fi



		case "$main_core_driver" in
@@ -119,16 +136,10 @@ verify_dpdk_governor() { 
					&& ((main_core_setspeed == main_core_freqs[-1])) \

					&& ((dir == 0))

				;;

		esac && all_set=1

		esac && ((main_core_set_cur_freq < old_main_core_set_cur_freq)) && all_set=1



		# Print stats after first sane sample was taken

		if ((old_main_core_setspeed != 0 && dir != -1)); then

			# Currently, we only determine if DPDK set sysfs attributes to min frequency supported by given cpu.

			# We don't act on actual frequency cpu ends up spinning at. The reason being all sibling threads of

			# the main cpu would need to be adjusted manually (i.e. their cpufreq attributes) in order for freq

			# to start going down. This is deemed as not necessary for the moment since the core objective of

			# the test is to determine if DPDK's governor did its job, within its current scope, not how it

			# impacts the system overall.

			printf 'MAIN DPDK cpu%u current frequency at %u KHz (%u-%u KHz), set frequency %u KHz %s %u KHz\n' \

				"$spdk_main_core" "$main_core_set_cur_freq" "$main_core_min_freq" "$main_core_max_freq" \

				"$main_core_setspeed" "${dir_map[dir]}" "$old_main_core_setspeed"
@@ -137,10 +148,15 @@ verify_dpdk_governor() { 
		fi



		old_main_core_setspeed=$main_core_setspeed

		old_main_core_set_cur_freq=$main_core_set_cur_freq

	done



	((all_set == 1))



	printf 'Main cpu%u frequency dropped by %u%%\n' \

		"$spdk_main_core" \

		$(((first_main_core_set_cur_freq - main_core_set_cur_freq) * 100 / (first_main_core_set_cur_freq - main_core_min_freq)))



	xtrace_restore

}