Commit d5fe62b2 authored by Michal Berger's avatar Michal Berger Committed by Tomasz Zawadzki
Browse files

autotest: Add helpers for controling pm tools 



And use them to hook collect-* perf scripts into autobuild,autopackage.

This is meant to serve as a debug exposing impact of the full SPDK build
on the underlying system.

Change-Id: I7e0fe48927dd5725ea415e99752fa33e608710ec
Signed-off-by: default avatarMichal Berger <michal.berger@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/21494


Reviewed-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: default avatarJim Harris <jim.harris@samsung.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
parent 539fc9cc

autotest.sh

+1 −13
Original line number Diff line number Diff line
@@ -48,19 +48,7 @@ if [ $(uname -s) = Linux ]; then 
		udevadm_pid=$!

	fi



	mkdir -p "$output_dir/power"



	"$rootdir/scripts/perf/pm/collect-cpu-load" -d "$output_dir/power" > /dev/null &

	echo $! > "$output_dir/collect-cpu-load.pid"

	"$rootdir/scripts/perf/pm/collect-vmstat" -d "$output_dir/power" > /dev/null &

	echo $! > "$output_dir/collect-vmstat.pid"



	if [[ $(< /sys/class/dmi/id/chassis_vendor) != QEMU ]]; then

		"$rootdir/scripts/perf/pm/collect-cpu-temp" -d "$output_dir/power" -l &

		echo $! > "$output_dir/collect-cpu-temp.pid"

		"$rootdir/scripts/perf/pm/collect-bmc-pm" -d "$output_dir/power" -l &

		echo $! > "$output_dir/collect-bmc-pm.pid"

	fi

	start_monitor_resources

fi



trap "autotest_cleanup || :; exit 1" SIGINT SIGTERM EXIT

scripts/perf/pm/collect-bmc-pm

+16 −10
Original line number Diff line number Diff line
@@ -5,6 +5,10 @@ 


set -e



pmdir=$(readlink -f "$(dirname "$0")")

rootdir=$(readlink -f "$pmdir/../../../")

source "$pmdir/common"



hex() { printf '0x%02x\n' "$@"; }



calc() { bc <<< "scale=2; $*"; }
@@ -401,10 +405,10 @@ dump_readings() { 


		readings+=("Total: ${#readings[@]}")

		sensor="${sensor//[[:space:]]/_}"

		printf '%s\n' "$avg" > "$output_dir/${prefix:+${prefix}_}avg_${sensor}.bmc.pm.txt"

		printf '%s\n' "${readings[@]}" > "$output_dir/${prefix:+${prefix}_}all_${sensor}.bmc.pm.txt"

		printf 'Dumped avg to %s\n' "$output_dir/${prefix:+${prefix}_}avg_${sensor}.bmc.pm.txt" >&2

		printf 'Dumped all to %s\n' "$output_dir/${prefix:+${prefix}_}all_${sensor}.bmc.pm.txt" >&2

		printf '%s\n' "$avg" > "$PM_OUTPUTDIR/${prefix:+${prefix}_}avg_${sensor}.bmc.pm.txt"

		printf '%s\n' "${readings[@]}" > "$PM_OUTPUTDIR/${prefix:+${prefix}_}all_${sensor}.bmc.pm.txt"

		printf 'Dumped avg to %s\n' "$PM_OUTPUTDIR/${prefix:+${prefix}_}avg_${sensor}.bmc.pm.txt" >&2

		printf 'Dumped all to %s\n' "$PM_OUTPUTDIR/${prefix:+${prefix}_}all_${sensor}.bmc.pm.txt" >&2

	done

}
@@ -413,6 +417,7 @@ utc() { 
}



cleanup() {

	rm_pm_pid

	[[ -f $sdr_cache && $remove_sdr_cache == yes ]] && rm "$sdr_cache"

	dump_readings

}
@@ -462,7 +467,6 @@ help() { 


is_root



output_dir=/tmp

interval=1

remove_sdr_cache=yes

log_to_file=no
@@ -479,7 +483,7 @@ while getopts :hi:s:d:t:xlp:c:r arg; do 
			help

			exit 0

			;;

		d) output_dir=$OPTARG ;;

		d) PM_OUTPUTDIR=$OPTARG ;;

		s) extra_power_sensors+=("$OPTARG") ;;

		i) interface=${OPTARG,,} ;;

		t) interval=$OPTARG ;;
@@ -492,16 +496,18 @@ while getopts :hi:s:d:t:xlp:c:r arg; do 
	esac

done



declare -r sdr_cache=$output_dir/sdr.cache

declare -r sdr_cache=$PM_OUTPUTDIR/sdr.cache

declare -r log_file=${prefix:+${prefix}_}${0##*/}.bmc.pm.log



mkdir -p "$output_dir"

mkdir -p "$PM_OUTPUTDIR"

if [[ $log_to_file == yes ]]; then

	printf 'Redirecting to %s\n' "$output_dir/$log_file" >&2

	exec > "$output_dir/$log_file" 2>&1

	printf 'Redirecting to %s\n' "$PM_OUTPUTDIR/$log_file" >&2

	exec > "$PM_OUTPUTDIR/$log_file" 2>&1

fi



save_pm_pid

trap 'cleanup' EXIT

trap 'retag' USR1



ipmi_supported

power_support

scripts/perf/pm/collect-cpu-load

+18 −6
Original line number Diff line number Diff line
@@ -13,12 +13,13 @@ source "$pmdir/common" 
help() {

	cat <<- HELP



		Usage: $0 [-h] [-c count] [-d dir] [-p prefix] [cpu0 cpu1 ...]

		Usage: $0 [-h] [-c count] [-d dir] [-l] [-p prefix] [cpu0 cpu1 ...]



		-h - Print this message.

		-c - Execute count times. 0 is the default and it means to run

		     indefinitely.

		-d - Directory where results should be saved. Default is /tmp.

		-l - Save output of the script to a log file (dir/${0##*/}.pm.log).

		-p - Add prefix to saved files.

		-t - How long to wait before each load read. Default is 1s.
@@ -31,27 +32,32 @@ help() { 
	HELP

}



cleanup() {

	rm_pm_pid

}



# get_cpu_time() uses these, so put some stubs in place

xtrace_disable() { :; }

xtrace_restore() { :; }



_get_cpu_time() {

	get_cpu_time "$@" > >(tee "$output_dir/$log_file")

	get_cpu_time "$@"

}



count=0

interval=1

output_dir=/tmp

log_to_file=no

prefix=""



while getopts c:d:hp:t: opt; do

while getopts c:d:hlp:t: opt; do

	case "$opt" in

		c) count=$OPTARG ;;

		d) output_dir=$OPTARG ;;

		d) PM_OUTPUTDIR=$OPTARG ;;

		h)

			help

			exit 0

			;;

		l) log_to_file=yes ;;

		p) prefix=$OPTARG ;;

		t) interval=$OPTARG ;;

		*) ;;
@@ -61,11 +67,17 @@ shift $((OPTIND - 1)) 


declare -r log_file=${prefix:+${prefix}_}${0##*/}.pm.log



mkdir -p "$output_dir"

mkdir -p "$PM_OUTPUTDIR"

if [[ $log_to_file == yes ]]; then

	printf 'Redirecting to %s\n' "$PM_OUTPUTDIR/$log_file" >&2

	exec > "$PM_OUTPUTDIR/$log_file" 2>&1

fi



cpus=("$@")

((${#cpus[@]} > 0)) || cpus=($(get_online_cpus))



save_pm_pid

trap 'cleanup' EXIT

trap 'retag' USR1



_get_cpu_time "$count" "" 1 "$interval" "${cpus[@]}"

scripts/perf/pm/collect-cpu-temp

+10 −10
Original line number Diff line number Diff line
@@ -175,8 +175,6 @@ init_modules() { 
	done

}



run_module() { "$1" | tee -a /dev/stderr; }



collect_readings() {

	local _count=$count module data
@@ -185,7 +183,7 @@ collect_readings() { 
	while ((count <= 0 ? 1 : _count--)); do

		for module in "${modules_to_run[@]}"; do

			local -n ref=${modules_out_refs["$module"]}

			data=$(run_module "$module")

			data=$("$module")

			[[ -n $data ]] && ref+=("$data")

		done

		sleep "${interval}s"
@@ -202,11 +200,13 @@ cleanup() { 
	for module in "${!modules_out_refs[@]}"; do

		local -n _ref=${modules_out_refs["$module"]}

		((${#_ref[@]} > 0)) || continue

		printf '%s\n' "${_ref[@]}" > "$output_dir/${prefix:+${prefix}_}$module.pm.txt"

		printf '%s\n' "${_ref[@]}" > "$PM_OUTPUTDIR/${prefix:+${prefix}_}$module.pm.txt"

		printf 'Dumped %s module to %s\n' \

			"$module" \

			"$output_dir/${prefix:+${prefix}_}$module.pm.txt"

			"$PM_OUTPUTDIR/${prefix:+${prefix}_}$module.pm.txt"

	done



	rm_pm_pid

}



help() {
@@ -236,13 +236,12 @@ count=0 
debug=0

interval=1

log_to_file=no

output_dir=/tmp

prefix=""



while getopts c:d:hlp:t:v opt; do

	case "$opt" in

		c) count=$OPTARG ;;

		d) output_dir=$OPTARG ;;

		d) PM_OUTPUTDIR=$OPTARG ;;

		h)

			help

			exit 0
@@ -258,12 +257,13 @@ shift $((OPTIND - 1)) 


declare -r log_file=${prefix:+${prefix}_}${0##*/}.pm.log



mkdir -p "$output_dir"

mkdir -p "$PM_OUTPUTDIR"

if [[ $log_to_file == yes ]]; then

	printf 'Redirecting to %s\n' "$output_dir/$log_file" >&2

	exec > "$output_dir/$log_file" 2>&1

	printf 'Redirecting to %s\n' "$PM_OUTPUTDIR/$log_file" >&2

	exec > "$PM_OUTPUTDIR/$log_file" 2>&1

fi



save_pm_pid

trap 'cleanup' EXIT

trap 'retag' USR1

scripts/perf/pm/collect-vmstat

+18 −6
Original line number Diff line number Diff line
@@ -12,12 +12,13 @@ source "$pmdir/common" 
help() {

	cat <<- HELP



		Usage: $0 [-h] [-c count] [-d dir] [-p prefix] [-r reprint_header_count]

		Usage: $0 [-h] [-c count] [-d dir] [-l] [-p prefix] [-r reprint_header_count]



		-h - Print this message.

		-c - Execute count times. 0 is the default and it means to run

		     indefinitely.

		-d - Directory where results should be saved. Default is /tmp.

		-l - Save output of the script to a log file (dir/${0##*/}.pm.log).

		-p - Add prefix to saved files.

		-r - Stat count after which header should be re-printed. Default is 20.

		-t - How long to wait before each stat dump. Default is 1s.
@@ -61,23 +62,28 @@ _vmstat() { 
		fi

		printf '%s\n' "${_vmstat[@]}"

		sleep "${interval}s"

	done > >(tee "$output_dir/$log_file")

	done

}



cleanup() {

	rm_pm_pid

}



count=0

interval=1

output_dir=/tmp

log_to_file=no

prefix=""

reprint_header=20



while getopts c:d:hp:r:t: opt; do

while getopts c:d:hlp:r:t: opt; do

	case "$opt" in

		c) count=$OPTARG ;;

		d) output_dir=$OPTARG ;;

		d) PM_OUTPUTDIR=$OPTARG ;;

		h)

			help

			exit 0

			;;

		l) log_to_file=yes ;;

		p) prefix=$OPTARG ;;

		r) reprint_header=$OPTARG ;;

		t) interval=$OPTARG ;;
@@ -87,8 +93,14 @@ done 


declare -r log_file=${prefix:+${prefix}_}${0##*/}.pm.log



mkdir -p "$output_dir"

mkdir -p "$PM_OUTPUTDIR"

if [[ $log_to_file == yes ]]; then

	printf 'Redirecting to %s\n' "$PM_OUTPUTDIR/$log_file" >&2

	exec > "$PM_OUTPUTDIR/$log_file" 2>&1

fi



save_pm_pid

trap 'cleanup' EXIT

trap 'retag' USR1



_vmstat "$count" "$interval" "$reprint_header"