Commit c3dc7fb5 authored by Karol Latecki's avatar Karol Latecki Committed by Jim Harris
Browse files

test/vhost_perf: parse fio json results to csv 



Parse fio output to CSV.
Calculate average read and write metrics while
parsing and present final results.

Change-Id: I8f6d7f84205be409f0ccbba7a42eb7888b2cb03d
Signed-off-by: default avatarKarol Latecki <karol.latecki@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/3253


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: default avatarMaciej Wawryk <maciejx.wawryk@intel.com>
Reviewed-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: default avatarMichal Berger <michalx.berger@intel.com>
parent 782681e0

test/vhost/common.sh

+75 −0
Original line number Diff line number Diff line
@@ -1155,6 +1155,81 @@ function run_fio() { 
	fi

}



# Parsing fio results for json output and client-server mode only!

function parse_fio_results() {

	local fio_log_dir=$1

	local fio_log_filename=$2

	local fio_csv_filename



	# Variables used in parsing loop

	local log_file

	local rwmode mixread mixwrite

	local lat_key lat_divisor

	local client_stats iops bw

	local read_avg_lat read_min_lat read_max_lat

	local write_avg_lat write_min_lat write_min_lat



	declare -A results

	results["iops"]=0

	results["bw"]=0

	results["avg_lat"]=0

	results["min_lat"]=0

	results["max_lat"]=0



	# Loop using the log filename to see if there are any other

	# matching files. This is in case we ran fio test multiple times.

	log_files=("$fio_log_dir/$fio_log_filename"*)

	for log_file in "${log_files[@]}"; do

		rwmode=$(jq -r '.["client_stats"][0]["job options"]["rw"]' "$log_file")

		mixread=1

		mixwrite=1

		if [[ $rwmode = *"rw"* ]]; then

			mixread=$(jq -r '.["client_stats"][0]["job options"]["rwmixread"]' "$log_file")

			mixread=$(bc -l <<< "scale=3; $mixread/100")

			mixwrite=$(bc -l <<< "scale=3; 1-$mixread")

		fi



		client_stats=$(jq -r '.["client_stats"][] | select(.jobname == "All clients")' "$log_file")



		# Check latency unit and later normalize to microseconds

		lat_key="lat_us"

		lat_divisor=1

		if jq -er '.read["lat_ns"]' &> /dev/null <<< $client_stats; then

			lat_key="lat_ns"

			lat_divisor=1000

		fi



		# Horrific bash float point arithmetic oprations below.

		# Viewer discretion is advised.

		iops=$(jq -r '[.read["iops"],.write["iops"]] | add' <<< $client_stats)

		bw=$(jq -r '[.read["bw"],.write["bw"]] | add' <<< $client_stats)

		read_avg_lat=$(jq -r --arg lat_key $lat_key '.read[$lat_key]["mean"]' <<< $client_stats)

		read_min_lat=$(jq -r --arg lat_key $lat_key '.read[$lat_key]["min"]' <<< $client_stats)

		read_max_lat=$(jq -r --arg lat_key $lat_key '.read[$lat_key]["max"]' <<< $client_stats)

		write_avg_lat=$(jq -r --arg lat_key $lat_key '.write[$lat_key]["mean"]' <<< $client_stats)

		write_min_lat=$(jq -r --arg lat_key $lat_key '.write[$lat_key]["min"]' <<< $client_stats)

		write_max_lat=$(jq -r --arg lat_key $lat_key '.write[$lat_key]["max"]' <<< $client_stats)



		results["iops"]=$(bc -l <<< "${results[iops]} + $iops")

		results["bw"]=$(bc -l <<< "${results[bw]} + $bw")

		results["avg_lat"]=$(bc -l <<< "${results[avg_lat]} + ($mixread*$read_avg_lat + $mixwrite*$write_avg_lat)/$lat_divisor")

		results["min_lat"]=$(bc -l <<< "${results[min_lat]} + ($mixread*$read_min_lat + $mixwrite*$write_min_lat)/$lat_divisor")

		results["max_lat"]=$(bc -l <<< "${results[max_lat]} + ($mixread*$read_max_lat + $mixwrite*$write_max_lat)/$lat_divisor")

	done



	results["iops"]=$(bc -l <<< "scale=3; ${results[iops]} / ${#log_files[@]}")

	results["bw"]=$(bc -l <<< "scale=3; ${results[bw]} / ${#log_files[@]}")

	results["avg_lat"]=$(bc -l <<< "scale=3; ${results[avg_lat]} / ${#log_files[@]}")

	results["min_lat"]=$(bc -l <<< "scale=3; ${results[min_lat]} / ${#log_files[@]}")

	results["max_lat"]=$(bc -l <<< "scale=3; ${results[max_lat]} / ${#log_files[@]}")



	fio_csv_filename="${fio_log_filename%%.*}.csv"

	cat <<- EOF > "$fio_log_dir/$fio_csv_filename"

		iops,bw,avg_lat,min_lat,max_lat

		${results["iops"]},${results["bw"]},${results["avg_lat"]},${results["min_lat"]},${results["max_lat"]}

	EOF

}



# Shutdown or kill any running VM and SPDK APP.

#

function at_app_exit() {

test/vhost/perf_bench/vhost_perf.sh

+2 −0
Original line number Diff line number Diff line
@@ -446,6 +446,8 @@ for fio_job in ${fio_jobs//,/ }; do 
		mv $VHOST_DIR/fio_results/$fio_log_fname $VHOST_DIR/fio_results/$fio_log_fname.$i

		sleep 1

	done



	parse_fio_results "$VHOST_DIR/fio_results" "$fio_log_fname"

done



notice "Shutting down virtual machines..."