Commit 20995d37 authored by Karol Latecki's avatar Karol Latecki Committed by Tomasz Zawadzki
Browse files

test/nvme_perf: remove --disk-no option from perf script 



Script was written in such a way that tests would be
performed starting with all found NVMe disks in configuration.
Then the number of disks for the workload would be reduced in
loop until reaching 1. This is the scenario used in test
case 1, described in SPDK NVMe Performance Benchmark document.

--disk-no allowed to avoid running test in loop, and specify
just a single value for the number of disks to be tested.

Both loop logic and --disk-no are no longer required as
tests are scheduled and run via CI system which picks the
parameters for each test run.

Change-Id: Iad0f4d0d259ac64680fd5bb69d7cbb7e0afff79c
Signed-off-by: default avatarKarol Latecki <karol.latecki@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/3295


Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarPaul Luse <paul.e.luse@intel.com>
Reviewed-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
parent 0655f92a

test/nvme/perf/common.sh

+0 −7
Original line number Diff line number Diff line
@@ -32,7 +32,6 @@ DISKNO=1 
CPUS_ALLOWED=1

NOIOSCALING=false

PRECONDITIONING=true

ONEWORKLOAD=false

DATE="$(date +'%m_%d_%Y_%H%M%S')"



function discover_bdevs() {
@@ -456,8 +455,6 @@ function usage() { 
	echo "                             - kernel-hybrid-polling"

	echo "                             - kernel-libaio"

	echo "                             - kernel-io-uring"

	echo "    --disk-no=INT,ALL     Number of disks to test on, this will run one workload on selected number od disks,"

	echo "                          it discards max-disk setting, if =ALL then test on all found disk. [default=$DISKNO]"

	echo "    --max-disk=INT,ALL    Number of disks to test on, this will run multiple workloads with increasing number of disk each run."

	echo "                          If =ALL then test on all found disk. [default=$DISKNO]"

	echo "    --cpu-allowed=INT     Comma-separated list of CPU cores used to run the workload. [default=$CPUS_ALLOWED]"
@@ -484,10 +481,6 @@ while getopts 'h-:' optchar; do 
				repeat-no=*) REPEAT_NO="${OPTARG#*=}" ;;

				fio-bin=*) FIO_BIN="${OPTARG#*=}" ;;

				driver=*) PLUGIN="${OPTARG#*=}" ;;

				disk-no=*)

					DISKNO="${OPTARG#*=}"

					ONEWORKLOAD=true

					;;

				max-disk=*) DISKNO="${OPTARG#*=}" ;;

				cpu-allowed=*) CPUS_ALLOWED="${OPTARG#*=}" ;;

				no-preconditioning) PRECONDITIONING=false ;;

test/nvme/perf/run_perf.sh

+96 −113
Original line number Diff line number Diff line
@@ -97,32 +97,30 @@ printf "%s,%s,%s,%s,%s,%s,%s,%s\n" $RUNTIME $RAMP_TIME $PLUGIN $IODEPTH $BLK_SIZ 
echo "num_of_disks,iops,avg_lat[usec],p99[usec],p99.99[usec],stdev[usec],avg_slat[usec],avg_clat[usec],bw[Kib/s]" >> $result_file

#Run each workolad $REPEAT_NO times

for ((j = 0; j < REPEAT_NO; j++)); do

	#Start with $DISKNO disks and remove 2 disks for each run to avoid preconditioning before each run.

	for ((k = DISKNO; k >= 1; k -= 2)); do

	cp $BASE_DIR/config.fio.tmp $BASE_DIR/config.fio

	echo "" >> $BASE_DIR/config.fio

	#The SPDK fio plugin supports submitting/completing I/Os to multiple SSDs from a single thread.

	#Therefore, the per thread queue depth is set to the desired IODEPTH/device X the number of devices per thread.

	if [[ "$PLUGIN" =~ "spdk-plugin" ]] && [[ "$NOIOSCALING" = false ]]; then

			qd=$((IODEPTH * k))

		qd=$((IODEPTH * DISKNO))

	else

		qd=$IODEPTH

	fi



	if [ $PLUGIN = "spdk-perf-bdev" ]; then

		run_bdevperf > $NVME_FIO_RESULTS

			iops_disks[$k]=$((${iops_disks[$k]} + $(get_bdevperf_results iops)))

			bw[$k]=$((${bw[$k]} + $(get_bdevperf_results bw_Kibs)))

		iops_disks=$((iops_disks + $(get_bdevperf_results iops)))

		bw=$((bw + $(get_bdevperf_results bw_Kibs)))

		cp $NVME_FIO_RESULTS $result_dir/perf_results_${MIX}_${PLUGIN}_${NO_CORES}cpus_${DATE}_${k}_disks_${j}.output

	elif [ $PLUGIN = "spdk-perf-nvme" ]; then

			run_nvmeperf $k > $NVME_FIO_RESULTS

		run_nvmeperf $DISKNO > $NVME_FIO_RESULTS

		read -r iops bandwidth mean_lat min_lat max_lat <<< $(get_nvmeperf_results)



			iops_disks[$k]=$((${iops_disks[$k]} + iops))

			bw[$k]=$((${bw[$k]} + bandwidth))

			mean_lat_disks_usec[$k]=$((${mean_lat_disks_usec[$k]} + mean_lat))

			min_lat_disks_usec[$k]=$((${min_lat_disks_usec[$k]} + min_lat))

			max_lat_disks_usec[$k]=$((${max_lat_disks_usec[$k]} + max_lat))

		iops_disks=$((iops_disks + iops))

		bw=$((bw + bandwidth))

		mean_lat_disks_usec=$((mean_lat_disks_usec + mean_lat))

		min_lat_disks_usec=$((min_lat_disks_usec + min_lat))

		max_lat_disks_usec=$((max_lat_disks_usec + max_lat))



		cp $NVME_FIO_RESULTS $result_dir/perf_results_${MIX}_${PLUGIN}_${NO_CORES}cpus_${DATE}_${k}_disks_${j}.output

	else
@@ -140,7 +138,7 @@ for ((j = 0; j < REPEAT_NO; j++)); do 
			log_avg_msec=250

		EOF



			create_fio_config $k $PLUGIN "$DISK_NAMES" "$DISKS_NUMA" "$CORES"

		create_fio_config $DISKNO $PLUGIN "$DISK_NAMES" "$DISKS_NUMA" "$CORES"

		echo "USING CONFIG:"

		cat $BASE_DIR/config.fio
@@ -160,64 +158,49 @@ for ((j = 0; j < REPEAT_NO; j++)); do 
		elif [[ $RW = *"write"* ]]; then

			rwmixread=0

		fi

			iops_disks[$k]=$((iops_disks[k] + $(get_results iops $rwmixread)))

			mean_lat_disks_usec[$k]=$((mean_lat_disks_usec[k] + $(get_results mean_lat_usec $rwmixread)))

			p99_lat_disks_usec[$k]=$((p99_lat_disks_usec[k] + $(get_results p99_lat_usec $rwmixread)))

			p99_99_lat_disks_usec[$k]=$((p99_99_lat_disks_usec[k] + $(get_results p99_99_lat_usec $rwmixread)))

			stdev_disks_usec[$k]=$((stdev_disks_usec[k] + $(get_results stdev_usec $rwmixread)))

		iops_disks=$((iops_disks + $(get_results iops $rwmixread)))

		mean_lat_disks_usec=$((mean_lat_disks_usec + $(get_results mean_lat_usec $rwmixread)))

		p99_lat_disks_usec=$((p99_lat_disks_usec + $(get_results p99_lat_usec $rwmixread)))

		p99_99_lat_disks_usec=$((p99_99_lat_disks_usec + $(get_results p99_99_lat_usec $rwmixread)))

		stdev_disks_usec=$((stdev_disks_usec + $(get_results stdev_usec $rwmixread)))



			mean_slat_disks_usec[$k]=$((mean_slat_disks_usec[k] + $(get_results mean_slat_usec $rwmixread)))

			mean_clat_disks_usec[$k]=$((mean_clat_disks_usec[k] + $(get_results mean_clat_usec $rwmixread)))

			bw[$k]=$((bw[k] + $(get_results bw_Kibs $rwmixread)))

		mean_slat_disks_usec=$((mean_slat_disks_usec + $(get_results mean_slat_usec $rwmixread)))

		mean_clat_disks_usec=$((mean_clat_disks_usec + $(get_results mean_clat_usec $rwmixread)))

		bw=$((bw + $(get_results bw_Kibs $rwmixread)))



		cp $NVME_FIO_RESULTS $result_dir/perf_results_${MIX}_${PLUGIN}_${NO_CORES}cpus_${DATE}_${k}_disks_${j}.json

		cp $BASE_DIR/config.fio $result_dir/config_${MIX}_${PLUGIN}_${NO_CORES}cpus_${DATE}_${k}_disks_${j}.fio

		rm -f $BASE_DIR/config.fio

	fi



		#if tested on only one number of disk

		if $ONEWORKLOAD; then

			break

		fi

	done

done

#Write results to csv file

for ((k = DISKNO; k >= 1; k -= 2)); do

	iops_disks[$k]=$((${iops_disks[$k]} / REPEAT_NO))



iops_disks=$((iops_disks / REPEAT_NO))

bw=$((bw / REPEAT_NO))

if [[ "$PLUGIN" =~ "plugin" ]]; then

		mean_lat_disks_usec[$k]=$((${mean_lat_disks_usec[$k]} / REPEAT_NO))

		p99_lat_disks_usec[$k]=$((${p99_lat_disks_usec[$k]} / REPEAT_NO))

		p99_99_lat_disks_usec[$k]=$((${p99_99_lat_disks_usec[$k]} / REPEAT_NO))

		stdev_disks_usec[$k]=$((${stdev_disks_usec[$k]} / REPEAT_NO))

		mean_slat_disks_usec[$k]=$((${mean_slat_disks_usec[$k]} / REPEAT_NO))

		mean_clat_disks_usec[$k]=$((${mean_clat_disks_usec[$k]} / REPEAT_NO))

	mean_lat_disks_usec=$((mean_lat_disks_usec / REPEAT_NO))

	p99_lat_disks_usec=$((p99_lat_disks_usec / REPEAT_NO))

	p99_99_lat_disks_usec=$((p99_99_lat_disks_usec / REPEAT_NO))

	stdev_disks_usec=$((stdev_disks_usec / REPEAT_NO))

	mean_slat_disks_usec=$((mean_slat_disks_usec / REPEAT_NO))

	mean_clat_disks_usec=$((mean_clat_disks_usec / REPEAT_NO))

elif [[ "$PLUGIN" == "spdk-perf-bdev" ]]; then

		mean_lat_disks_usec[$k]=0

		p99_lat_disks_usec[$k]=0

		p99_99_lat_disks_usec[$k]=0

		stdev_disks_usec[$k]=0

		mean_slat_disks_usec[$k]=0

		mean_clat_disks_usec[$k]=0

	mean_lat_disks_usec=0

	p99_lat_disks_usec=0

	p99_99_lat_disks_usec=0

	stdev_disks_usec=0

	mean_slat_disks_usec=0

	mean_clat_disks_usec=0

elif [[ "$PLUGIN" == "spdk-perf-nvme" ]]; then

		mean_lat_disks_usec[$k]=$((${mean_lat_disks_usec[$k]} / REPEAT_NO))

		p99_lat_disks_usec[$k]=0

		p99_99_lat_disks_usec[$k]=0

		stdev_disks_usec[$k]=0

		mean_slat_disks_usec[$k]=0

		mean_clat_disks_usec[$k]=0

	mean_lat_disks_usec=$((mean_lat_disks_usec / REPEAT_NO))

	p99_lat_disks_usec=0

	p99_99_lat_disks_usec=0

	stdev_disks_usec=0

	mean_slat_disks_usec=0

	mean_clat_disks_usec=0

fi



	bw[$k]=$((${bw[$k]} / REPEAT_NO))



	printf "%s,%s,%s,%s,%s,%s,%s,%s,%s\n" ${k} ${iops_disks[$k]} ${mean_lat_disks_usec[$k]} ${p99_lat_disks_usec[$k]} \

		${p99_99_lat_disks_usec[$k]} ${stdev_disks_usec[$k]} ${mean_slat_disks_usec[$k]} ${mean_clat_disks_usec[$k]} ${bw[$k]} >> $result_file



	#if tested on only one numeber of disk

	if $ONEWORKLOAD; then

		break

	fi

done

printf "%s,%s,%s,%s,%s,%s,%s,%s,%s\n" ${DISKNO} ${iops_disks} ${mean_lat_disks_usec} ${p99_lat_disks_usec} \

	${p99_99_lat_disks_usec} ${stdev_disks_usec} ${mean_slat_disks_usec} ${mean_clat_disks_usec} ${bw} >> $result_file



if [ $PLUGIN = "kernel-io-uring" ]; then

	# Reload the nvme driver so that other test runs are not affected