Commit 18fbf2e1 authored by Karol Latecki's avatar Karol Latecki Committed by Tomasz Zawadzki
Browse files

test/nvme-perf: add nvme perf option 



Allow running tests with nvme/perf. This will be needed
for comparison with bdevperf and/or fio nvme plugin.

Change-Id: Ie351d1b3c9d8eae430161779f9818c04d87e6ed7
Signed-off-by: default avatarKarol Latecki <karol.latecki@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/479821


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
parent 63a60a0c

test/nvme/perf/common.sh

+41 −2
Original line number Diff line number Diff line
@@ -6,6 +6,7 @@ ROOT_DIR=$(readlink -f $BASE_DIR/../../..) 
PLUGIN_DIR_NVME=$ROOT_DIR/examples/nvme/fio_plugin

PLUGIN_DIR_BDEV=$ROOT_DIR/examples/bdev/fio_plugin

BDEVPERF_DIR=$ROOT_DIR/test/bdev/bdevperf

NVMEPERF_DIR=$ROOT_DIR/examples/nvme/perf

. $ROOT_DIR/scripts/common.sh || exit 1

. $ROOT_DIR/test/common/autotest_common.sh

NVME_FIO_RESULTS=$BASE_DIR/result.json
@@ -53,7 +54,7 @@ function get_cores_numa_node(){ 
function get_numa_node(){

	local plugin=$1

	local disks=$2

	if [[ "$plugin" == "spdk-plugin-nvme" ]]; then

	if [[ "$plugin" =~ "nvme" ]]; then

		for bdf in $disks; do

			local driver

			driver=$(grep DRIVER /sys/bus/pci/devices/$bdf/uevent |awk -F"=" '{print $2}')
@@ -82,7 +83,7 @@ function get_numa_node(){ 


function get_disks(){

	local plugin=$1

	if [[ "$plugin" == "spdk-plugin-nvme" ]]; then

	if [[ "$plugin" =~ "nvme" ]]; then

		for bdf in $(iter_pci_class_code 01 08 02); do

			driver=$(grep DRIVER /sys/bus/pci/devices/$bdf/uevent |awk -F"=" '{print $2}')

			if [ "$driver" = "vfio-pci" ] || [ "$driver" = "uio_pci_generic" ]; then
@@ -288,6 +289,26 @@ function get_bdevperf_results(){ 
	esac

}



function get_nvmeperf_results() {

	local iops

	local bw_MBs

	local mean_lat_usec

	local max_lat_usec

	local min_lat_usec



	read -r iops bw_MBs mean_lat_usec min_lat_usec max_lat_usec<<< $(cat $NVME_FIO_RESULTS | tr -s " " | grep -oP "(?<=Total : )(.*+)")



	# We need to get rid of the decimal spaces due

	# to use of arithmetic expressions instead of "bc" for calculations

	iops=${iops%.*}

	bw_MBs=${bw_MBs%.*}

	mean_lat_usec=${mean_lat_usec%.*}

	min_lat_usec=${min_lat_usec%.*}

	max_lat_usec=${max_lat_usec%.*}



	echo "$iops $(bc <<< "$bw_MBs * 1024") $mean_lat_usec $min_lat_usec $max_lat_usec"

}



function run_spdk_nvme_fio(){

	local plugin=$1

	echo "** Running fio test, this can take a while, depending on the run-time and ramp-time setting."
@@ -314,6 +335,23 @@ function run_bdevperf(){ 
	sleep 1

}



function run_nvmeperf() {

	# Prepare -r argument string for nvme perf command

	local r_opt

	local disks



	# Limit the number of disks to $1 if needed

	disks=( $(get_disks nvme) )

	disks=( "${disks[@]:0:$1}" )

	r_opt=$(printf -- ' -r "trtype:PCIe traddr:%s"' "${disks[@]}")



	echo "** Running nvme perf test, this can take a while, depending on the run-time setting."



	# Run command in separate shell as this solves quoting issues related to r_opt var

	$SHELL -c "$NVMEPERF_DIR/perf $r_opt -q $IODEPTH -o $BLK_SIZE -w $RW -M $MIX -t $RUNTIME -c [$CPUS_ALLOWED]"

	sleep 1

}



function wait_for_nvme_reload() {

	local nvmes=$1
@@ -368,6 +406,7 @@ function usage() 
	echo

	echo "Test setup parameters:"

	echo "    --driver=STR          Selects tool used for testing. Choices available:"

	echo "                             - spdk-perf-nvme (SPDK nvme perf)"

	echo "                             - spdk-perf-bdev (SPDK bdev perf)"

	echo "                             - spdk-plugin-nvme (SPDK nvme fio plugin)"

	echo "                             - spdk-plugin-bdev (SPDK bdev fio plugin)"

test/nvme/perf/run_perf.sh

+20 −2
Original line number Diff line number Diff line
@@ -123,6 +123,17 @@ do 
			run_bdevperf > $NVME_FIO_RESULTS

			iops_disks[$k]=$((${iops_disks[$k]} + $(get_bdevperf_results iops)))

			bw[$k]=$((${bw[$k]} + $(get_bdevperf_results bw_Kibs)))

			cp $NVME_FIO_RESULTS $BASE_DIR/results/$result_dir/perf_results_${MIX}_${PLUGIN}_${NO_CORES}cpus_${DATE}_${k}_disks_${j}.output

		elif [ $PLUGIN = "spdk-perf-nvme" ]; then

			run_nvmeperf $k > $NVME_FIO_RESULTS

			read -r iops bandwidth mean_lat min_lat max_lat <<< $(get_nvmeperf_results)



			iops_disks[$k]=$((${iops_disks[$k]} + iops))

			bw[$k]=$((${bw[$k]} + bandwidth))

			mean_lat_disks_usec[$k]=$((${mean_lat_disks_usec[$k]} + mean_lat))

			min_lat_disks_usec[$k]=$((${min_lat_disks_usec[$k]} + min_lat))

			max_lat_disks_usec[$k]=$((${max_lat_disks_usec[$k]} + max_lat))



			cp $NVME_FIO_RESULTS $BASE_DIR/results/$result_dir/perf_results_${MIX}_${PLUGIN}_${NO_CORES}cpus_${DATE}_${k}_disks_${j}.output

		else

			create_fio_config $k $PLUGIN "$DISK_NAMES" "$DISKS_NUMA" "$CORES"
@@ -178,20 +189,27 @@ for (( k=DISKNO; k >= 1; k-=2 )) 
do

	iops_disks[$k]=$((${iops_disks[$k]} / REPEAT_NO))



	if [[ $PLUGIN != "spdk-perf-bdev" ]]; then

	if [[ "$PLUGIN" =~ "plugin" ]]; then

		mean_lat_disks_usec[$k]=$((${mean_lat_disks_usec[$k]} / REPEAT_NO))

		p99_lat_disks_usec[$k]=$((${p99_lat_disks_usec[$k]} / REPEAT_NO))

		p99_99_lat_disks_usec[$k]=$((${p99_99_lat_disks_usec[$k]} / REPEAT_NO))

		stdev_disks_usec[$k]=$((${stdev_disks_usec[$k]} / REPEAT_NO))

		mean_slat_disks_usec[$k]=$((${mean_slat_disks_usec[$k]} / REPEAT_NO))

		mean_clat_disks_usec[$k]=$((${mean_clat_disks_usec[$k]} / REPEAT_NO))

	else

	elif [[ "$PLUGIN" == "spdk-perf-bdev" ]]; then

		mean_lat_disks_usec[$k]=0

		p99_lat_disks_usec[$k]=0

		p99_99_lat_disks_usec[$k]=0

		stdev_disks_usec[$k]=0

		mean_slat_disks_usec[$k]=0

		mean_clat_disks_usec[$k]=0

	elif [[ "$PLUGIN" == "spdk-perf-nvme" ]]; then

		mean_lat_disks_usec[$k]=$((${mean_lat_disks_usec[$k]} / REPEAT_NO))

		p99_lat_disks_usec[$k]=0

		p99_99_lat_disks_usec[$k]=0

		stdev_disks_usec[$k]=0

		mean_slat_disks_usec[$k]=0

		mean_clat_disks_usec[$k]=0

	fi



	bw[$k]=$((${bw[$k]} / REPEAT_NO))