Commit 6baf5d83 authored by Michal Berger's avatar Michal Berger Committed by Tomasz Zawadzki
Browse files

nvme/perf: Add xnvme support into benchmark scripts 



Signed-off-by: default avatarMichal Berger <michal.berger@intel.com>
Change-Id: Id77e499d25452bc07d595ed41e683c99082babd9
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/14242


Reviewed-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: default avatarKarol Latecki <karol.latecki@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
parent 47b87952

test/nvme/perf/common.sh

+29 −3
Original line number Diff line number Diff line 
#!/usr/bin/env bash

source "$rootdir/test/dd/common.sh"



function discover_bdevs() {

	local rootdir=$1
@@ -230,7 +231,7 @@ function create_fio_config() { 
		echo "gtod_reduce=1" >> $testdir/config.fio

	fi



	if [[ $PLUGIN =~ "uring" ]]; then

	if [[ $PLUGIN =~ "uring" || $PLUGIN =~ "xnvme" ]]; then

		cat <<- EOF >> $testdir/config.fio

			fixedbufs=1

			hipri=1
@@ -287,7 +288,7 @@ function create_fio_config() { 


				if [[ "$plugin" == "spdk-plugin-nvme" ]]; then

					fio_job_section+=("filename=trtype=PCIe traddr=${disks[$n]//:/.} ns=1 #NVMe NUMA Node ${disks_numa[$n]}")

				elif [[ "$plugin" == "spdk-plugin-bdev" ]]; then

				elif [[ "$plugin" == "spdk-plugin-bdev" || "$plugin" == "spdk-plugin-bdev-xnvme" ]]; then

					fio_job_section+=("filename=${disks[$n]} #NVMe NUMA Node ${disks_numa[$n]}")

				elif [[ "$plugin" =~ "kernel" ]]; then

					fio_job_section+=("filename=/dev/${disks[$n]} #NVMe NUMA Node ${disks_numa[$n]}")
@@ -393,7 +394,7 @@ function run_spdk_nvme_fio() { 
	echo "** Running fio test, this can take a while, depending on the run-time and ramp-time setting."

	if [[ "$plugin" = "spdk-plugin-nvme" ]]; then

		LD_PRELOAD=$plugin_dir/spdk_nvme $FIO_BIN $testdir/config.fio --output-format=json "${@:2}" --ioengine=spdk

	elif [[ "$plugin" = "spdk-plugin-bdev" ]]; then

	elif [[ "$plugin" = "spdk-plugin-bdev" || "$plugin" = "spdk-plugin-bdev-xnvme" ]]; then

		LD_PRELOAD=$plugin_dir/spdk_bdev $FIO_BIN $testdir/config.fio --output-format=json "${@:2}" --ioengine=spdk_bdev --spdk_json_conf=$testdir/bdev.conf --spdk_mem=4096

	fi
@@ -494,3 +495,28 @@ function verify_disk_number() { 
		false

	fi

}



function create_spdk_xnvme_bdev_conf() {

	local bdev_io_cache_size=$1 bdev_io_pool_size=$2

	local blocks block_idx io_mechanism=libaio



	(($#)) && local -A method_bdev_set_options_0



	blocks=($(get_disks))



	if [[ -n $bdev_io_cache_size ]]; then

		method_bdev_set_options_0["bdev_io_cache_size"]=$bdev_io_cache_size

	fi

	if [[ -n $bdev_io_pool_size ]]; then

		method_bdev_set_options_0["bdev_io_pool_size"]=$bdev_io_pool_size

	fi



	for block_idx in "${!blocks[@]}"; do

		local -A method_bdev_xnvme_create_$block_idx

		local -n rpc_ref=method_bdev_xnvme_create_$block_idx

		rpc_ref["filename"]=/dev/${blocks[block_idx]}

		rpc_ref["io_mechanism"]=io_uring

		rpc_ref["name"]=${blocks[block_idx]}

	done

	gen_conf > "$testdir/bdev.conf"

}

test/nvme/perf/run_perf.sh

+21 −8
Original line number Diff line number Diff line
@@ -87,8 +87,10 @@ function usage() { 
	echo "    --driver=STR            Selects tool used for testing. Choices available:"

	echo "                               - spdk-perf-nvme (SPDK nvme perf)"

	echo "                               - spdk-perf-bdev (SPDK bdev perf)"

	echo "                               - spdk-perf-xnvme-bdev (SPDK xnvme bdev perf with io_uring io_mechanism)"

	echo "                               - spdk-plugin-nvme (SPDK nvme fio plugin)"

	echo "                               - spdk-plugin-bdev (SPDK bdev fio plugin)"

	echo "                               - spdk-plugin-bdev-xnvme (SPDK bdev fio plugin with xnvme bdevs)"

	echo "                               - kernel-classic-polling"

	echo "                               - kernel-hybrid-polling"

	echo "                               - kernel-libaio"
@@ -198,14 +200,25 @@ echo "num_of_disks,iops,avg_lat[usec],p90[usec],p99[usec],p99.99[usec],stdev[use 


trap 'rm -f *.state $testdir/bdev.conf; kill $perf_pid; wait $dpdk_mem_pid; print_backtrace' ERR SIGTERM SIGABRT



if [[ "$PLUGIN" =~ "bdev" ]]; then

if [[ "$PLUGIN" =~ "xnvme" ]]; then

	create_spdk_xnvme_bdev_conf "$BDEV_CACHE" "$BDEV_POOL"

elif [[ "$PLUGIN" =~ "bdev" ]]; then

	create_spdk_bdev_conf "$BDEV_CACHE" "$BDEV_POOL"

fi



if [[ -s $testdir/bdev.conf ]]; then

	echo "INFO: Generated bdev.conf file:"

	cat $testdir/bdev.conf

fi



verify_disk_number

if [[ "$PLUGIN" =~ "xnvme" ]]; then

	DISK_NAMES=$(get_disks)

	DISKS_NUMA=$(get_numa_node "" "$DISK_NAMES")

else

	DISK_NAMES=$(get_disks $PLUGIN)

	DISKS_NUMA=$(get_numa_node $PLUGIN "$DISK_NAMES")

fi

CORES=$(get_cores "$CPUS_ALLOWED")

NO_CORES_ARRAY=($CORES)

NO_CORES=${#NO_CORES_ARRAY[@]}
@@ -214,7 +227,7 @@ if $PRECONDITIONING; then 
	preconditioning

fi



if [[ "$PLUGIN" =~ "kernel" ]]; then

if [[ "$PLUGIN" =~ "kernel" || "$PLUGIN" =~ "xnvme" ]]; then

	$rootdir/scripts/setup.sh reset

	fio_ioengine_opt="${KERNEL_ENGINES[$PLUGIN]}"
@@ -226,7 +239,7 @@ if [[ "$PLUGIN" =~ "kernel" ]]; then 
		for disk in $DISK_NAMES; do

			echo 0 > /sys/block/$disk/queue/io_poll_delay

		done

	elif [[ $PLUGIN = "kernel-io-uring" ]]; then

	elif [[ $PLUGIN = "kernel-io-uring" || $PLUGIN =~ "xnvme" ]]; then

		modprobe -rv nvme

		modprobe nvme poll_queues=8

		wait_for_nvme_reload $DISK_NAMES
@@ -304,7 +317,7 @@ mean_slat_disks_usec=0 
mean_clat_disks_usec=0

#Run each workload $REPEAT_NO times

for ((j = 0; j < REPEAT_NO; j++)); do

	if [ $PLUGIN = "spdk-perf-bdev" ]; then

	if [[ $PLUGIN == "spdk-perf-bdev" || $PLUGIN =~ "xnvme-bdev" ]]; then

		run_bdevperf > $TMP_RESULT_FILE

		read -r iops bandwidth <<< $(get_bdevperf_results)

		iops_disks=$(bc "$iops_disks + $iops")
@@ -378,7 +391,7 @@ fi 
#Write results to csv file

iops_disks=$(bc "$iops_disks / $REPEAT_NO")

bw=$(bc "$bw / $REPEAT_NO")

if [[ "$PLUGIN" =~ "plugin" ]] || [[ "$PLUGIN" =~ "kernel" ]]; then

if [[ "$PLUGIN" =~ "plugin" || "$PLUGIN" =~ "kernel" ]] && [[ ! $PLUGIN =~ "xnvme-bdev" ]]; then

	mean_lat_disks_usec=$(bc "$mean_lat_disks_usec / $REPEAT_NO")

	p90_lat_disks_usec=$(bc "$p90_lat_disks_usec / $REPEAT_NO")

	p99_lat_disks_usec=$(bc "$p99_lat_disks_usec / $REPEAT_NO")
@@ -397,7 +410,7 @@ if [[ -n "$CPUFREQ" ]]; then 
	cpupower frequency-set -g $cpu_governor

fi



if [ $PLUGIN = "kernel-io-uring" ]; then

if [[ $PLUGIN = "kernel-io-uring" || $PLUGIN =~ "xnvme" ]]; then

	# Reload the nvme driver so that other test runs are not affected

	modprobe -rv nvme

	modprobe nvme