Commit 6330ee8f authored by Karol Latecki's avatar Karol Latecki Committed by Tomasz Zawadzki
Browse files

test/nvme_perf: unify fio job section for kernel and spdk 



For some reason the routine for generating fio
job/filename sections was different for kernel and
spdk fio modes. This made fio results hard to compare
because even configuration files themselves were not
comparable.
The function was modified to create the same fio
configuration files for kernel and spdk.

While at it, iodepth calculation has been slightly
modified as well. It is now being calculated for each
[ ] job section, so that each "filename" gets the
actual iodepth value specified in test run parameters.

While generating the config file the script will add
NUMA node information as a comment for "cpus_allowed"
and "filename" sections, so that it's clearly visible
how resources are aligned. In case of cross numa
configuration additional warnings will be printed to
screen as well.

For example, a configuration file for running test with
7 NVMe drives, 3 CPUs and effective (per filename) queue
depth of 32 would look like this:

run_perf.sh --cpu-allowed=0,1,2 --iodepth=32
  --driver=kernel-classic-polling [...other params...]
--------
[filename0]
iodepth=96
cpus_allowed=0 #CPU NUMA Node X
filename=/dev/nvme0n1 #NVMe NUMA Node X
filename=/dev/nvme1n1 #NVMe NUMA Node X
filename=/dev/nvme2n1 #NVMe NUMA Node X

[filename1]
iodepth=64
cpus_allowed=1 #CPU NUMA Node X
filename=/dev/nvme3n1 #NVMe NUMA Node X
filename=/dev/nvme4n1 #NVMe NUMA Node X

[filename2]
iodepth=64
cpus_allowed=2 #CPU NUMA Node X
filename=/dev/nvme5n1 #NVMe NUMA Node X
filename=/dev/nvme6n1 #NVMe NUMA Node X
--------

Change-Id: Ida2f781fbb93c4a8c62154e711151152843ab997
Signed-off-by: default avatarKarol Latecki <karol.latecki@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/3370


Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: default avatarMichal Berger <michalx.berger@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
parent b9150897

test/nvme/perf/common.sh

+58 −71
Original line number Diff line number Diff line
@@ -203,21 +203,12 @@ function create_fio_config() { 
	local disks_numa=($4)

	local cores=($5)

	local total_disks=${#disks[@]}

	local no_cores=${#cores[@]}

	local filename=""



	local fio_job_section=()

	local num_cores=${#cores[@]}

	local disks_per_core=$((disk_no / num_cores))

	local disks_per_core_mod=$((disk_no % num_cores))

	local cores_numa

	cores_numa=($(get_cores_numa_node "$5"))

	local disks_per_core=$((disk_no / no_cores))

	local disks_per_core_mod=$((disk_no % no_cores))



	# SPDK fio plugin supports submitting/completing I/Os to multiple SSDs from a single thread.

	# Therefore, the per thread queue depth is set to the desired IODEPTH/device X the number of devices per thread.

	# TODO: Shouldn't this be applicable to running kernel fio tests as well? Because, what's the difference?

	QD=$IODEPTH

	if [[ "$PLUGIN" =~ "spdk-plugin" ]] && [[ "$NOIOSCALING" = false ]]; then

		QD=$((IODEPTH * DISKNO))

	fi

	cores_numa=($(get_cores_numa_node "${cores[*]}"))



	# Following part of this function still leverages global variables a lot.

	# It's a mix of local variables passed as aruments to function with global variables. This is messy.
@@ -230,65 +221,62 @@ function create_fio_config() { 


		rw=$RW

		rwmixread=$MIX

		iodepth=$QD

		bs=$BLK_SIZE

		runtime=$RUNTIME

		ramp_time=$RAMP_TIME

		numjobs=$NUMJOBS

		log_avg_msec=$SAMPLING_INT



	EOF



	# For kernel dirver, each disk will be alligned with all cpus on the same NUMA node

	if [[ "$plugin" =~ "kernel" ]]; then

		for ((i = 0; i < disk_no; i++)); do

			sed -i -e "\$a[filename${i}]" $testdir/config.fio

			filename="/dev/${disks[$i]}"

			sed -i -e "\$afilename=$filename" $testdir/config.fio

			cpu_used=""

			for ((j = 0; j < no_cores; j++)); do

				core_numa=${cores_numa[$j]}

				if [ "${disks_numa[$i]}" = "$core_numa" ]; then

					cpu_used+="${cores[$j]},"

				fi

			done

			sed -i -e "\$acpus_allowed=$cpu_used" $testdir/config.fio

			echo "" >> $testdir/config.fio

		done

	else

		for ((i = 0; i < no_cores; i++)); do

	for i in "${!cores[@]}"; do

		local m=0 #Counter of disks per NUMA node

		local n=0 #Counter of all disks in test

		core_numa=${cores_numa[$i]}



		total_disks_per_core=$disks_per_core

			if [ "$disks_per_core_mod" -gt "0" ]; then

		# Check how many "stray" disks are unassigned to CPU cores

		# Assign one disk to current CPU core and substract it from the total of

		# unassigned disks

		if [[ "$disks_per_core_mod" -gt "0" ]]; then

			total_disks_per_core=$((disks_per_core + 1))

			disks_per_core_mod=$((disks_per_core_mod - 1))

		fi



			if [ "$total_disks_per_core" = "0" ]; then

				break

		# SPDK fio plugin supports submitting/completing I/Os to multiple SSDs from a single thread.

		# Therefore, the per thread queue depth is set to the desired IODEPTH/device X the number of devices per thread.

		QD=$IODEPTH

		if [[ "$NOIOSCALING" = false ]]; then

			QD=$((IODEPTH * total_disks_per_core))

		fi



			sed -i -e "\$a[filename${i}]" $testdir/config.fio

			#use cpus_allowed as cpumask works only for cores 1-32

			sed -i -e "\$acpus_allowed=${cores[$i]}" $testdir/config.fio

			m=0 #counter of disks per cpu core numa

			n=0 #counter of all disks

			while [ "$m" -lt "$total_disks_per_core" ]; do

				if [ ${disks_numa[$n]} = $core_numa ]; then

					m=$((m + 1))

					if [[ "$plugin" = "spdk-plugin-nvme" ]]; then

						filename='trtype=PCIe traddr='${disks[$n]//:/.}' ns=1'

					elif [[ "$plugin" = "spdk-plugin-bdev" ]]; then

						filename=${disks[$n]}

		fio_job_section+=("")

		fio_job_section+=("[filename${i}]")

		fio_job_section+=("iodepth=$QD")

		fio_job_section+=("cpus_allowed=${cores[$i]} #CPU NUMA Node ${cores_numa[$i]}")



		while [[ "$m" -lt "$total_disks_per_core" ]]; do

			# Try to add disks to job section if it's NUMA node matches NUMA

			# for currently selected CPU

			if [[ "${disks_numa[$n]}" == "$core_numa" ]]; then

				if [[ "$plugin" == "spdk-plugin-nvme" ]]; then

					fio_job_section+=("filename=trtype=PCIe traddr=${disks[$n]//:/.} ns=1 #NVMe NUMA Node ${disks_numa[$n]}")

				elif [[ "$plugin" == "spdk-plugin-bdev" ]]; then

					fio_job_section+=("filename=${disks[$n]} #NVMe NUMA Node ${disks_numa[$n]}")

				elif [[ "$plugin" =~ "kernel" ]]; then

					fio_job_section+=("filename=/dev/${disks[$n]} #NVMe NUMA Node ${disks_numa[$n]}")

				fi

					sed -i -e "\$afilename=$filename" $testdir/config.fio

					#Mark numa of n'th disk as "x" to mark it as claimed

				m=$((m + 1))



				#Mark numa of n'th disk as "x" to mark it as claimed for next loop iterations

				disks_numa[$n]="x"

			fi

			n=$((n + 1))

				# If there is no more disks with numa node same as cpu numa node, switch to other numa node.

				if [ $n -ge $total_disks ]; then

					if [ "$core_numa" = "1" ]; then



			# If there is no more disks with numa node same as cpu numa node, switch to

			# other numa node, go back to start of loop and try again.

			if [[ $n -ge $total_disks ]]; then

				echo "WARNING! Cannot assign any more NVMes for CPU ${cores[$i]}"

				echo "NVMe assignment for this CPU will be cross-NUMA."

				if [[ "$core_numa" == "1" ]]; then

					core_numa=0

				else

					core_numa=1
@@ -296,10 +284,9 @@ function create_fio_config() { 
				n=0

			fi

		done

			echo "" >> $testdir/config.fio

	done

	fi



	printf "%s\n" "${fio_job_section[@]}" >> $testdir/config.fio

	echo "INFO: Generated fio configuration file:"

	cat $testdir/config.fio

}