Commit 01425928 authored by Karol Latecki's avatar Karol Latecki Committed by Tomasz Zawadzki
Browse files

test/vhost: fix vhost migration tests 



Bug introduced in:
9fc706f4 test/vhost: improve run_fio --local option

The original "nohup + no-wait" for spawned local VM fio
processes was probably as expected. Turns out we can't
"wait" for fio processes in some cases, and migration
is one of them.

run_fio() must start fio and exit immediately, so that
parent script can have a chance to do the migration
process while fio is still running.

Signed-off-by: default avatarKarol Latecki <karol.latecki@intel.com>
Change-Id: I3c0ab65e96bfa0b020eb42df76b14deda2b43e83
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/1937


Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarSeth Howell <seth.howell@intel.com>
Reviewed-by: default avatarMaciej Wawryk <maciejx.wawryk@intel.com>
Reviewed-by: default avatarDarek Stojaczyk <dariusz.stojaczyk@intel.com>
Reviewed-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
parent 7565da8a

test/vhost/common.sh

+6 −1
Original line number Diff line number Diff line
@@ -1070,6 +1070,7 @@ function run_fio() 
	local run_plugin_mode=false

	local fio_start_cmd

	local fio_output_format="normal"

	local wait_for_fio=true



	for arg in "$@"; do

		case "$arg" in
@@ -1087,6 +1088,7 @@ function run_fio() 
				run_server_mode=false ;;

			--json) fio_output_format="json" ;;

			--hide-results) hide_results=true ;;

			--no-wait-for-fio) wait_for_fio=false ;;

		*)

			error "Invalid argument '$arg'"

			return 1
@@ -1134,12 +1136,15 @@ function run_fio() 
			fi



			notice "Running local fio on VM $vm_num"

			vm_exec $vm_num "$vm_fio_bin --output=/root/$log_fname --output-format=$fio_output_format /root/$job_fname" &

			vm_exec $vm_num "$vm_fio_bin --output=/root/$log_fname --output-format=$fio_output_format /root/$job_fname & echo \$! > /root/fio.pid" &

			vm_exec_pids+=("$!")

		fi

	done



	if ! $run_server_mode; then

		if ! $wait_for_fio; then

			return 0

		fi

		echo "Waiting for guest fio instances to finish.."

		wait "${vm_exec_pids[@]}"

test/vhost/migration/migration-tc1.sh

+6 −4
Original line number Diff line number Diff line
@@ -55,6 +55,8 @@ function migration_tc1() 
	# incoming VM - the one we want to migrate

	# targe VM - the one which will accept migration

	local job_file="$testdir/migration-tc1.job"

	local log_file

	log_file="/root/$(basename ${job_file%%.*}).log"



	# Run vhost

	vhost_run 0
@@ -74,14 +76,14 @@ function migration_tc1() 
	notice "Starting FIO"



	vm_check_scsi_location $incoming_vm

	run_fio $fio_bin --job-file="$job_file" --local --vm="${incoming_vm}$(printf ':/dev/%s' $SCSI_DISK)"

	run_fio $fio_bin --job-file="$job_file" --no-wait-for-fio --local --vm="${incoming_vm}$(printf ':/dev/%s' $SCSI_DISK)"



	# Wait a while to let the FIO time to issue some IO

	sleep 5



	# Check if fio is still running before migration

	if ! is_fio_running $incoming_vm; then

		vm_exec $incoming_vm "cat /root/$(basename ${job_file}).out"

		vm_exec $incoming_vm "cat $log_file"

		error "FIO is not running before migration: process crashed or finished too early"

	fi
@@ -90,7 +92,7 @@ function migration_tc1() 


	# Check if fio is still running after migration

	if ! is_fio_running $target_vm; then

		vm_exec $target_vm "cat /root/$(basename ${job_file}).out"

		vm_exec $target_vm "cat $log_file"

		error "FIO is not running after migration: process crashed or finished too early"

	fi
@@ -105,7 +107,7 @@ function migration_tc1() 
	done



	notice "Fio result is:"

	vm_exec $target_vm "cat /root/$(basename ${job_file}).out"

	vm_exec $target_vm "cat $log_file"



	notice "Migration DONE"

test/vhost/migration/migration-tc2.sh

+6 −4
Original line number Diff line number Diff line
@@ -162,20 +162,22 @@ function migration_tc2() 
	# incoming VM - the one we want to migrate

	# targe VM - the one which will accept migration

	local job_file="$testdir/migration-tc2.job"

	local log_file

	log_file="/root/$(basename ${job_file%%.*}).log"



	migration_tc2_configure_vhost



	# Run fio before migration

	notice "Starting FIO"

	vm_check_scsi_location $incoming_vm

	run_fio $fio_bin --job-file="$job_file" --local --vm="${incoming_vm}$(printf ':/dev/%s' $SCSI_DISK)"

	run_fio $fio_bin --job-file="$job_file" --no-wait-for-fio --local --vm="${incoming_vm}$(printf ':/dev/%s' $SCSI_DISK)"



	# Wait a while to let the FIO time to issue some IO

	sleep 5



	# Check if fio is still running before migration

	if ! is_fio_running $incoming_vm; then

		vm_exec $incoming_vm "cat /root/$(basename ${job_file}).out"

		vm_exec $incoming_vm "cat $log_file"

		error "FIO is not running before migration: process crashed or finished too early"

	fi
@@ -184,7 +186,7 @@ function migration_tc2() 


	# Check if fio is still running after migration

	if ! is_fio_running $target_vm; then

		vm_exec $target_vm "cat /root/$(basename ${job_file}).out"

		vm_exec $target_vm "cat $log_file"

		error "FIO is not running after migration: process crashed or finished too early"

	fi
@@ -199,7 +201,7 @@ function migration_tc2() 
	done



	notice "Fio result is:"

	vm_exec $target_vm "cat /root/$(basename ${job_file}).out"

	vm_exec $target_vm "cat $log_file"



	migration_tc2_cleanup_vhost_config

	notice "Migration TC2 SUCCESS"

test/vhost/migration/migration-tc3a.sh

+3 −2
Original line number Diff line number Diff line
@@ -8,6 +8,7 @@ target_vm_ctrlr=naa.VhostScsi0.$target_vm 
share_dir=$TEST_DIR/share

spdk_repo_share_dir=$TEST_DIR/share_spdk

job_file=$testdir/migration-tc3.job

log_file="/root/$(basename ${job_file%%.*}).log"



if [ -z "$MGMT_TARGET_IP" ]; then

	error "No IP address of target is given"
@@ -206,11 +207,11 @@ function migration_tc3() 
	notice "Starting fio on local VM"

	vm_check_scsi_location $incoming_vm



	run_fio $fio_bin --job-file="$job_file" --local --vm="${incoming_vm}$(printf ':/dev/%s' $SCSI_DISK)"

	run_fio $fio_bin --job-file="$job_file" --no-wait-for-fio --local --vm="${incoming_vm}$(printf ':/dev/%s' $SCSI_DISK)"

	sleep 5



	if ! is_fio_running $incoming_vm; then

		vh_ssh $incoming_vm "cat /root/$(basename ${job_file}).out"

		vm_exec $incoming_vm "cat $log_file"

		error "Fio not running on local VM before starting migration!"

	fi

test/vhost/migration/migration-tc3b.sh

+2 −2
Original line number Diff line number Diff line
@@ -58,7 +58,7 @@ if ! vm_os_booted $target_vm; then 
fi



if ! is_fio_running $target_vm; then

	vm_exec $target_vm "cat /root/migration-tc3.job.out"

	vm_exec $target_vm "cat /root/migration-tc3.log"

	error "FIO is not running on remote server after migration!"

fi
@@ -73,7 +73,7 @@ while is_fio_running $target_vm; do 
done



notice "FIO result after migration:"

vm_exec $target_vm "cat /root/migration-tc3.job.out"

vm_exec $target_vm "cat /root/migration-tc3.log"



host_2_cleanup_vhost

echo "DONE" > $share_dir/DONE