Commit 575a291f authored by Pawel Wodkowski's avatar Pawel Wodkowski Committed by Jim Harris
Browse files

test/vhost: introduce notice, warning, error and fail message 


helpers

So we can better trace what failed.
notice - just echo to stdout
warning - just echo to stderr
error - echo to stderr and return false, so trap ERR can catch this
fail - like err but call 'exit 1' at the end so if no trap ERR is used
it will exit anyway.

Change-Id: I5c7b3682fd6c0d81c07c58a5ec965155c7593407
Signed-off-by: default avatarPawel Wodkowski <pawelx.wodkowski@intel.com>
Reviewed-on: https://review.gerrithub.io/392218


Reviewed-by: default avatarKarol Latecki <karol.latecki@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Tested-by: default avatarSPDK Automated Test System <sys_sgsw@intel.com>
parent df6e317e

test/vhost/common/common.sh

+103 −67
Original line number Diff line number Diff line 
set -e



: ${SPDK_VHOST_VERBOSE=false}



BASE_DIR=$(readlink -f $(dirname $0))



# Default running dir -> spdk/..
@@ -10,10 +12,53 @@ SPDK_BUILD_DIR=$BASE_DIR/../../../ 


SPDK_VHOST_SCSI_TEST_DIR=$TEST_DIR/vhost



function message()

{

	if ! $SPDK_VHOST_VERBOSE; then

		local verbose_out=""

	elif [[ ${FUNCNAME[2]} == "source" ]]; then

		local verbose_out=" (file $(basename ${BASH_SOURCE[1]}):${BASH_LINENO[1]})"

	else

		local verbose_out=" (function ${FUNCNAME[2]}:${BASH_LINENO[1]})"

	fi



	local msg_type="$1"

	shift

	echo -e "${msg_type}${verbose_out}: $@"

}



function fail()

{

	echo "===========" >&2

	message "FAIL" "$@" >&2

	echo "===========" >&2

	exit 1

}



function error()

{

	echo "===========" >&2

	message "ERROR" "$@" >&2

	echo "===========" >&2

	# Don't 'return 1' since the stack trace will be incomplete (why?) missing upper command.

	false

}



function warning()

{

	message "WARN" "$@" >&2

}



function notice()

{

	message "INFO" "$@"

}





# SSH key file

: ${SPDK_VHOST_SSH_KEY_FILE="$HOME/.ssh/spdk_vhost_id_rsa"}

if [[ ! -e "$SPDK_VHOST_SSH_KEY_FILE" ]]; then

	echo "Could not find SSH key file $SPDK_VHOST_SSH_KEY_FILE"

	error "Could not find SSH key file $SPDK_VHOST_SSH_KEY_FILE"

	exit 1

fi

echo "Using SSH key file $SPDK_VHOST_SSH_KEY_FILE"
@@ -39,15 +84,6 @@ else 
	set +x

fi



function error()

{

	echo "==========="

	echo -e "ERROR: $@"

	echo "==========="

	# Don't 'return 1' since the stack trace will be incomplete (why?) missing upper command.

	false

}



function spdk_vhost_run()

{

	local vhost_conf_path="$1"
@@ -57,7 +93,7 @@ function spdk_vhost_run() 
	local vhost_socket="$SPDK_VHOST_SCSI_TEST_DIR/usvhost"

	local vhost_conf_template="$vhost_conf_path/vhost.conf.in"

	local vhost_conf_file="$vhost_conf_path/vhost.conf"

	echo "INFO: starting vhost app in background"

	notice "starting vhost app in background"

	[[ -r "$vhost_pid_file" ]] && spdk_vhost_kill

	[[ -d $SPDK_VHOST_SCSI_TEST_DIR ]] && rm -f $SPDK_VHOST_SCSI_TEST_DIR/*

	mkdir -p $SPDK_VHOST_SCSI_TEST_DIR
@@ -77,18 +113,18 @@ function spdk_vhost_run() 


	local cmd="$vhost_app -m $vhost_reactor_mask -p $vhost_master_core -c $vhost_conf_file"



	echo "INFO: Loging to:   $vhost_log_file"

	echo "INFO: Config file: $vhost_conf_file"

	echo "INFO: Socket:      $vhost_socket"

	echo "INFO: Command:     $cmd"

	notice "Loging to:   $vhost_log_file"

	notice "Config file: $vhost_conf_file"

	notice "Socket:      $vhost_socket"

	notice "Command:     $cmd"



	cd $SPDK_VHOST_SCSI_TEST_DIR; $cmd &

	vhost_pid=$!

	echo $vhost_pid > $vhost_pid_file



	echo "INFO: waiting for app to run..."

	notice "waiting for app to run..."

	waitforlisten "$vhost_pid"

	echo "INFO: vhost started - pid=$vhost_pid"

	notice "vhost started - pid=$vhost_pid"



	rm $vhost_conf_file

}
@@ -98,15 +134,15 @@ function spdk_vhost_kill() 
	local vhost_pid_file="$SPDK_VHOST_SCSI_TEST_DIR/vhost.pid"



	if [[ ! -r $vhost_pid_file ]]; then

		echo "WARN: no vhost pid file found"

		warning "no vhost pid file found"

		return 0

	fi



	local vhost_pid="$(cat $vhost_pid_file)"

	echo "INFO: killing vhost (PID $vhost_pid) app"

	notice "killing vhost (PID $vhost_pid) app"



	if /bin/kill -INT $vhost_pid >/dev/null; then

		echo "INFO: sent SIGINT to vhost app - waiting 60 seconds to exit"

		notice "sent SIGINT to vhost app - waiting 60 seconds to exit"

		for ((i=0; i<60; i++)); do

			if /bin/kill -0 $vhost_pid; then

				echo "."
@@ -116,7 +152,7 @@ function spdk_vhost_kill() 
			fi

		done

		if /bin/kill -0 $vhost_pid; then

			echo "ERROR: vhost was NOT killed - sending SIGABRT"

			error "ERROR: vhost was NOT killed - sending SIGABRT"

			/bin/kill -ABRT $vhost_pid

			rm $vhost_pid_file

			return 1
@@ -125,7 +161,7 @@ function spdk_vhost_kill() 
		error "vhost NOT killed - you need to kill it manually"

		return 1

	else

		echo "INFO: vhost was no running"

		notice "vhost was no running"

	fi



	rm $vhost_pid_file
@@ -139,7 +175,7 @@ function assert_number() 
{

	[[ "$1" =~ [0-9]+ ]] && return 0



	echo "${FUNCNAME[1]}() - ${BASH_LINENO[1]}: ERROR Invalid or missing paramter: need number but got '$1'" > /dev/stderr

	error "Invalid or missing paramter: need number but got '$1'"

	return 1;

}
@@ -150,7 +186,7 @@ function vm_num_is_valid() 
{

	[[ "$1" =~ ^[0-9]+$ ]] && return 0



	echo "${FUNCNAME[1]}() - ${BASH_LINENO[1]}: ERROR Invalid or missing paramter: vm number '$1'" > /dev/stderr

	error "Invalid or missing paramter: vm number '$1'"

	return 1;

}
@@ -243,7 +279,7 @@ function vm_is_running() 
		return 0

	else

		if [[ $EUID -ne 0 ]]; then

			echo "WARNING: not root - assuming we running since can't be checked"

			warning "not root - assuming VM running since can't be checked"

			return 0

		fi
@@ -286,16 +322,16 @@ function vm_shutdown() 
	fi



	if ! vm_is_running $1; then

		echo "INFO: VM$1 ($vm_dir) is not running"

		notice "VM$1 ($vm_dir) is not running"

		return 0

	fi



	# Temporarily disabling exit flag for next ssh command, since it will

	# "fail" due to shutdown

	echo "Shutting down virtual machine $vm_dir"

	notice "Shutting down virtual machine $vm_dir"

	set +e

	vm_ssh $1 "nohup sh -c 'shutdown -h -P now'" || true

	echo "INFO: VM$1 is shutting down - wait a while to complete"

	notice "VM$1 is shutting down - wait a while to complete"

	set -e

}
@@ -308,16 +344,15 @@ function vm_kill() 
	local vm_dir="$VM_BASE_DIR/$1"



	if [[ ! -r $vm_dir/qemu.pid ]]; then

		#echo "WARN: VM$1 pid not found - not killing"

		return 0

	fi



	local vm_pid="$(cat $vm_dir/qemu.pid)"



	echo "Killing virtual machine $vm_dir (pid=$vm_pid)"

	notice "Killing virtual machine $vm_dir (pid=$vm_pid)"

	# First kill should fail, second one must fail

	if /bin/kill $vm_pid; then

		echo "INFO: process $vm_pid killed"

		notice "process $vm_pid killed"

		rm $vm_dir/qemu.pid

	elif vm_is_running $1; then

		error "Process $vm_pid NOT killed"
@@ -345,7 +380,7 @@ function vm_shutdown_all() 
		vm_shutdown $(basename $vm)

	done



	echo "INFO: Waiting for VMs to shutdown..."

	notice "Waiting for VMs to shutdown..."

	timeo=10

	while [[ $timeo -gt 0 ]]; do

		all_vms_down=1
@@ -357,7 +392,7 @@ function vm_shutdown_all() 
		done



		if [[ $all_vms_down == 1 ]]; then

			echo "INFO: All VMs successfully shut down"

			notice "All VMs successfully shut down"

			shopt -u nullglob

			return 0

		fi
@@ -408,7 +443,8 @@ function vm_setup() 


		vm_num_is_valid $vm_num || return 1

		local vm_dir="$VM_BASE_DIR/$vm_num"

		[[ -d $vm_dir ]] && echo "WARNING: removing existing VM in '$vm_dir'"

		[[ -d $vm_dir ]] && warning "removing existing VM in '$vm_dir'"

		# FIXME: why this is just echo???

		echo "rm -rf $vm_dir"

	else

		local vm_dir=""
@@ -425,7 +461,7 @@ function vm_setup() 
		return 1

	fi



	echo "INFO: Creating new VM in $vm_dir"

	notice "Creating new VM in $vm_dir"

	mkdir -p $vm_dir

	if [[ ! -r $os ]]; then

		error "file not found: $os"
@@ -446,7 +482,7 @@ function vm_setup() 


	local task_mask=${!qemu_mask_param}



	echo "INFO: TASK MASK: $task_mask"

	notice "TASK MASK: $task_mask"

	local cmd="taskset -a $task_mask $INSTALL_DIR/bin/qemu-system-x86_64 ${eol}"

	local vm_socket_offset=$(( 10000 + 100 * vm_num ))
@@ -466,7 +502,7 @@ function vm_setup() 


	#-cpu host

	local node_num=${!qemu_numa_node_param}

	echo "INFO: NUMA NODE: $node_num"

	notice "NUMA NODE: $node_num"

	cmd+="-m 1024 --enable-kvm -smp $cpu_num -vga std -vnc :$vnc_socket -daemonize -snapshot ${eol}"

	cmd+="-object memory-backend-file,id=mem,size=1G,mem-path=/dev/hugepages,share=on,prealloc=yes,host-nodes=$node_num,policy=bind ${eol}"

	cmd+="-numa node,memdev=mem ${eol}"
@@ -506,10 +542,10 @@ function vm_setup() 
							return 1

					fi



					echo "INFO: Creating Virtio disc $raw_disk"

					notice "Creating Virtio disc $raw_disk"

					dd if=/dev/zero of=$raw_disk bs=1024k count=10240

				else

					echo "INFO: Using existing image $raw_disk"

					notice "Using existing image $raw_disk"

				fi



				cmd+="-device virtio-scsi-pci ${eol}"
@@ -517,7 +553,7 @@ function vm_setup() 
				cmd+="-drive if=none,id=hd$i,file=$raw_disk,format=raw$raw_cache ${eol}"

				;;

			spdk_vhost_scsi)

				echo "INFO: using socket $SPDK_VHOST_SCSI_TEST_DIR/naa.$disk.$vm_num"

				notice "using socket $SPDK_VHOST_SCSI_TEST_DIR/naa.$disk.$vm_num"

				cmd+="-chardev socket,id=char_$disk,path=$SPDK_VHOST_SCSI_TEST_DIR/naa.$disk.$vm_num ${eol}"

				cmd+="-device vhost-user-scsi-pci,id=scsi_$disk,num_queues=$cpu_num,chardev=char_$disk ${eol}"

				;;
@@ -528,7 +564,7 @@ function vm_setup() 
					size="20G"

				fi

				disk=${disk%%_*}

				echo "INFO: using socket $SPDK_VHOST_SCSI_TEST_DIR/naa.$disk.$vm_num"

				notice "using socket $SPDK_VHOST_SCSI_TEST_DIR/naa.$disk.$vm_num"

				cmd+="-chardev socket,id=char_$disk,path=$SPDK_VHOST_SCSI_TEST_DIR/naa.$disk.$vm_num ${eol}"

				cmd+="-device vhost-user-blk-pci,num_queues=$cpu_num,chardev=char_$disk,"

				cmd+="logical_block_size=4096,size=$size ${eol}"
@@ -541,7 +577,7 @@ function vm_setup() 
					error "$disk_type - disk(wnn)=$disk does not look like WNN number"

					return 1

				fi

				echo "Using kernel vhost disk wwn=$disk"

				notice "Using kernel vhost disk wwn=$disk"

				cmd+=" -device vhost-scsi-pci,wwpn=$disk ${eol}"

				;;

			*)
@@ -554,7 +590,7 @@ function vm_setup() 
	# remove last $eol

	cmd="${cmd%\\\\\\n  }"



	echo "Saving to $vm_dir/run.sh:"

	notice "Saving to $vm_dir/run.sh:"

	(

	echo '#!/bin/bash'

	echo 'if [[ $EUID -ne 0 ]]; then '
@@ -598,7 +634,7 @@ function vm_run() 
		case "$optchar" in

			a) run_all=true ;;

			*)

				echo "vm_run Unknown param $OPTARG"

				error "Unknown param $OPTARG"

				return 1

			;;

		esac
@@ -623,11 +659,11 @@ function vm_run() 


	for vm in $vms_to_run; do

		if vm_is_running $(basename $vm); then

			echo "WARNING: VM$(basename $vm) ($vm) already running"

			warning "VM$(basename $vm) ($vm) already running"

			continue

		fi



		echo "INFO: running $vm/run.sh"

		notice "running $vm/run.sh"

		if ! $vm/run.sh; then

			error "FAILED to run vm $vm"

			return 1
@@ -646,7 +682,7 @@ function vm_wait_for_boot() 
	[[ $timeout_time -lt 10 ]] && timeout_time=10

	local timeout_time=$(date -d "+$timeout_time seconds" +%s)



	echo "Waiting for VMs to boot"

	notice "Waiting for VMs to boot"

	shift

	if [[ "$@" == "" ]]; then

		local vms_to_check="$VM_BASE_DIR/[0-9]*"
@@ -660,26 +696,26 @@ function vm_wait_for_boot() 
	for vm in $vms_to_check; do

		local vm_num=$(basename $vm)

		local i=0

		echo "INFO: waiting for VM$vm_num ($vm)"

		notice "waiting for VM$vm_num ($vm)"

		while ! vm_os_booted $vm_num; do

			if ! vm_is_running $vm_num; then

				echo

				echo "ERROR: VM $vm_num is not running"

				echo "================"

				echo "QEMU LOG:"



				warning "VM $vm_num is not running"

				warning "================"

				warning "QEMU LOG:"

				if [[ -r $vm/qemu.log ]]; then

					cat $vm/qemu.log

				else

					echo "LOG not found"

					warning "LOG not found"

				fi



				echo "VM LOG:"

				warning "VM LOG:"

				if [[ -r $vm/serial.log ]]; then

					cat $vm/serial.log

				else

					echo "LOG not found"

					warning "LOG not found"

				fi

				echo "================"

				warning "================"

				return 1

			fi
@@ -695,10 +731,10 @@ function vm_wait_for_boot() 
			sleep 1

		done

		echo ""

		echo "INFO: VM$vm_num ready"

		notice "VM$vm_num ready"

	done



	echo "INFO: all VMs ready"

	notice "all VMs ready"

	return 0

}
@@ -712,16 +748,16 @@ function vm_start_fio_server() 
			case "$OPTARG" in

				fio-bin=*) local fio_bin="${OPTARG#*=}" ;;

				readonly) local readonly="--readonly" ;;

				*) echo "Invalid argument '$OPTARG'" && return 1;;

				*) error "Invalid argument '$OPTARG'" && return 1;;

			esac

			;;

			*) echo "Invalid argument '$OPTARG'" && return 1;;

			*) error "Invalid argument '$OPTARG'" && return 1;;

		esac

	done



	shift $(( OPTIND - 1 ))

	for vm_num in $@; do

		echo "INFO: Starting fio server on VM$vm_num"

		notice "Starting fio server on VM$vm_num"

		if [[ $fio_bin != "" ]]; then

			cat $fio_bin | vm_ssh $vm_num 'cat > /root/fio; chmod +x /root/fio'

			vm_ssh $vm_num /root/fio $readonly --eta=never --server --daemonize=/root/fio.pid
@@ -757,7 +793,7 @@ function vm_check_scsi_location() 
function vm_reset_scsi_devices()

{

	for disk in "${@:2}"; do

		echo "INFO: VM$1 Performing device reset on disk $disk"

		notice "VM$1 Performing device reset on disk $disk"

		vm_ssh $1 sg_reset /dev/$disk -vNd

	done

}
@@ -819,14 +855,14 @@ function run_fio() 
#

function at_app_exit()

{

	echo "INFO: APP EXITING"

	echo "INFO: killing all VMs"

	notice "APP EXITING"

	notice "killing all VMs"

	vm_kill_all

	# Kill vhost application

	echo "INFO: killing vhost app"

	notice "killing vhost app"

	spdk_vhost_kill



	echo "INFO: EXIT DONE"

	notice "EXIT DONE"

}



function error_exit()
@@ -834,7 +870,7 @@ function error_exit() 
	trap - ERR

	print_backtrace

	set +e

	echo "Error on $1 $2"

	error "Error on $1 $2"



	at_app_exit

	exit 1

test/vhost/common/run_vhost.sh

+3 −4
Original line number Diff line number Diff line
@@ -40,12 +40,11 @@ while getopts 'xh-:' optchar; do 
done



if [[ $EUID -ne 0 ]]; then

	echo "Go away user come back as root"

	exit 1

	fail "Go away user come back as root"

fi



echo "INFO: $0"

echo

notice "$0"

notice ""



. $COMMON_DIR/common.sh

test/vhost/common/vm_run.sh

+2 −3
Original line number Diff line number Diff line
@@ -36,14 +36,13 @@ done 
. $COMMON_DIR/common.sh



if [[ $EUID -ne 0 ]]; then

	echo "Go away user come back as root"

	exit 1

	fail "Go away user come back as root"

fi



if $run_all; then

	vm_run -a

else

	shift $((OPTIND-1))

	echo "INFO: running VMs: $@"

	notice "running VMs: $@"

	vm_run "$@"

fi

test/vhost/common/vm_setup.sh

+1 −2
Original line number Diff line number Diff line
@@ -70,8 +70,7 @@ done 
[[ -z "$os" ]] && os="$TEST_DIR/debian.qcow2"

[[ $test_type =~ "spdk_vhost" ]] && [[ -z "$disk" ]] && disk="$SPDK_VHOST_SCSI_TEST_DIR/usvhost"

if [[ $test_type == "kernel_vhost" ]] && [[ -z "$disk" ]]; then

	echo "ERROR: for $test_type '--disk=WWN' is mandatory"

	exit 1

	fail "for $test_type '--disk=WWN' is mandatory"

fi



vm_setup \

test/vhost/common/vm_shutdown.sh

+4 −4
Original line number Diff line number Diff line
@@ -45,22 +45,22 @@ fi 


if $all; then

	if do_kill; then

		echo 'INFO: killing all VMs'

		notice "killing all VMs"

		vm_kill_all

	else

		echo 'INFO: shutting down all VMs'

		notice "shutting down all VMs"

		vm_shutdown_all

	fi

else

	shift $((OPTIND-1))



	if do_kill; then

		echo 'INFO: killing VMs: $@'

		notice "INFO: killing VMs: $@"

		for vm in $@; do

			vm_kill $vm

		done

	else

		echo 'INFO: shutting down all VMs'

		notice "shutting down all VMs"

		vm_shutdown_all

	fi

fi