test/vhost: add live migration test case 2

	local param

	local vhost_num=0

	local vhost_conf_path=""

	local memory=1024

	for param in "$@"; do

		case $param in

				assert_number "$vhost_num"

				;;

			--conf-path=*) local vhost_conf_path="${param#*=}" ;;

			--memory=*) local memory=${param#*=} ;;

			*)

				error "Invalid parameter '$param'"

				return 1

		esac

	done

	local vhost_dir="$(get_vhost_dir $vhost_num)"

	if [[ -z "$vhost_conf_path" ]]; then

		error "Missing mandatory parameter '--conf-path'"

	cp $vhost_conf_template $vhost_conf_file

	$SPDK_BUILD_DIR/scripts/gen_nvme.sh >> $vhost_conf_file

	local cmd="$vhost_app -m $reactor_mask -p $master_core -c $vhost_conf_file -r $vhost_dir/rpc.sock"

	local cmd="$vhost_app -m $reactor_mask -p $master_core -c $vhost_conf_file -s $memory -r $vhost_dir/rpc.sock"

	notice "Loging to:   $vhost_log_file"

	notice "Config file: $vhost_conf_file"

	notice "Socket:      $vhost_socket"

	notice "Command:     $cmd"

	timing_enter vhost_start

	cd $vhost_dir; $cmd &

	vhost_pid=$!

	echo $vhost_pid > $vhost_pid_file

	notice "waiting for app to run..."

	waitforlisten "$vhost_pid" "$vhost_dir/rpc.sock"

	notice "vhost started - pid=$vhost_pid"

	timing_exit vhost_start

	rm $vhost_conf_file

}

function spdk_vhost_kill()

{

	local rc=0

	local vhost_num=0

	if [[ ! -z "$1" ]]; then

		vhost_num=$1

		return 0

	fi

	timing_enter vhost_kill

	local vhost_pid="$(cat $vhost_pid_file)"

	notice "killing vhost (PID $vhost_pid) app"

			error "ERROR: vhost was NOT killed - sending SIGABRT"

			/bin/kill -ABRT $vhost_pid

			rm $vhost_pid_file

			return 1

		fi

			rc=1

		else

			#check vhost return code, activate trap on error

			wait $vhost_pid

		fi

	elif /bin/kill -0 $vhost_pid; then

		error "vhost NOT killed - you need to kill it manually"

		return 1

		rc=1

	else

		notice "vhost was no running"

	fi

	timing_exit vhost_kill

	if [[ $rc == 0 ]]; then

		rm $vhost_pid_file

	fi

	return $rc

}

###

[global]

blocksize_range=4k-512k

iodepth=128

ioengine=libaio

filename=

group_reporting

thread

numjobs=1

direct=1

do_verify=1

verify=md5

verify_fatal=1

verify_dump=1

verify_backlog=8

[randwrite]

rw=randwrite

runtime=15

time_based

stonewall

source $SPDK_BUILD_DIR/test/nvmf/common.sh

function migration_tc2_cleanup_nvmf_tgt()

{

	local i

	if [[ ! -r "$nvmf_dir/nvmf_tgt.pid" ]]; then

		warning "Pid file '$nvmf_dir/nvmf_tgt.pid' does not exist. "

		return

	fi

	if [[ ! -z "$1" ]]; then

		trap 'error_exit "${FUNCNAME}" "${LINENO}"' INT ERR EXIT

		pkill --signal $1 -F $nvmf_dir/nvmf_tgt.pid || true

		sleep 5

		if ! pkill -F $nvmf_dir/nvmf_tgt.pid; then

			fail "failed to kill nvmf_tgt app"

		fi

	else

		pkill --signal SIGTERM -F $nvmf_dir/nvmf_tgt.pid || true

		for (( i=0; i<20; i++ )); do

			if ! pkill --signal 0 -F $nvmf_dir/nvmf_tgt.pid; then

				break

			fi

			sleep 0.5

		done

		if pkill --signal 0 -F $nvmf_dir/nvmf_tgt.pid; then

			error "nvmf_tgt failed to shutdown"

		fi

	fi

	rm $nvmf_dir/nvmf_tgt.pid

	unset -v nvmf_dir rpc_nvmf

}

function migration_tc2_cleanup_vhost_config()

{

	timing_enter migration_tc2_cleanup_vhost_config

	trap 'migration_tc2_cleanup_nvmf_tgt SIGKILL; error_exit "${FUNCNAME}" "${LINENO}"' INT ERR EXIT

	notice "Shutting down all VMs"

	vm_shutdown_all

	notice "Removing vhost devices & controllers via RPC ..."

	# Delete bdev first to remove all LUNs and SCSI targets

	$rpc_0 delete_bdev Nvme0n1

	$rpc_0 remove_vhost_controller $incoming_vm_ctrlr

	$rpc_1 delete_bdev Nvme0n1

	$rpc_1 remove_vhost_controller $target_vm_ctrlr

	notice "killing vhost app"

	spdk_vhost_kill 0

	spdk_vhost_kill 1

	unset -v incoming_vm target_vm incoming_vm_ctrlr target_vm_ctrlr

	unset -v rpc_0 rpc_1

	trap 'error_exit "${FUNCNAME}" "${LINENO}"' INT ERR EXIT

	migration_tc2_cleanup_nvmf_tgt

	timing_exit migration_tc2_cleanup_vhost_config

}

function migration_tc2_configure_vhost()

{

	timing_enter migration_tc2_configure_vhost

	# Those are global intentionaly - they will be unset in cleanup handler

	nvmf_dir="$TEST_DIR/nvmf_tgt"

	incoming_vm=1

	target_vm=2

	incoming_vm_ctrlr=naa.VhostScsi0.$incoming_vm

	target_vm_ctrlr=naa.VhostScsi0.$target_vm

	rpc_nvmf="python $SPDK_BUILD_DIR/scripts/rpc.py -s $nvmf_dir/rpc.sock"

	rpc_0="python $SPDK_BUILD_DIR/scripts/rpc.py -s $(get_vhost_dir 0)/rpc.sock"

	rpc_1="python $SPDK_BUILD_DIR/scripts/rpc.py -s $(get_vhost_dir 1)/rpc.sock"

	# Default cleanup/error handlers will not shutdown nvmf_tgt app so setup it

	# here to teardown in cleanup function

	trap 'migration_tc2_error_cleanup; error_exit "${FUNCNAME}" "${LINENO}"' INT ERR EXIT

	# Run nvmf_tgt and two vhost instances:

	# nvmf_tgt uses core id 2 (-m 0x4)

	# First uses core id 0 (vhost_0_reactor_mask=0x1)

	# Second uses core id 1 (vhost_1_reactor_mask=0x2)

	# This force to use VM 1 and 2.

	timing_enter start_nvmf_tgt

	notice "Running nvmf_tgt..."

	mkdir -p $nvmf_dir

	rm -f $nvmf_dir/*

	cp $SPDK_BUILD_DIR/test/nvmf/nvmf.conf $nvmf_dir/nvmf.conf

	$SPDK_BUILD_DIR/scripts/gen_nvme.sh >> $nvmf_dir/nvmf.conf

	$SPDK_BUILD_DIR/app/nvmf_tgt/nvmf_tgt -s 512 -m 0x4 -c $nvmf_dir/nvmf.conf -r $nvmf_dir/rpc.sock &

	local nvmf_tgt_pid=$!

	echo $nvmf_tgt_pid > $nvmf_dir/nvmf_tgt.pid

	waitforlisten "$nvmf_tgt_pid" "$nvmf_dir/rpc.sock"

	timing_exit start_nvmf_tgt

	spdk_vhost_run --conf-path=$BASE_DIR --memory=512 --vhost-num=0

	# Those are global intentionaly

	vhost_1_reactor_mask=0x2

	vhost_1_master_core=1

	spdk_vhost_run --conf-path=$BASE_DIR --memory=512 --vhost-num=1

	local rdma_ip_list=$(get_available_rdma_ips)

	local nvmf_target_ip=$(echo "$rdma_ip_list" | head -n 1)

	if [[ -z "$nvmf_target_ip" ]]; then

		fail "no NIC for nvmf target"

	fi

	notice "Configuring nvmf_tgt, vhost devices & controllers via RPC ..."

	# Construct shared bdevs and controllers

	$rpc_nvmf construct_nvmf_subsystem nqn.2016-06.io.spdk:cnode1 \

		"trtype:RDMA traddr:$nvmf_target_ip trsvcid:4420" "" -a -s SPDK00000000000001 -n Nvme0n1

	$rpc_0 construct_nvme_bdev -b Nvme0 -t rdma -f ipv4 -a $nvmf_target_ip -s 4420 -n "nqn.2016-06.io.spdk:cnode1"

	$rpc_0 construct_vhost_scsi_controller $incoming_vm_ctrlr

	$rpc_0 add_vhost_scsi_lun $incoming_vm_ctrlr 0 Nvme0n1

	$rpc_1 construct_nvme_bdev -b Nvme0 -t rdma -f ipv4 -a $nvmf_target_ip -s 4420 -n "nqn.2016-06.io.spdk:cnode1"

	$rpc_1 construct_vhost_scsi_controller $target_vm_ctrlr

	$rpc_1 add_vhost_scsi_lun $target_vm_ctrlr 0 Nvme0n1

	notice "Setting up VMs"

	vm_setup --os="$os_image" --force=$incoming_vm --disk-type=spdk_vhost_scsi --disks=VhostScsi0 \

		--migrate-to=$target_vm  --memory=1024 --vhost-num=0

	vm_setup --force=$target_vm --disk-type=spdk_vhost_scsi --disks=VhostScsi0 --incoming=$incoming_vm --memory=1024 \

		--vhost-num=1

	# Run everything

	vm_run $incoming_vm $target_vm

	# Wait only for incoming VM, as target is waiting for migration

	vm_wait_for_boot 600 $incoming_vm

	notice "Configuration done"

	timing_exit migration_tc2_configure_vhost

}

function migration_tc2_error_cleanup()

{

	trap - SIGINT ERR EXIT

	set -x

	vm_kill_all

	migration_tc2_cleanup_vhost_config

	notice "Migration TC2 FAILED"

}

function migration_tc2()

{

	# Use 2 VMs:

	# incoming VM - the one we want to migrate

	# targe VM - the one which will accept migration

	local job_file="$BASE_DIR/migration-tc2.job"

	migration_tc2_configure_vhost

	# Run fio before migration

	notice "Starting FIO"

	vm_check_scsi_location $incoming_vm

	run_fio $fio_bin --job-file="$job_file" --local --vm="${incoming_vm}$(printf ':/dev/%s' $SCSI_DISK)"

	# Wait a while to let the FIO time to issue some IO

	sleep 5

	# Check if fio is still running before migration

	if ! is_fio_running $incoming_vm; then

		vm_ssh $incoming_vm "cat /root/$(basename ${job_file}).out"

		error "FIO is not running before migration: process crashed or finished too early"

	fi

	vm_migrate $incoming_vm

	sleep 3

	# Check if fio is still running after migration

	if ! is_fio_running $target_vm; then

		vm_ssh $target_vm "cat /root/$(basename ${job_file}).out"

		error "FIO is not running after migration: process crashed or finished too early"

	fi

	notice "Waiting for fio to finish"

	local timeout=40

	while is_fio_running $target_vm; do

		sleep 1

		echo -n "."

		if (( timeout-- == 0 )); then

			error "timeout while waiting for FIO!"

		fi

	done

	notice "Fio result is:"

	vm_ssh $target_vm "cat /root/$(basename ${job_file}).out"

	migration_tc2_cleanup_vhost_config

	notice "Migration TC2 SUCCESS"

}

migration_tc2

		fail "source VM $1 or destination VM is not properly configured for live migration"

	fi

	timing_enter vm_migrate

	notice "Migrating VM $1 to VM "$(basename $target_vm_dir)

	echo -e \

		"migrate_set_speed 1g\n" \

	fi

	notice "Migration complete"

	timing_exit vm_migrate

}

function is_fio_running()

	notice "==============================="

	notice "Running Migration test case ${test_case}"

	notice "==============================="

	timing_enter migration-tc${test_case}

	source $BASE_DIR/migration-tc${test_case}.sh

	timing_exit migration-tc${test_case}

done

notice "Migration Test SUCCESS"

	-m|--migration)

		echo 'Running migration suite...'

		$WORKDIR/migration/migration.sh -x \

		--fio-bin=$FIO_BIN --os=$VM_IMAGE --test-cases=1

		--fio-bin=$FIO_BIN --os=$VM_IMAGE --test-cases=1,2

		;;

	-i|--integrity)

		echo 'Running SCSI integrity suite...'