Commit 598ba73f authored by Pawel Kaminski's avatar Pawel Kaminski Committed by Daniel Verkamp
Browse files

test/vhost: Add tests for hot-attach and hot-detach features. 



Test plan for hot-attach and hot-detach included.
File with CPU core masks for vhost and qemu updated because more
than one virtual machine is needed to run the tests.

Change-Id: I6ba02f65398d09e2ef3335c2d5b0d6c04d3e393c
Signed-off-by: default avatarPawel Kaminski <pawelx.kaminski@intel.com>
Reviewed-on: https://review.gerrithub.io/372268


Tested-by: default avatarSPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
parent eb8b1e20

test/vhost/common/autotest.config

+33 −0
Original line number Diff line number Diff line
@@ -3,3 +3,36 @@ vhost_master_core=0 


VM_0_qemu_mask=0x6

VM_0_qemu_numa_node=0



VM_1_qemu_mask=0x18

VM_1_qemu_numa_node=0



VM_2_qemu_mask=0x60

VM_2_qemu_numa_node=0



VM_3_qemu_mask=0x180

VM_3_qemu_numa_node=0



VM_4_qemu_mask=0x600

VM_4_qemu_numa_node=0



VM_5_qemu_mask=0x1800

VM_5_qemu_numa_node=0



VM_6_qemu_mask=0x1800000

VM_6_qemu_numa_node=1



VM_7_qemu_mask=0x6000000

VM_7_qemu_numa_node=1



VM_8_qemu_mask=0x18000000

VM_8_qemu_numa_node=1



VM_9_qemu_mask=0x60000000

VM_9_qemu_numa_node=1



VM_10_qemu_mask=0x180000000

VM_10_qemu_numa_node=1



VM_11_qemu_mask=0x600000000

VM_11_qemu_numa_node=1

test/vhost/common/common.sh

+35 −12
Original line number Diff line number Diff line
@@ -249,12 +249,8 @@ function vm_fio_socket() 
	cat $vm_dir/fio_socket

}



# Execute ssh command on given VM

# param $1 virtual machine number

#

function vm_ssh()

function vm_create_ssh_config()

{

	vm_num_is_valid $1 || return 1

	local ssh_config="$VM_BASE_DIR/ssh_config"

	if [[ ! -f $ssh_config ]]; then

		(
@@ -270,6 +266,16 @@ function vm_ssh() 
		echo ""

		) > $ssh_config

	fi

}



# Execute ssh command on given VM

# param $1 virtual machine number

#

function vm_ssh()

{

	vm_num_is_valid $1 || return 1

	vm_create_ssh_config

	local ssh_config="$VM_BASE_DIR/ssh_config"



	local ssh_cmd="ssh -i $SPDK_VHOST_SSH_KEY_FILE -F $ssh_config \

		-p $(vm_ssh_socket $1) 127.0.0.1"
@@ -278,6 +284,23 @@ function vm_ssh() 
	$ssh_cmd "$@"

}



# Execute scp command on given VM

# param $1 virtual machine number

#

function vm_scp()

{

	vm_num_is_valid $1 || return 1

	vm_create_ssh_config

	local ssh_config="$VM_BASE_DIR/ssh_config"



	local scp_cmd="scp -i $SPDK_VHOST_SSH_KEY_FILE -F $ssh_config \

		-P $(vm_ssh_socket $1) "



	shift

	$scp_cmd "$@"

}





# check if specified VM is running

# param $1 VM num

function vm_is_running()

test/vhost/hotplug/common.sh

0 → 100644
+174 −0
Original line number Diff line number Diff line 
#!/usr/bin/env bash

set -e

BASE_DIR=$(readlink -f $(dirname $0))

[[ -z "$TEST_DIR" ]] && TEST_DIR="$(cd $BASE_DIR/../../../../ && pwd)"



dry_run=false

no_shutdown=false

fio_bin="fio"

fio_jobs="$BASE_DIR/fio_jobs/"

test_type=spdk_vhost_scsi

reuse_vms=false

force_build=false

vms=()

used_vms=""

disk_split=""

x=""





function usage() {

    [[ ! -z $2 ]] && ( echo "$2"; echo ""; )

    echo "Shortcut script for doing automated hotattach/hotdetach test"

    echo "Usage: $(basename $1) [OPTIONS]"

    echo

    echo "-h, --help                print help and exit"

    echo "    --test-type=TYPE      Perform specified test:"

    echo "                          virtio - test host virtio-scsi-pci using file as disk image"

    echo "                          kernel_vhost - use kernel driver vhost-scsi"

    echo "                          spdk_vhost_scsi - use spdk vhost scsi"

    echo "                          spdk_vhost_blk - use spdk vhost block"

    echo "-x                        set -x for script debug"

    echo "    --fio-bin=FIO         Use specific fio binary (will be uploaded to VM)"

    echo "    --fio-jobs=           Fio configs to use for tests. Can point to a directory or"

    echo "    --work-dir=WORK_DIR   Where to find build file. Must exist. [default: $TEST_DIR]"

    echo "    --vm=NUM[,OS][,DISKS] VM configuration. This parameter might be used more than once:"

    echo "                          NUM - VM number (mandatory)"

    echo "                          OS - VM os disk path (optional)"

    echo "                          DISKS - VM os test disks/devices path (virtio - optional, kernel_vhost - mandatory)"

    echo "                          If test-type=spdk_vhost_blk then each disk can have additional size parameter, e.g."

    echo "                          --vm=X,os.qcow,DISK_size_35G; unit can be M or G; default - 20G"

    exit 0

}



while getopts 'xh-:' optchar; do

    case "$optchar" in

        -)

        case "$OPTARG" in

            help) usage $0 ;;

            work-dir=*) TEST_DIR="${OPTARG#*=}" ;;

            fio-bin=*) fio_bin="${OPTARG#*=}" ;;

            fio-jobs=*) fio_jobs="${OPTARG#*=}" ;;

            test-type=*) test_type="${OPTARG#*=}" ;;

            vm=*) vms+=("${OPTARG#*=}") ;;

            *) usage $0 "Invalid argument '$OPTARG'" ;;

        esac

        ;;

    h) usage $0 ;;

    x) set -x

        x="-x" ;;

    *) usage $0 "Invalid argument '$OPTARG'"

    esac

done

shift $(( OPTIND - 1 ))



fio_job=$BASE_DIR/fio_jobs/default_integrity.job

tmp_attach_job=$BASE_DIR/fio_jobs/fio_attach.job.tmp

tmp_detach_job=$BASE_DIR/fio_jobs/fio_detach.job.tmp

. $BASE_DIR/../common/common.sh



rpc_py="python $SPDK_BUILD_DIR/scripts/rpc.py -s 127.0.0.1 "



function print_test_fio_header() {

    echo "==============="

    echo ""

    echo "INFO: Testing..."



    echo "INFO: Running fio jobs ..."

    if [ $# -gt 0 ]; then

        echo $1

    fi

}



function run_vhost() {

    echo "==============="

    echo ""

    echo "INFO: running SPDK"

    echo ""

    $BASE_DIR/../common/run_vhost.sh $x --work-dir=$TEST_DIR --conf-dir=$BASE_DIR

    echo

}



function vms_setup() {

    for vm_conf in ${vms[@]}; do

        IFS=',' read -ra conf <<< "$vm_conf"

        setup_cmd="$BASE_DIR/../common/vm_setup.sh $x --work-dir=$TEST_DIR --test-type=$test_type"

        if [[ x"${conf[0]}" == x"" ]] || ! assert_number ${conf[0]}; then

            echo "ERROR: invalid VM configuration syntax $vm_conf"

            exit 1;

        fi



        # Sanity check if VM is not defined twice

        for vm_num in $used_vms; do

            if [[ $vm_num -eq ${conf[0]} ]]; then

                echo "ERROR: VM$vm_num defined more than twice ( $(printf "'%s' " "${vms[@]}"))!"

                exit 1

            fi

        done



        setup_cmd+=" -f ${conf[0]}"

        used_vms+=" ${conf[0]}"

        [[ x"${conf[1]}" != x"" ]] && setup_cmd+=" --os=${conf[1]}"

        [[ x"${conf[2]}" != x"" ]] && setup_cmd+=" --disk=${conf[2]}"



        $setup_cmd

    done

}



function vms_setup_and_run() {

    vms_setup

    # Run everything

    $BASE_DIR/../common/vm_run.sh $x --work-dir=$TEST_DIR $used_vms

    vm_wait_for_boot 600 $used_vms

}



function vms_prepare() {

    for vm_num in $1; do

        vm_dir=$VM_BASE_DIR/$vm_num



        qemu_mask_param="VM_${vm_num}_qemu_mask"



        host_name="VM-${vm_num}-${!qemu_mask_param}"

        echo "INFO: Setting up hostname: $host_name"

        vm_ssh $vm_num "hostname $host_name"

        vm_start_fio_server --fio-bin=$fio_bin $readonly $vm_num

    done

}



function vms_reboot_all() {

    echo "Rebooting all vms "

    for vm_num in $1; do

        vm_ssh $vm_num "reboot" || true

    done



    vm_wait_for_boot 600 $1

}



function check_fio_retcode() {

    fio_retcode=$3

    echo $1

    retcode_expected=$2

    if [ $retcode_expected == 0 ]; then

        if [ $fio_retcode != 0 ]; then

            echo "    Fio test ended with error."

            vm_shutdown_all

            spdk_vhost_kill

            exit 1

        else

            echo "    Fio test ended with success."

        fi

    else

        if [ $fio_retcode != 0 ]; then

            echo "    Fio test ended with expected error."

        else

            echo "    Fio test ended with unexpected success."

            vm_shutdown_all

            spdk_vhost_kill

            exit 1

        fi

    fi

}



function reboot_all_and_prepare() {

    vms_reboot_all $1

    vms_prepare $1

}

test/vhost/hotplug/fio_jobs/default_integrity.job

0 → 100644
+15 −0
Original line number Diff line number Diff line 
[global]

blocksize=4k

iodepth=512

iodepth_batch=128

iodepth_low=256

ioengine=libaio

group_reporting

thread

numjobs=1

direct=1

rw=randwrite

do_verify=1

verify=md5

verify_backlog=1024

runtime=10

test/vhost/hotplug/scsi_hotattach.sh

0 → 100755
+95 −0
Original line number Diff line number Diff line 
#!/usr/bin/env bash

set -e

BASE_DIR=$(readlink -f $(dirname $0))

[[ -z "$TEST_DIR" ]] && TEST_DIR="$(cd $BASE_DIR/../../../../ && pwd)"



. $BASE_DIR/common.sh



function prepare_fio_cmd_tc1() {

    print_test_fio_header



    run_fio="$fio_bin --eta=never "

    for vm_num in $1; do

        cp $fio_job $tmp_attach_job

        vm_dir=$VM_BASE_DIR/$vm_num

        vm_check_scsi_location $vm_num

        for disk in $SCSI_DISK; do

            echo "[nvme-host$disk]" >> $tmp_attach_job

            echo "filename=/dev/$disk" >> $tmp_attach_job

        done

        vm_scp $vm_num $tmp_attach_job 127.0.0.1:/root/default_integrity_discs.job

        run_fio+="--client=127.0.0.1,$(vm_fio_socket ${vm_num}) --remote-config /root/default_integrity_discs.job "

        rm $tmp_attach_job

    done

}



# Check if fio test passes on device attached to first controller.

function hotattach_tc1() {

    echo "Hotattach test case 1"



    $rpc_py add_vhost_scsi_lun naa.Nvme0n1p0.0 0 Nvme0n1p0



    sleep 3

    prepare_fio_cmd_tc1 "0"

    $run_fio

    check_fio_retcode "Hotattach test case 1: Iteration 1." 0 $?

}



# Run fio test for previously attached device.

# During test attach another device to first controller and check fio status.

function hotattach_tc2() {

    echo "Hotattach test case 2"

    prepare_fio_cmd_tc1 "0"



    $run_fio &

    last_pid=$!

    sleep 3

    $rpc_py add_vhost_scsi_lun naa.Nvme0n1p0.0 1 Nvme0n1p1

    wait $last_pid

    check_fio_retcode "Hotattach test case 2: Iteration 1." 0 $?

}



# Run fio test for previously attached devices.

# During test attach another device to second controller and check fio status.

function hotattach_tc3() {

    echo "Hotattach test case 3"

    prepare_fio_cmd_tc1 "0"



    $run_fio &

    last_pid=$!

    sleep 3

    $rpc_py add_vhost_scsi_lun naa.Nvme0n1p1.0 0 Nvme0n1p2

    wait $last_pid

    check_fio_retcode "Hotattach test case 3: Iteration 1." 0 $?

}



# Run fio test for previously attached devices.

# During test attach another device to third controller(VM2) and check fio status.

# At the end after rebooting VMs run fio test for all devices and check fio status.

function hotattach_tc4() {

    echo "Hotattach test case 4"



    prepare_fio_cmd_tc1 "0"



    $run_fio &

    last_pid=$!

    sleep 3

    $rpc_py add_vhost_scsi_lun naa.Nvme0n1p2.1 0 Nvme0n1p3

    wait $last_pid

    check_fio_retcode "Hotattach test case 4: Iteration 1." 0 $?



    prepare_fio_cmd_tc1 "0 1"

    $run_fio

    check_fio_retcode "Hotattach test case 4: Iteration 2." 0 $?



    reboot_all_and_prepare "0 1"



    prepare_fio_cmd_tc1 "0 1"

    $run_fio

    check_fio_retcode "Hotattach test case 4: Iteration 3." 0 $?

}



hotattach_tc1

hotattach_tc2

hotattach_tc3

hotattach_tc4