Commit 1f0bff73 authored by Chen Wang's avatar Chen Wang Committed by Ben Walker
Browse files

test: refactor the run_test function to add detailed information 



1.Refactor the run_test function which used to add detailed information
  during run test suites and test cases.
2.Refactor the lvol feature test scripts to make sure their log is the same.
3.Users can use "run_test suite command" to run test suites
  and use "run_test case command" to run test cases.
4.Update the vhost and lvol test as example.

Change-Id: I7b6387019a861bd1c4f89b9a7712e53150aea8fa
Signed-off-by: default avatarChen Wang <chenx.wang@intel.com>
Reviewed-on: https://review.gerrithub.io/403610


Chandler-Test-Pool: SPDK Automated Test System <sys_sgsw@intel.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
parent 57a5c602

autotest.sh

+38 −38
Original line number Diff line number Diff line
@@ -85,7 +85,7 @@ timing_exit nvmf_setup 


if [ $SPDK_TEST_UNITTEST -eq 1 ]; then

	timing_enter unittest

	run_test ./test/unit/unittest.sh

	run_test suite ./test/unit/unittest.sh

	report_test_completion "unittest"

	timing_exit unittest

fi
@@ -93,109 +93,109 @@ fi 
timing_enter lib



if [ $SPDK_TEST_BLOCKDEV -eq 1 ]; then

	run_test test/bdev/blockdev.sh

	run_test suite test/bdev/blockdev.sh

	if [ $(uname -s) = Linux ]; then

		run_test test/bdev/bdevjson/json_config.sh

		run_test suite test/bdev/bdevjson/json_config.sh

		if modprobe -n nbd; then

			run_test test/bdev/nbdjson/json_config.sh

			run_test suite test/bdev/nbdjson/json_config.sh

		fi

	fi

fi



if [ $SPDK_TEST_EVENT -eq 1 ]; then

	run_test test/event/event.sh

	run_test suite test/event/event.sh

fi



if [ $SPDK_TEST_NVME -eq 1 ]; then

	run_test test/nvme/nvme.sh

	run_test suite test/nvme/nvme.sh

	if [ $SPDK_TEST_NVME_CLI -eq 1 ]; then

		run_test test/nvme/spdk_nvme_cli.sh

		run_test suite test/nvme/spdk_nvme_cli.sh

	fi

	# Only test hotplug without ASAN enabled. Since if it is

	# enabled, it catches SEGV earlier than our handler which

	# breaks the hotplug logic

	if [ $SPDK_RUN_ASAN -eq 0 ]; then

		run_test test/nvme/hotplug.sh intel

		run_test suite test/nvme/hotplug.sh intel

	fi

fi



run_test test/env/env.sh

run_test suite test/env/env.sh



if [ $SPDK_TEST_IOAT -eq 1 ]; then

	run_test test/ioat/ioat.sh

	run_test suite test/ioat/ioat.sh

fi



timing_exit lib



if [ $SPDK_TEST_ISCSI -eq 1 ]; then

	run_test ./test/iscsi_tgt/iscsi_tgt.sh posix

	run_test ./test/iscsi_tgt/iscsijson/json_config.sh

	run_test suite ./test/iscsi_tgt/iscsi_tgt.sh posix

	run_test suite ./test/iscsi_tgt/iscsijson/json_config.sh

fi



if [ $SPDK_TEST_BLOBFS -eq 1 ]; then

	run_test ./test/blobfs/rocksdb/rocksdb.sh

	run_test ./test/blobstore/blobstore.sh

	run_test suite ./test/blobfs/rocksdb/rocksdb.sh

	run_test suite ./test/blobstore/blobstore.sh

fi



if [ $SPDK_TEST_NVMF -eq 1 ]; then

	run_test ./test/nvmf/nvmf.sh

	run_test ./test/nvmf/nvmfjson/json_config.sh

	run_test suite ./test/nvmf/nvmf.sh

	run_test suite ./test/nvmf/nvmfjson/json_config.sh

fi



if [ $SPDK_TEST_VHOST -eq 1 ]; then

	timing_enter vhost

	timing_enter negative

	run_test ./test/vhost/spdk_vhost.sh --negative

	run_test suite ./test/vhost/spdk_vhost.sh --negative

	timing_exit negative



	timing_enter vhost_json_config

	run_test ./test/vhost/json_config/json_config.sh

	run_test suite ./test/vhost/json_config/json_config.sh

	timing_exit vhost_json_config



	if [ $RUN_NIGHTLY -eq 1 ]; then

		timing_enter integrity_blk

		run_test ./test/vhost/spdk_vhost.sh --integrity-blk

		run_test suite ./test/vhost/spdk_vhost.sh --integrity-blk

		timing_exit integrity_blk



		timing_enter integrity

		run_test ./test/vhost/spdk_vhost.sh --integrity

		run_test suite ./test/vhost/spdk_vhost.sh --integrity

		timing_exit integrity



		timing_enter fs_integrity_scsi

		run_test ./test/vhost/spdk_vhost.sh --fs-integrity-scsi

		run_test suite ./test/vhost/spdk_vhost.sh --fs-integrity-scsi

		timing_exit fs_integrity_scsi



		timing_enter fs_integrity_blk

		run_test ./test/vhost/spdk_vhost.sh --fs-integrity-blk

		run_test suite ./test/vhost/spdk_vhost.sh --fs-integrity-blk

		timing_exit fs_integrity_blk



		timing_enter integrity_lvol_scsi_nightly

		run_test ./test/vhost/spdk_vhost.sh --integrity-lvol-scsi-nightly

		run_test suite ./test/vhost/spdk_vhost.sh --integrity-lvol-scsi-nightly

		timing_exit integrity_lvol_scsi_nightly



		timing_enter integrity_lvol_blk_nightly

		run_test ./test/vhost/spdk_vhost.sh --integrity-lvol-blk-nightly

		run_test suite ./test/vhost/spdk_vhost.sh --integrity-lvol-blk-nightly

		timing_exit integrity_lvol_blk_nightly



		timing_enter vhost_migration

		run_test ./test/vhost/spdk_vhost.sh --migration

		run_test suite ./test/vhost/spdk_vhost.sh --migration

		timing_exit vhost_migration



		# timing_enter readonly

		# run_test ./test/vhost/spdk_vhost.sh --readonly

		# run_test suite ./test/vhost/spdk_vhost.sh --readonly

		# timing_exit readonly

	fi



	timing_enter integrity_lvol_scsi

	run_test ./test/vhost/spdk_vhost.sh --integrity-lvol-scsi

	run_test suite ./test/vhost/spdk_vhost.sh --integrity-lvol-scsi

	timing_exit integrity_lvol_scsi



	timing_enter integrity_lvol_blk

	run_test ./test/vhost/spdk_vhost.sh --integrity-lvol-blk

	run_test suite ./test/vhost/spdk_vhost.sh --integrity-lvol-blk

	timing_exit integrity_lvol_blk



	timing_enter spdk_cli

	run_test ./test/spdkcli/vhost.sh

	run_test suite ./test/spdkcli/vhost.sh

	timing_exit spdk_cli



	timing_exit vhost
@@ -208,27 +208,27 @@ if [ $SPDK_TEST_LVOL -eq 1 ]; then 
	test_cases+="600,601,650,651,652,654,655,"

	test_cases+="700,701,702,750,751,752,753,754,755,756,757,758,759,"

	test_cases+="800,801,802,803,804,10000"

	run_test ./test/lvol/lvol.sh --test-cases=$test_cases

	run_test suite ./test/lvol/lvol.sh --test-cases=$test_cases

	report_test_completion "lvol"

	timing_exit lvol

fi



if [ $SPDK_TEST_VHOST_INIT -eq 1 ]; then

	run_test ./test/vhost/initiator/blockdev.sh

	run_test ./test/vhost/initiator/json_config.sh

	run_test ./test/spdkcli/virtio.sh

	run_test suite ./test/vhost/initiator/blockdev.sh

	run_test suite ./test/vhost/initiator/json_config.sh

	run_test suite ./test/spdkcli/virtio.sh

	report_test_completion "vhost_initiator"

fi



if [ $SPDK_TEST_PMDK -eq 1 ]; then

	run_test ./test/pmem/pmem.sh -x

	run_test ./test/pmem/json_config/json_config.sh

	run_test ./test/spdkcli/pmem.sh

	run_test suite ./test/pmem/pmem.sh -x

	run_test suite ./test/pmem/json_config/json_config.sh

	run_test suite ./test/spdkcli/pmem.sh

fi



if [ $SPDK_TEST_RBD -eq 1 ]; then

	run_test ./test/bdev/bdevjson/rbd_json_config.sh

	run_test ./test/spdkcli/rbd.sh

	run_test suite ./test/bdev/bdevjson/rbd_json_config.sh

	run_test suite ./test/spdkcli/rbd.sh

fi



timing_enter cleanup

test/common/autotest_common.sh

+4 −2
Original line number Diff line number Diff line
@@ -424,14 +424,16 @@ function kill_stub() { 


function run_test() {

	set +x

	local test_type="$(echo $1 | tr 'a-z' 'A-Z')"

	shift

	echo "************************************"

	echo "START TEST $1"

	echo "START TEST $test_type $@"

	echo "************************************"

	set -x

	time "$@"

	set +x

	echo "************************************"

	echo "END TEST $1"

	echo "END TEST $test_type $@"

	echo "************************************"

	set -x

}

test/iscsi_tgt/iscsi_tgt.sh

+15 −15
Original line number Diff line number Diff line
@@ -32,24 +32,24 @@ create_veth_interfaces $TEST_TYPE 
start_stub "-s 2048 -i 0 -m $ISCSI_TEST_CORE_MASK"

trap "kill_stub; cleanup_veth_interfaces $TEST_TYPE; exit 1" SIGINT SIGTERM EXIT



run_test ./test/iscsi_tgt/calsoft/calsoft.sh

run_test ./test/iscsi_tgt/filesystem/filesystem.sh

run_test ./test/iscsi_tgt/reset/reset.sh

run_test ./test/iscsi_tgt/rpc_config/rpc_config.sh $TEST_TYPE

run_test ./test/iscsi_tgt/lvol/iscsi_lvol.sh

run_test ./test/iscsi_tgt/fio/fio.sh

run_test ./test/iscsi_tgt/qos/qos.sh

run_test suite ./test/iscsi_tgt/calsoft/calsoft.sh

run_test suite ./test/iscsi_tgt/filesystem/filesystem.sh

run_test suite ./test/iscsi_tgt/reset/reset.sh

run_test suite ./test/iscsi_tgt/rpc_config/rpc_config.sh $TEST_TYPE

run_test suite ./test/iscsi_tgt/lvol/iscsi_lvol.sh

run_test suite ./test/iscsi_tgt/fio/fio.sh

run_test suite ./test/iscsi_tgt/qos/qos.sh



if [ $RUN_NIGHTLY -eq 1 ]; then

	if [ $SPDK_TEST_PMDK -eq 1 ]; then

		run_test ./test/iscsi_tgt/pmem/iscsi_pmem.sh 4096 10

		run_test suite ./test/iscsi_tgt/pmem/iscsi_pmem.sh 4096 10

	fi

	run_test ./test/iscsi_tgt/ip_migration/ip_migration.sh

	run_test ./test/iscsi_tgt/ext4test/ext4test.sh

	run_test ./test/iscsi_tgt/digests/digests.sh

	run_test suite ./test/iscsi_tgt/ip_migration/ip_migration.sh

	run_test suite ./test/iscsi_tgt/ext4test/ext4test.sh

	run_test suite ./test/iscsi_tgt/digests/digests.sh

fi

if [ $SPDK_TEST_RBD -eq 1 ]; then

	run_test ./test/iscsi_tgt/rbd/rbd.sh

	run_test suite ./test/iscsi_tgt/rbd/rbd.sh

fi



trap "cleanup_veth_interfaces $TEST_TYPE; exit 1" SIGINT SIGTERM EXIT
@@ -59,15 +59,15 @@ if [ $SPDK_TEST_NVMF -eq 1 ]; then 
	# TODO: enable remote NVMe controllers with multi-process so that

	#  we can use the stub for this test

	# Test configure remote NVMe device from rpc and conf file

	run_test ./test/iscsi_tgt/nvme_remote/fio_remote_nvme.sh

	run_test suite ./test/iscsi_tgt/nvme_remote/fio_remote_nvme.sh

fi



if [ $RUN_NIGHTLY -eq 1 ]; then

	run_test ./test/iscsi_tgt/multiconnection/multiconnection.sh

	run_test suite ./test/iscsi_tgt/multiconnection/multiconnection.sh

fi



if [ $SPDK_TEST_ISCSI_INITIATOR -eq 1 ]; then

	run_test ./test/iscsi_tgt/initiator/initiator.sh

	run_test suite ./test/iscsi_tgt/initiator/initiator.sh

fi



cleanup_veth_interfaces $TEST_TYPE

test/lvol/lvol_test.py

+0 −8
Original line number Diff line number Diff line
@@ -3,13 +3,6 @@ import sys 
from test_cases import *





def check_fail_count(fail_count, num_test):

    if not fail_count:

        print("Test: {num_test} - PASS".format(num_test=num_test))

    else:

        print("Test: {num_test} - FAIL".format(num_test=num_test))





if __name__ == "__main__":

    rpc_py = None

    total_size = None
@@ -37,7 +30,6 @@ if __name__ == "__main__": 
            fail_count = 0

            exec("fail_count += tc.test_case{num_test}"

                 "()".format(num_test=num_test))

            check_fail_count(fail_count, num_test)

            if fail_count:

                tc_failed.append(num_test)

test/lvol/test_cases.py

+9 −6
Original line number Diff line number Diff line
@@ -149,13 +149,16 @@ def case_message(func): 
            10000: 'SIGTERM',

        }

        num = int(func.__name__.strip('test_case')[:])

        print("========================================================")

        print("Test Case {num}: Start".format(num=num))

        print("Test Name: {name}".format(name=test_name[num]))

        print("========================================================")

        print("************************************")

        print("START TEST CASE {name}".format(name=test_name[num]))

        print("************************************")

        fail_count = func(*args, **kwargs)

        print("Test Case {num}: END\n".format(num=num))

        print("========================================================")

        print("************************************")

        if not fail_count:

            print("END TEST CASE {name} PASS".format(name=test_name[num]))

        else:

            print("END TEST CASE {name} FAIL".format(name=test_name[num]))

        print("************************************")

        return fail_count

    return inner