Commit a562812d authored by Seth Howell's avatar Seth Howell Committed by Jim Harris
Browse files

autotest: add test completion tracking 



Change-Id: I1ca0578a010db2ff8535505bfd981cd1c368e403
Signed-off-by: default avatarSeth Howell <seth.howell@intel.com>
Reviewed-on: https://review.gerrithub.io/392240


Tested-by: default avatarSPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: default avatarDaniel Verkamp <daniel.verkamp@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
parent 431b25ec

autobuild.sh

+15 −0
Original line number Diff line number Diff line
@@ -39,7 +39,22 @@ make_timing_label='make' 
if [ $SPDK_RUN_SCANBUILD -eq 1 ] && hash scan-build; then

	scanbuild="scan-build -o $out/scan-build-tmp --status-bugs"

	make_timing_label='scanbuild_make'

	report_test_completion "scanbuild"



fi



if [ $SPDK_RUN_VALGRIND -eq 1 ]; then

	report_test_completion "valgrind"

fi



if [ $SPDK_RUN_ASAN -eq 1 ]; then

	report_test_completion "asan"

fi



if [ $SPDK_RUN_UBSAN -eq 1 ]; then

	report_test_completion "ubsan"

fi



echo $scanbuild

$MAKE $MAKEFLAGS clean

autorun_post.py

+60 −0
Original line number Diff line number Diff line
@@ -63,9 +63,69 @@ def prepDocumentation(output_dir, repo_dir): 
    shutil.move(docDir, os.path.join(output_dir, 'doc'))





def aggregateCompletedTests(output_dir, repo_dir):

    test_list = {}

    test_with_asan = {}

    test_with_ubsan = {}

    asan_enabled = False

    ubsan_enabled = False

    test_unit_with_valgrind = False

    testFilePath = os.path.join(output_dir, '**', 'all_tests.txt')

    completionFilePath = os.path.join(output_dir, '**', 'test_completions.txt')

    testFiles = glob.glob(testFilePath, recursive=True)

    completionFiles = glob.glob(completionFilePath, recursive=True)



    if len(testFiles) == 0:

        print("Unable to perform test completion aggregator. No input files.")

        return 0

    for item in testFiles:

        with open(item, 'r') as raw_test_list:

            for line in raw_test_list:

                test_list[line.strip()] = (False, False, False)

    for item in completionFiles:

        with open(item, 'r') as completion_list:

            completions = completion_list.read()



            if "asan" not in completions:

                asan_enabled = False

            else:

                asan_enabled = True



            if "ubsan" not in completions:

                ubsan_enabled = False

            else:

                ubsan_enabled = True



            if "valgrind" in completions and "unittest" in completions:

                test_unit_with_valgrind = True

            for line in completions.split('\n'):

                try:

                    test_list[line.strip()] = (True, asan_enabled | test_list[line.strip()][1], ubsan_enabled | test_list[line.strip()][1])

                except KeyError:

                    continue



    print("\n\n-----Tests Missing From Build------")

    if not test_unit_with_valgrind:

        print("UNITTEST_WITH_VALGRIND\n")

    for item in sorted(test_list):

        if test_list[item][0] is False:

            print(item)



    print("\n\n-----Tests Missing ASAN------")

    for item in sorted(test_list):

        if test_list[item][1] is False:

            print(item)



    print("\n\n-----Tests Missing UBSAN------")

    for item in sorted(test_list):

        if test_list[item][2] is False:

            print(item)





def main(output_dir, repo_dir):

    generateCoverageReport(output_dir, repo_dir)

    prepDocumentation(output_dir, repo_dir)

    aggregateCompletedTests(output_dir, repo_dir)





if __name__ == "__main__":

autotest.sh

+4 −0
Original line number Diff line number Diff line
@@ -20,6 +20,8 @@ trap "process_core; $rootdir/scripts/setup.sh reset; exit 1" SIGINT SIGTERM EXIT 


timing_enter autotest



create_test_list



src=$(readlink -f $(dirname $0))

out=$PWD

cd $src
@@ -88,6 +90,7 @@ fi 
if [ $SPDK_TEST_UNITTEST -eq 1 ]; then

	timing_enter unittest

	run_test ./unittest.sh

	report_test_completion "unittest"

	timing_exit unittest

fi
@@ -189,6 +192,7 @@ if [ $SPDK_TEST_LVOL -eq 1 ]; then 
	test_cases+="300,301,450,451,452,550,600,601,650,651,652,654,655,"

	test_cases+="700,701,10000"

	run_test ./test/lvol/lvol.sh --test-cases=$test_cases

	report_test_completion "lvol"

	timing_exit lvol

fi

test/blobfs/rocksdb/rocksdb.sh

+1 −0
Original line number Diff line number Diff line
@@ -127,4 +127,5 @@ trap - SIGINT SIGTERM EXIT 


rm -f $ROCKSDB_CONF



report_test_completion "blobfs"

timing_exit rocksdb

test/common/autotest_common.sh

+8 −0
Original line number Diff line number Diff line
@@ -170,6 +170,14 @@ function timing_finish() { 
	fi

}



function create_test_list() {

	grep -rsh --exclude="autotest_common.sh" --exclude="$rootdir/test/common/autotest_common.sh" -e "report_test_completion" $rootdir | sed 's/report_test_completion//g; s/[[:blank:]]//g; s/"//g;' > $output_dir/all_tests.txt || true

}



function report_test_completion() {

	echo "$1" >> $output_dir/test_completions.txt

}



function process_core() {

	ret=0

	for core in $(find . -type f \( -name 'core*' -o -name '*.core' \)); do