Commit ea781d6d authored by Seth Howell's avatar Seth Howell Committed by Tomasz Zawadzki
Browse files

test: report test domains for tests. 



This will allow us to run test cases under multiple suites without
clogging up the logs too much, but it will also preserve information
about which test suites were run or (more importantly) not run.

Change-Id: I2434a54a0877ae36b9f84bfab8a62653ac1172f8
Signed-off-by: default avatarSeth Howell <seth.howell@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/477367


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarKarol Latecki <karol.latecki@intel.com>
Reviewed-by: default avatarPaul Luse <paul.e.luse@intel.com>
parent 4e3104d0

autorun_post.py

+17 −11
Original line number Diff line number Diff line
@@ -19,17 +19,17 @@ def highest_value(inp): 




def generateTestCompletionTables(output_dir, completion_table):

    data_table = pd.DataFrame(completion_table, columns=["Agent", "Test", "With Asan", "With UBsan"])

    data_table = pd.DataFrame(completion_table, columns=["Agent", "Domain", "Test", "With Asan", "With UBsan"])

    data_table.to_html(os.path.join(output_dir, 'completions_table.html'))

    os.makedirs(os.path.join(output_dir, "post_process"), exist_ok=True)



    pivot_by_agent = pd.pivot_table(data_table, index=["Agent", "Test"])

    pivot_by_agent = pd.pivot_table(data_table, index=["Agent", "Domain", "Test"])

    pivot_by_agent.to_html(os.path.join(output_dir, "post_process", 'completions_table_by_agent.html'))

    pivot_by_test = pd.pivot_table(data_table, index=["Test", "Agent"])

    pivot_by_test = pd.pivot_table(data_table, index=["Domain", "Test", "Agent"])

    pivot_by_test.to_html(os.path.join(output_dir, "post_process", 'completions_table_by_test.html'))

    pivot_by_asan = pd.pivot_table(data_table, index=["Test"], values=["With Asan"], aggfunc=highest_value)

    pivot_by_asan = pd.pivot_table(data_table, index=["Domain", "Test"], values=["With Asan"], aggfunc=highest_value)

    pivot_by_asan.to_html(os.path.join(output_dir, "post_process", 'completions_table_by_asan.html'))

    pivot_by_ubsan = pd.pivot_table(data_table, index=["Test"], values=["With UBsan"], aggfunc=highest_value)

    pivot_by_ubsan = pd.pivot_table(data_table, index=["Domain", "Test"], values=["With UBsan"], aggfunc=highest_value)

    pivot_by_ubsan.to_html(os.path.join(output_dir, "post_process", 'completions_table_by_ubsan.html'))
@@ -101,12 +101,12 @@ def getCompletions(completionFile, test_list, test_completion_table): 
    ubsan_enabled = "ubsan" in completions



    for line in completions.splitlines():

        line = line.strip()

        try:

            test_list[line] = (True, asan_enabled | test_list[line][1], ubsan_enabled | test_list[line][2])

            test_completion_table.append([agent_name, line, asan_enabled, ubsan_enabled])

            domain, test_name = line.strip().split()

            test_list[test_name] = (True, asan_enabled | test_list[test_name][1], ubsan_enabled | test_list[test_name][2])

            test_completion_table.append([agent_name, domain, test_name, asan_enabled, ubsan_enabled])

            try:

                test_completion_table.remove(["None", line, False, False])

                test_completion_table.remove(["None", "None", test_name, False, False])

            except ValueError:

                continue

        except KeyError:
@@ -139,8 +139,14 @@ def aggregateCompletedTests(output_dir, repo_dir): 


    with open(testFiles[0], 'r') as raw_test_list:

        for line in raw_test_list:

            test_list[line.strip()] = (False, False, False)

            test_completion_table.append(["None", line.strip(), False, False])

            try:

                test_name = line.strip()

            except Exception:

                print("Failed to parse a test type.")

                return 1



            test_list[test_name] = (False, False, False)

            test_completion_table.append(["None", "None", test_name, False, False])



    for completionFile in completionFiles:

        getCompletions(completionFile, test_list, test_completion_table)

test/common/autotest_common.sh

+17 −2
Original line number Diff line number Diff line
@@ -591,6 +591,12 @@ function run_test() { 
	local test_name="$1"

	shift



	if [ -n "$test_domain" ]; then

		export test_domain="${test_domain}.${test_name}"

	else

		export test_domain="$test_name"

	fi



	timing_enter $test_name

	echo "************************************"

	echo "START TEST $test_name"
@@ -601,9 +607,18 @@ function run_test() { 
	echo "************************************"

	echo "END TEST $test_name"

	echo "************************************"



	echo "$test_name" >> $output_dir/test_completions.txt

	timing_exit $test_name



	export test_domain=${test_domain%"$test_name"}

	if [ -n "$test_domain" ]; then

		export test_domain=${test_domain%?}

	fi



	if [ -z "$test_domain" ]; then

		echo "top_level $test_name" >> $output_dir/test_completions.txt

	else

		echo "$test_domain $test_name" >> $output_dir/test_completions.txt

	fi

	xtrace_restore

}