Commit 3a359b79 authored by Karol Latecki's avatar Karol Latecki Committed by Tomasz Zawadzki
Browse files

scripts/nvmf_perf: move results parsing from Target class 



Results parsing is not Target specific. Result
files get copied from initiator the system where
the script is run. In future we might want to
add a feature to work with remote Targets which
would make the parse method fail.

Signed-off-by: default avatarKarol Latecki <karol.latecki@intel.com>
Change-Id: I215391e0ff7a3ac0881f5a092513dd087fbd2e2c
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/14845


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: default avatarMichal Berger <michal.berger@intel.com>
parent 93e056ba

scripts/perf/nvmf/common.py

+176 −0
Original line number Diff line number Diff line 
import os

import re

import json

import logging

from subprocess import check_output

from collections import OrderedDict

from json.decoder import JSONDecodeError





def get_nvme_devices_count():
@@ -23,3 +29,173 @@ def get_nvme_devices(): 
    output = [x for x in output.split("\n") if "nvme" in x]

    print("Done getting kernel NVMe names")

    return output





def read_json_stats(file):

    with open(file, "r") as json_data:

        data = json.load(json_data)

        job_pos = 0  # job_post = 0 because using aggregated results



        # Check if latency is in nano or microseconds to choose correct dict key

        def get_lat_unit(key_prefix, dict_section):

            # key prefix - lat, clat or slat.

            # dict section - portion of json containing latency bucket in question

            # Return dict key to access the bucket and unit as string

            for k, _ in dict_section.items():

                if k.startswith(key_prefix):

                    return k, k.split("_")[1]



        def get_clat_percentiles(clat_dict_leaf):

            if "percentile" in clat_dict_leaf:

                p99_lat = float(clat_dict_leaf["percentile"]["99.000000"])

                p99_9_lat = float(clat_dict_leaf["percentile"]["99.900000"])

                p99_99_lat = float(clat_dict_leaf["percentile"]["99.990000"])

                p99_999_lat = float(clat_dict_leaf["percentile"]["99.999000"])



                return [p99_lat, p99_9_lat, p99_99_lat, p99_999_lat]

            else:

                # Latest fio versions do not provide "percentile" results if no

                # measurements were done, so just return zeroes

                return [0, 0, 0, 0]



        read_iops = float(data["jobs"][job_pos]["read"]["iops"])

        read_bw = float(data["jobs"][job_pos]["read"]["bw"])

        lat_key, lat_unit = get_lat_unit("lat", data["jobs"][job_pos]["read"])

        read_avg_lat = float(data["jobs"][job_pos]["read"][lat_key]["mean"])

        read_min_lat = float(data["jobs"][job_pos]["read"][lat_key]["min"])

        read_max_lat = float(data["jobs"][job_pos]["read"][lat_key]["max"])

        clat_key, clat_unit = get_lat_unit("clat", data["jobs"][job_pos]["read"])

        read_p99_lat, read_p99_9_lat, read_p99_99_lat, read_p99_999_lat = get_clat_percentiles(

            data["jobs"][job_pos]["read"][clat_key])



        if "ns" in lat_unit:

            read_avg_lat, read_min_lat, read_max_lat = [x / 1000 for x in [read_avg_lat, read_min_lat, read_max_lat]]

        if "ns" in clat_unit:

            read_p99_lat = read_p99_lat / 1000

            read_p99_9_lat = read_p99_9_lat / 1000

            read_p99_99_lat = read_p99_99_lat / 1000

            read_p99_999_lat = read_p99_999_lat / 1000



        write_iops = float(data["jobs"][job_pos]["write"]["iops"])

        write_bw = float(data["jobs"][job_pos]["write"]["bw"])

        lat_key, lat_unit = get_lat_unit("lat", data["jobs"][job_pos]["write"])

        write_avg_lat = float(data["jobs"][job_pos]["write"][lat_key]["mean"])

        write_min_lat = float(data["jobs"][job_pos]["write"][lat_key]["min"])

        write_max_lat = float(data["jobs"][job_pos]["write"][lat_key]["max"])

        clat_key, clat_unit = get_lat_unit("clat", data["jobs"][job_pos]["write"])

        write_p99_lat, write_p99_9_lat, write_p99_99_lat, write_p99_999_lat = get_clat_percentiles(

            data["jobs"][job_pos]["write"][clat_key])



        if "ns" in lat_unit:

            write_avg_lat, write_min_lat, write_max_lat = [x / 1000 for x in [write_avg_lat, write_min_lat, write_max_lat]]

        if "ns" in clat_unit:

            write_p99_lat = write_p99_lat / 1000

            write_p99_9_lat = write_p99_9_lat / 1000

            write_p99_99_lat = write_p99_99_lat / 1000

            write_p99_999_lat = write_p99_999_lat / 1000



    return [read_iops, read_bw, read_avg_lat, read_min_lat, read_max_lat,

            read_p99_lat, read_p99_9_lat, read_p99_99_lat, read_p99_999_lat,

            write_iops, write_bw, write_avg_lat, write_min_lat, write_max_lat,

            write_p99_lat, write_p99_9_lat, write_p99_99_lat, write_p99_999_lat]





def parse_results(results_dir, csv_file):

    files = os.listdir(results_dir)

    fio_files = filter(lambda x: ".fio" in x, files)

    json_files = [x for x in files if ".json" in x]



    headers = ["read_iops", "read_bw", "read_avg_lat_us", "read_min_lat_us", "read_max_lat_us",

               "read_p99_lat_us", "read_p99.9_lat_us", "read_p99.99_lat_us", "read_p99.999_lat_us",

               "write_iops", "write_bw", "write_avg_lat_us", "write_min_lat_us", "write_max_lat_us",

               "write_p99_lat_us", "write_p99.9_lat_us", "write_p99.99_lat_us", "write_p99.999_lat_us"]



    aggr_headers = ["iops", "bw", "avg_lat_us", "min_lat_us", "max_lat_us",

                    "p99_lat_us", "p99.9_lat_us", "p99.99_lat_us", "p99.999_lat_us"]



    header_line = ",".join(["Name", *headers])

    aggr_header_line = ",".join(["Name", *aggr_headers])



    # Create empty results file

    with open(os.path.join(results_dir, csv_file), "w") as fh:

        fh.write(aggr_header_line + "\n")

    rows = set()



    for fio_config in fio_files:

        logging.info("Getting FIO stats for %s" % fio_config)

        job_name, _ = os.path.splitext(fio_config)



        # Look in the filename for rwmixread value. Function arguments do

        # not have that information.

        # TODO: Improve this function by directly using workload params instead

        # of regexing through filenames.

        if "read" in job_name:

            rw_mixread = 1

        elif "write" in job_name:

            rw_mixread = 0

        else:

            rw_mixread = float(re.search(r"m_(\d+)", job_name).group(1)) / 100



        # If "_CPU" exists in name - ignore it

        # Initiators for the same job could have different num_cores parameter

        job_name = re.sub(r"_\d+CPU", "", job_name)

        job_result_files = [x for x in json_files if x.startswith(job_name)]

        logging.info("Matching result files for current fio config:")

        for j in job_result_files:

            logging.info("\t %s" % j)



        # There may have been more than 1 initiator used in test, need to check that

        # Result files are created so that string after last "_" separator is server name

        inits_names = set([os.path.splitext(x)[0].split("_")[-1] for x in job_result_files])

        inits_avg_results = []

        for i in inits_names:

            logging.info("\tGetting stats for initiator %s" % i)

            # There may have been more than 1 test run for this job, calculate average results for initiator

            i_results = [x for x in job_result_files if i in x]

            i_results_filename = re.sub(r"run_\d+_", "", i_results[0].replace("json", "csv"))



            separate_stats = []

            for r in i_results:

                try:

                    stats = read_json_stats(os.path.join(results_dir, r))

                    separate_stats.append(stats)

                    logging.info(stats)

                except JSONDecodeError:

                    logging.error("ERROR: Failed to parse %s results! Results might be incomplete!" % r)



            init_results = [sum(x) for x in zip(*separate_stats)]

            init_results = [x / len(separate_stats) for x in init_results]

            inits_avg_results.append(init_results)



            logging.info("\tAverage results for initiator %s" % i)

            logging.info(init_results)

            with open(os.path.join(results_dir, i_results_filename), "w") as fh:

                fh.write(header_line + "\n")

                fh.write(",".join([job_name, *["{0:.3f}".format(x) for x in init_results]]) + "\n")



        # Sum results of all initiators running this FIO job.

        # Latency results are an average of latencies from accros all initiators.

        inits_avg_results = [sum(x) for x in zip(*inits_avg_results)]

        inits_avg_results = OrderedDict(zip(headers, inits_avg_results))

        for key in inits_avg_results:

            if "lat" in key:

                inits_avg_results[key] /= len(inits_names)



        # Aggregate separate read/write values into common labels

        # Take rw_mixread into consideration for mixed read/write workloads.

        aggregate_results = OrderedDict()

        for h in aggr_headers:

            read_stat, write_stat = [float(value) for key, value in inits_avg_results.items() if h in key]

            if "lat" in h:

                _ = rw_mixread * read_stat + (1 - rw_mixread) * write_stat

            else:

                _ = read_stat + write_stat

            aggregate_results[h] = "{0:.3f}".format(_)



        rows.add(",".join([job_name, *aggregate_results.values()]))



    # Save results to file

    for row in rows:

        with open(os.path.join(results_dir, csv_file), "a") as fh:

            fh.write(row + "\n")

    logging.info("You can find the test results in the file %s" % os.path.join(results_dir, csv_file))

scripts/perf/nvmf/run_nvmf.py

+1 −172
Original line number Diff line number Diff line
@@ -4,7 +4,6 @@ 
#  All rights reserved.

#



from json.decoder import JSONDecodeError

import os

import re

import sys
@@ -18,7 +17,6 @@ import itertools 
import configparser

import time

import uuid

from collections import OrderedDict



import paramiko

import pandas as pd
@@ -476,175 +474,6 @@ class Target(Server): 
                    ip_bdev_map.append((ip, c))

        return ip_bdev_map



    @staticmethod

    def read_json_stats(file):

        with open(file, "r") as json_data:

            data = json.load(json_data)

            job_pos = 0  # job_post = 0 because using aggregated results



            # Check if latency is in nano or microseconds to choose correct dict key

            def get_lat_unit(key_prefix, dict_section):

                # key prefix - lat, clat or slat.

                # dict section - portion of json containing latency bucket in question

                # Return dict key to access the bucket and unit as string

                for k, _ in dict_section.items():

                    if k.startswith(key_prefix):

                        return k, k.split("_")[1]



            def get_clat_percentiles(clat_dict_leaf):

                if "percentile" in clat_dict_leaf:

                    p99_lat = float(clat_dict_leaf["percentile"]["99.000000"])

                    p99_9_lat = float(clat_dict_leaf["percentile"]["99.900000"])

                    p99_99_lat = float(clat_dict_leaf["percentile"]["99.990000"])

                    p99_999_lat = float(clat_dict_leaf["percentile"]["99.999000"])



                    return [p99_lat, p99_9_lat, p99_99_lat, p99_999_lat]

                else:

                    # Latest fio versions do not provide "percentile" results if no

                    # measurements were done, so just return zeroes

                    return [0, 0, 0, 0]



            read_iops = float(data["jobs"][job_pos]["read"]["iops"])

            read_bw = float(data["jobs"][job_pos]["read"]["bw"])

            lat_key, lat_unit = get_lat_unit("lat", data["jobs"][job_pos]["read"])

            read_avg_lat = float(data["jobs"][job_pos]["read"][lat_key]["mean"])

            read_min_lat = float(data["jobs"][job_pos]["read"][lat_key]["min"])

            read_max_lat = float(data["jobs"][job_pos]["read"][lat_key]["max"])

            clat_key, clat_unit = get_lat_unit("clat", data["jobs"][job_pos]["read"])

            read_p99_lat, read_p99_9_lat, read_p99_99_lat, read_p99_999_lat = get_clat_percentiles(

                data["jobs"][job_pos]["read"][clat_key])



            if "ns" in lat_unit:

                read_avg_lat, read_min_lat, read_max_lat = [x / 1000 for x in [read_avg_lat, read_min_lat, read_max_lat]]

            if "ns" in clat_unit:

                read_p99_lat = read_p99_lat / 1000

                read_p99_9_lat = read_p99_9_lat / 1000

                read_p99_99_lat = read_p99_99_lat / 1000

                read_p99_999_lat = read_p99_999_lat / 1000



            write_iops = float(data["jobs"][job_pos]["write"]["iops"])

            write_bw = float(data["jobs"][job_pos]["write"]["bw"])

            lat_key, lat_unit = get_lat_unit("lat", data["jobs"][job_pos]["write"])

            write_avg_lat = float(data["jobs"][job_pos]["write"][lat_key]["mean"])

            write_min_lat = float(data["jobs"][job_pos]["write"][lat_key]["min"])

            write_max_lat = float(data["jobs"][job_pos]["write"][lat_key]["max"])

            clat_key, clat_unit = get_lat_unit("clat", data["jobs"][job_pos]["write"])

            write_p99_lat, write_p99_9_lat, write_p99_99_lat, write_p99_999_lat = get_clat_percentiles(

                data["jobs"][job_pos]["write"][clat_key])



            if "ns" in lat_unit:

                write_avg_lat, write_min_lat, write_max_lat = [x / 1000 for x in [write_avg_lat, write_min_lat, write_max_lat]]

            if "ns" in clat_unit:

                write_p99_lat = write_p99_lat / 1000

                write_p99_9_lat = write_p99_9_lat / 1000

                write_p99_99_lat = write_p99_99_lat / 1000

                write_p99_999_lat = write_p99_999_lat / 1000



        return [read_iops, read_bw, read_avg_lat, read_min_lat, read_max_lat,

                read_p99_lat, read_p99_9_lat, read_p99_99_lat, read_p99_999_lat,

                write_iops, write_bw, write_avg_lat, write_min_lat, write_max_lat,

                write_p99_lat, write_p99_9_lat, write_p99_99_lat, write_p99_999_lat]



    def parse_results(self, results_dir, csv_file):

        files = os.listdir(results_dir)

        fio_files = filter(lambda x: ".fio" in x, files)

        json_files = [x for x in files if ".json" in x]



        headers = ["read_iops", "read_bw", "read_avg_lat_us", "read_min_lat_us", "read_max_lat_us",

                   "read_p99_lat_us", "read_p99.9_lat_us", "read_p99.99_lat_us", "read_p99.999_lat_us",

                   "write_iops", "write_bw", "write_avg_lat_us", "write_min_lat_us", "write_max_lat_us",

                   "write_p99_lat_us", "write_p99.9_lat_us", "write_p99.99_lat_us", "write_p99.999_lat_us"]



        aggr_headers = ["iops", "bw", "avg_lat_us", "min_lat_us", "max_lat_us",

                        "p99_lat_us", "p99.9_lat_us", "p99.99_lat_us", "p99.999_lat_us"]



        header_line = ",".join(["Name", *headers])

        aggr_header_line = ",".join(["Name", *aggr_headers])



        # Create empty results file

        with open(os.path.join(results_dir, csv_file), "w") as fh:

            fh.write(aggr_header_line + "\n")

        rows = set()



        for fio_config in fio_files:

            self.log.info("Getting FIO stats for %s" % fio_config)

            job_name, _ = os.path.splitext(fio_config)



            # Look in the filename for rwmixread value. Function arguments do

            # not have that information.

            # TODO: Improve this function by directly using workload params instead

            # of regexing through filenames.

            if "read" in job_name:

                rw_mixread = 1

            elif "write" in job_name:

                rw_mixread = 0

            else:

                rw_mixread = float(re.search(r"m_(\d+)", job_name).group(1)) / 100



            # If "_CPU" exists in name - ignore it

            # Initiators for the same job could have different num_cores parameter

            job_name = re.sub(r"_\d+CPU", "", job_name)

            job_result_files = [x for x in json_files if x.startswith(job_name)]

            self.log.info("Matching result files for current fio config:")

            for j in job_result_files:

                self.log.info("\t %s" % j)



            # There may have been more than 1 initiator used in test, need to check that

            # Result files are created so that string after last "_" separator is server name

            inits_names = set([os.path.splitext(x)[0].split("_")[-1] for x in job_result_files])

            inits_avg_results = []

            for i in inits_names:

                self.log.info("\tGetting stats for initiator %s" % i)

                # There may have been more than 1 test run for this job, calculate average results for initiator

                i_results = [x for x in job_result_files if i in x]

                i_results_filename = re.sub(r"run_\d+_", "", i_results[0].replace("json", "csv"))



                separate_stats = []

                for r in i_results:

                    try:

                        stats = self.read_json_stats(os.path.join(results_dir, r))

                        separate_stats.append(stats)

                        self.log.info(stats)

                    except JSONDecodeError:

                        self.log.error("ERROR: Failed to parse %s results! Results might be incomplete!" % r)



                init_results = [sum(x) for x in zip(*separate_stats)]

                init_results = [x / len(separate_stats) for x in init_results]

                inits_avg_results.append(init_results)



                self.log.info("\tAverage results for initiator %s" % i)

                self.log.info(init_results)

                with open(os.path.join(results_dir, i_results_filename), "w") as fh:

                    fh.write(header_line + "\n")

                    fh.write(",".join([job_name, *["{0:.3f}".format(x) for x in init_results]]) + "\n")



            # Sum results of all initiators running this FIO job.

            # Latency results are an average of latencies from accros all initiators.

            inits_avg_results = [sum(x) for x in zip(*inits_avg_results)]

            inits_avg_results = OrderedDict(zip(headers, inits_avg_results))

            for key in inits_avg_results:

                if "lat" in key:

                    inits_avg_results[key] /= len(inits_names)



            # Aggregate separate read/write values into common labels

            # Take rw_mixread into consideration for mixed read/write workloads.

            aggregate_results = OrderedDict()

            for h in aggr_headers:

                read_stat, write_stat = [float(value) for key, value in inits_avg_results.items() if h in key]

                if "lat" in h:

                    _ = rw_mixread * read_stat + (1 - rw_mixread) * write_stat

                else:

                    _ = read_stat + write_stat

                aggregate_results[h] = "{0:.3f}".format(_)



            rows.add(",".join([job_name, *aggregate_results.values()]))



        # Save results to file

        for row in rows:

            with open(os.path.join(results_dir, csv_file), "a") as fh:

                fh.write(row + "\n")

        self.log.info("You can find the test results in the file %s" % os.path.join(results_dir, csv_file))



    def measure_sar(self, results_dir, sar_file_prefix):

        cpu_number = os.cpu_count()

        sar_idle_sum = 0
@@ -1717,7 +1546,7 @@ if __name__ == "__main__": 
            i.restore_sysctl()

            if i.enable_adq:

                i.reload_driver("ice")

        target_obj.parse_results(args.results, args.csv_filename)

        parse_results(args.results, args.csv_filename)

    finally:

        for i in initiators:

            try: