vhost_test: cleanup fiotest python script (6df7c093) · Commits · Public Repositories / spdk

test/vhost/fiotest/autotest.sh

+1 −1

Original line number	Diff line number	Diff line
		@@ -218,7 +218,7 @@ for vm_num in $used_vms; do

		run_fio+="127.0.0.1:$(cat $vm_dir/fio_socket):"
		for disk in $SCSI_DISK; do
		run_fio+="$disk:"
		run_fio+="/dev/$disk:"
		done
		run_fio="${run_fio::-1}"
		run_fio+=","

test/vhost/fiotest/run_fio.py

+83 −215

Original line number	Diff line number	Diff line
		@@ -4,55 +4,25 @@ import os
		import sys
		import getopt
		import subprocess
		import itertools
		import datetime
		import signal
		import re

		fio_bin = "fio"
		perf_vmex = False

		fio_template = """
		[global]
		ioengine=%(ioengine)s
		size=%(size)s
		filename=%(filename)s
		numjobs=%(numjobs)s
		bs=%(blocksize)s
		iodepth=%(iodepth)s
		direct=%(direct)s
		rw=%(testtype)s
		group_reporting
		thread
		%(verify)s

		[nvme-host]
		"""


		def show_help(fio_args_dict):

		def show_help():
		print("""Usage: python run_fio.py [options] [args]
		Args:
		[VMs] (ex. vm1_IP:vm1_port,vm2_IP:vm2_port,etc...)
		[fio filename arg], ex. /dev/sda)
		[VMs] (ex. vm1_IP:vm1_port:vm1_disk1:vm_disk2,vm2_IP:vm2_port:vm2_disk1,etc...)
		Options:
		-h, --help Show this message.
		-j, --job-files Paths to files with custom FIO jobs configuration.
		-F, --fio-bin Location of FIO binary (Default "fio")
		-s, --size Size of IO for job. Will be distributed among
		number of numjobs (Default: %(size)s)
		-t, --testtype Type of FIO test (Default: %(testtype)s)
		-b, --blocksize Blocksize for FIO test (Default: %(blocksize)s)
		-i, --iodepth IO depth for FIO test (Default: %(iodepth)s)
		-I, --ioengine Type of FIO ioengine to use (Default: %(ioengine)s)
		-n, --numjobs Number of threads for job (Default: %(numjobs)s)
		-D, --direct Use non-buffered IO? (Default: %(direct)s)
		-v, --verify Verify after writing to file (Default: %(verify)s)
		-f, --fio-bin Location of FIO binary (Default "fio")
		-o, --out Directory used to save generated job files and
		files with test results (Default: same dir where
		this script is located)
		-p, --perf-vmex Enable aggregating statistic for VMEXITS
		""" % fio_args_dict)
		-p, --perf-vmex Enable aggregating statistic for VMEXITS for VMs
		""")


		def exec_cmd(cmd, blocking):
		@@ -71,242 +41,140 @@ def save_file(path, mode, contents):
		fh.close()


		def prep_fio_cfg_file(out_dir, fio_cfg, vm_nb):
		job_file = os.path.join(out_dir, "fio_job_vm{0}".format(vm_nb))
		print "file {0} written".format(job_file)
		save_file(job_file, "w", fio_cfg)
		return job_file


		def calc_size(size, numjobs):
		return str(int(filter(lambda x: x.isdigit(), size)) / int(numjobs)) + \
		filter(lambda x: x.isalpha(), size)


		def cfg_product(fio_args_dict):
		return (dict(zip(fio_args_dict, x)) for
		x in itertools.product(*fio_args_dict.itervalues()))


		def run_fio(vms, fio_cfg_file, out_path):

		global perf_vmex
		# Prepare command template for FIO
		fio_cmd = fio_bin
		fio_cmd = " ".join([fio_cmd, "--eta=never"])
		print fio_cfg_file
		fio_cfg_name = (os.path.basename(fio_cfg_file)).split(".")[0]
		for i, vm in enumerate(vms):
		print("Starting thread {0} for VM: {1}".format(i, vm))
		def run_fio(vms, fio_cfg_fname, out_path, perf_vmex=False):
		global fio_bin
		fio_cfg_prefix = fio_cfg_fname.split(".")[0]

		# Build command for FIO
		fio_cmd = " ".join([fio_bin, "--eta=never"])
		for vm in vms:
		# vm[0] = IP address, vm[1] = Port number
		fio_cmd = " ".join([fio_cmd,
		"--client={0},{1}".format(vm[0], vm[1])])
		fio_cmd = " ".join([fio_cmd,
		"--remote-config /root/fio.job{0}".format(i)])

		print fio_cmd
		"--client={vm_ip},{vm_port}".format(vm_ip=vm[0], vm_port=vm[1]),
		"--remote-config /root/{cfg}".format(cfg=fio_cfg_fname)])
		print(fio_cmd)

		if perf_vmex:
		perf_dir = os.path.join(out_path, "perf_stats")
		try:
		os.mkdir(perf_dir)
		except OSError:
		pass

		# Start gathering perf statistics for host and VM guests
		perf_rec_file = os.path.join(out_path, "perf.data.kvm")
		perf_rec_file = os.path.join(perf_dir, "perf.data.kvm")
		perf_run_cmd = "perf kvm --host --guest " + \
		"-o {0} stat record -a".format(perf_rec_file)
		print perf_run_cmd
		print(perf_run_cmd)
		perf_p = exec_cmd(perf_run_cmd, blocking=False)

		# Run FIO test on VMs
		rc, out = exec_cmd(fio_cmd, blocking=True)

		# if for some reason output contains lines with "eta" - remove them
		out = re.sub(r'.+\[eta\s+\d{2}m\:\d{2}s\]', '', out)
		out = re.sub(r'.+\[eta\s+\d{2}m:\d{2}s\]', '', out)

		if rc != 0:
		print(rc, out)
		return rc
		print("ERROR! While executing FIO jobs - RC: {rc}, Err message: {out}".format(rc=rc, out=out))
		sys.exit(rc)
		else:
		print out
		save_file(os.path.join(out_path, "".join([fio_cfg_name, ".log"])), "w", out)
		# out = out[out.find("Disk"):]
		# out = out[out.find(":")+2:]
		# JSON format nos supported on Debian IMG for now, not parsing
		# data = json.loads(out)
		# pprint(data)
		pass
		print(out)
		save_file(os.path.join(out_path, ".".join([fio_cfg_prefix, "log"])), "w", out)

		if perf_vmex:
		# Stop gathering perf statistics and prepare some result files
		perf_p.send_signal(signal.SIGINT)
		perf_p.wait()

		perf_stat_cmd = "perf kvm --host " + \
		"-i {0} stat report".format(perf_rec_file)
		perf_stat_cmd = "perf kvm --host -i {perf_rec} stat report --event vmexit"\
		.format(perf_rec=perf_rec_file)

		print(" ".join([perf_stat_cmd, "--event vmexit"]))
		rc, out = exec_cmd(" ".join([perf_stat_cmd, "--event vmexit"]),
		blocking=True)

		print("VMexit host stats:")
		print("{0}".format(out))
		save_file(os.path.join(out_path, "vmexit_stats"),
		"w", "{0}".format(out))
		print("{perf_out}".format(perf_out=out))
		save_file(os.path.join(perf_dir, "vmexit_stats_" + fio_cfg_prefix),
		"w", "{perf_out}".format(perf_out=out))
		try:
		os.remove(perf_rec_file)
		except OSError:
		pass


		def main():
		global fio_bin

		abspath = os.path.abspath(__file__)
		dname = os.path.dirname(abspath)
		os.chdir(os.path.join(dname, "../../.."))

		global fio_bin
		global perf_vmex
		job_file_opt = False
		vms = []
		split_disks = []
		filenames = ""
		fio_cfgs = []
		perf_vmex = False
		out_dir = os.path.join(os.getcwd(), "fio_results")
		fio_cfg_files = []
		rc = 0

		fio_args_def = {
		'size': ["10G"],
		'numjobs': ["1"],
		'testtype': ["randread"],
		'blocksize': ["4k"],
		'iodepth': ["128"],
		'ioengine': ["libaio"],
		'direct': ["1"],
		'verify': [""]
		}

		fio_args = fio_args_def.copy()

		try:
		opts, args = getopt.getopt(sys.argv[1:], "hj:t:b:i:D:n:F:I:s:v:o:S:p",
		["help", "job-file=", "testtype=",
		"blocksize=", "iodepth=", "direct=",
		"numjobs=", "fio-bin=", "ioengine=",
		"size=", "verify=", "out=",
		"split-disks=", "perf"])
		except:
		show_help(fio_args_def)
		opts, args = getopt.getopt(sys.argv[1:], "hj:f:o:p",
		["help", "job-file=", "fio-bin=",
		"out=", "perf-vmex"])
		except getopt.GetoptError:
		show_help()
		sys.exit(1)

		for o, a in opts:
		print o, a
		if o in ("-j", "--job-file"):
		fio_cfg_files = a.split(",")
		job_file_opt = True
		fio_args = fio_args_def.copy()
		fio_cfgs = a.split(",")
		elif o in ("-h", "--help"):
		show_help(fio_args_def)
		show_help()
		sys.exit(1)
		elif o in ("-p", "--perf-vmex"):
		perf_vmex = True
		elif o in ("-o", "--out"):
		out_dir = os.path.join(a, "fio_results")
		elif o in ("-F", "--fio-bin"):
		elif o in ("-f", "--fio-bin"):
		fio_bin = a
		elif o in ("-S", "--split-disks"):
		split_disks = [x.split("-") for x in a.split(",")]
		split_disks = [[int(x) - 1 for x in y] for y in split_disks]
		print split_disks
		elif o in ("-s", "--size"):
		fio_args["size"] = a.split(",")
		elif o in ("-t", "--testtype"):
		fio_args["testtype"] = a.split(",")
		elif o in ("-b", "--blocksize"):
		fio_args["blocksize"] = a.split(",")
		elif o in ("-i", "--iodepth"):
		fio_args["iodepth"] = a.split(",")
		elif o in ("-D", "--direct"):
		fio_args["direct"] = a.split(",")
		elif o in ("-n", "--numjobs"):
		fio_args["numjobs"] = a.split(",")
		elif o in ("-I", "--ioengine"):
		fio_args["ioengine"] = a.split(",")
		elif o in ("-v", "--verify"):
		fio_args["verify"] = a.split(",")
		fio_args["verify"] = ["" if x in "0" else
		"verify=crc32" for x in fio_args["verify"]]

		if len(fio_cfgs) < 1:
		print("ERROR! No FIO jobs provided!")
		sys.exit(1)

		if len(args) < 1:
		show_help(fio_args_def)
		show_help()
		sys.exit(1)
		else:
		# Get IP, Port tuples from args and filename for fio config
		vms = [tuple(x.split(":")) for x in args[0].split(",")]
		filenames = [["/dev/" + y for y in x[2:]] for x in vms]
		vms = [x[0:2] for x in vms]
		# Get IP, port and fio 'filename' information from positional args
		for arg in args[0].split(","):
		_ = arg.split(":")
		ip, port, filenames = _[0], _[1], ":".join(_[2:])
		vms.append((ip, port, filenames))

		if not os.path.exists(out_dir):
		os.mkdir(out_dir)

		if job_file_opt is True:
		for fio_cfg in fio_cfg_files:
		print("Running job file: {0}".format(fio_cfg))
		for fio_cfg in fio_cfgs:
		fio_cfg_fname = os.path.basename(fio_cfg)
		print("Running job file: {0}".format(fio_cfg_fname))

		for i, vm in enumerate(zip(vms, filenames)):
		fnames = vm[1]
		if split_disks:
		if len(split_disks[i]) < 2:
		filename = fnames[split_disks[i][0]:split_disks[i][0] + 1]
		filename = ":".join(filename)
		else:
		filename = fnames[split_disks[i][0]:split_disks[i][1] + 1]
		filename = ":".join(filename)
		else:
		filename = ":".join(fnames)
		for i, vm in enumerate(vms):
		# VM - tuple of IP / Port / Filename for VM to run test
		print("Preparing VM {0} - {1} for FIO job".format(i, vm[0]))

		a = exec_cmd("./test/vhost/fiotest/vm_ssh.sh " +
		"{0} sh -c 'rm fio.job{1}'"
		.format(i, i), blocking=True)
		exec_cmd("./test/vhost/fiotest/vm_ssh.sh {vm_num} sh -c 'rm {cfg}'"
		.format(vm_num=i, cfg=fio_cfg_fname), blocking=True)

		for cfg in fio_cfg.split("\n"):
		with open(cfg, "r") as fh:
		lines = fh.readlines()
		for line in lines:
		# Copy FIO config to VM
		with open(fio_cfg, "r") as fio_cfg_fh:
		for line in fio_cfg_fh.readlines():
		if "filename" in line:
		line = "filename=" + filename
		a = exec_cmd("./test/vhost/fiotest/vm_ssh.sh " +
		"{0} sh -c 'echo {1} >> fio.job{2}'"
		.format(i, line.strip(), i), blocking=True)
		fh.close()
		rc = run_fio(vms, fio_cfg, out_dir)
		else:
		for cfg in cfg_product(fio_args):
		# Update fio "size" parameter so that total work done by
		# all numjobs is equal to assigned size and not size*numjobs
		cfg["size"] = calc_size(cfg["size"], cfg["numjobs"])

		# Prepare this test run FIO job file
		for i, vm in enumerate(zip(vms, filenames)):
		fnames = vm[1]
		if split_disks:
		if len(split_disks[i]) < 2:
		filename = fnames[split_disks[i][0]:split_disks[i][0] + 1]
		filename = ":".join(filename)
		else:
		filename = fnames[split_disks[i][0]:split_disks[i][1] + 1]
		filename = ":".join(filename)
		else:
		filename = ":".join(fnames)

		cfg.update({"filename": filename})
		fio_cfg = fio_template % cfg
		fio_cfg_files.append(prep_fio_cfg_file(out_dir,
		fio_cfg, i))
		a = exec_cmd("./test/vhost/fiotest/vm_ssh.sh " +
		"{0} sh -c 'rm fio.job{1}'"
		.format(i, i), blocking=True)
		for line in fio_cfg.split("\n"):
		a = exec_cmd("./test/vhost/fiotest/vm_ssh.sh " +
		"{0} sh -c 'echo {1} >> fio.job{2}'"
		.format(i, line.strip(), i), blocking=True)

		rc = run_fio(vms, cfg, out_dir)

		return rc
		line = "filename=" + vm[2]
		out = exec_cmd("./test/vhost/fiotest/vm_ssh.sh {vm_num} sh -c 'echo {line} >> {cfg}'"
		.format(vm_num=i, line=line.strip(), cfg=fio_cfg_fname), blocking=True)
		if out[0] != 0:
		print("ERROR! While copying FIO job config file to VM {vm_num} - {vm_ip}"
		.format(vm_num=1, vm_ip=vm[0]))
		sys.exit(1)

		run_fio(vms, fio_cfg_fname, out_dir, perf_vmex)

		if __name__ == "__main__":
		sys.exit(main())