Commit 2c8baa46 authored by wawryk's avatar wawryk Committed by Tomasz Zawadzki
Browse files

scripts/nvmf: add io_uring engine option for Kernel Initiator 



Enable "kernel_engine" option for Kernel Initiator
class to enable ability to use io_uring engine on
initiator side.

For NVMe-oF this has limitations as apparently it's
not possible to enable polling on the initiator side,
which in turn makes fio "hipri" option also not
possible to use.

Adding #TODO sections with notes in case this is
later fixed in Kernel NVMe driver.

Signed-off-by: default avatarwawryk <maciejx.wawryk@intel.com>
Signed-off-by: default avatarKarol Latecki <karol.latecki@intel.com>
Change-Id: I31da52946692015237263abc77d8425b5eae5b98
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/8134


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarZiye Yang <ziye.yang@intel.com>
Reviewed-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
parent 03323b09

scripts/perf/nvmf/README.md

+7 −1
Original line number Diff line number Diff line
@@ -167,7 +167,8 @@ There can be one or more `initiatorX` setting sections, depending on the test se 
  "cpus_allowed_policy": "shared",

  "num_cores": 4,

  "cpu_frequency": 2100000,

  "adq_enable": false

  "adq_enable": false,

  "kernel_engine": "io_uring"

}

```
@@ -212,6 +213,11 @@ Optional, common: 
- irq_scripts_dir - path to scripts directory of Mellanox mlnx-tools package;

  Used to run set_irq_affinity.sh script.

  Default: /usr/src/local/mlnx-tools/ofed_scripts

- kernel_engine - Select fio ioengine mode to run tests. io_uring libraries and

  io_uring capable fio binaries must be present on Initiator systems!

  Available options:

  - libaio (default)

  - io_uring



Optional, SPDK Initiator only:

scripts/perf/nvmf/run_nvmf.py

+47 −1
Original line number Diff line number Diff line
@@ -802,7 +802,7 @@ rate_iops={rate_iops} 
            ioengine = "%s/build/fio/spdk_bdev" % self.spdk_dir

            spdk_conf = "spdk_json_conf=%s/bdev.conf" % self.spdk_dir

        else:

            ioengine = "libaio"

            ioengine = self.ioengine

            spdk_conf = ""

            out = self.exec_cmd(["sudo", "nvme", "list", "|", "grep", "-E", "'SPDK|Linux'",

                                 "|", "awk", "'{print $1}'"])
@@ -837,6 +837,16 @@ rate_iops={rate_iops} 
        fio_config = fio_conf_template.format(ioengine=ioengine, spdk_conf=spdk_conf,

                                              rw=rw, rwmixread=rwmixread, block_size=block_size,

                                              ramp_time=ramp_time, run_time=run_time, rate_iops=rate_iops)



        # TODO: hipri disabled for now, as it causes fio errors:

        # io_u error on file /dev/nvme2n1: Operation not supported

        # See comment in KernelInitiator class, kernel_init_connect() function

        if hasattr(self, "ioengine") and "io_uring" in self.ioengine:

            fio_config = fio_config + """

fixedbufs=1

registerfiles=1

#hipri=1

"""

        if num_jobs:

            fio_config = fio_config + "numjobs=%s \n" % num_jobs

        if self.cpus_allowed is not None:
@@ -1172,13 +1182,25 @@ class KernelInitiator(Initiator): 


        # Defaults

        self.extra_params = ""

        self.ioengine = "libaio"



        if "extra_params" in initiator_config:

            self.extra_params = initiator_config["extra_params"]



        if "kernel_engine" in initiator_config:

            self.ioengine = initiator_config["kernel_engine"]

            if "io_uring" in self.ioengine:

                self.extra_params = "--nr-poll-queues=8"



    def __del__(self):

        self.ssh_connection.close()



    def get_connected_nvme_list(self):

        json_obj = json.loads(self.exec_cmd(["sudo", "nvme", "list", "-o", "json"]))

        nvme_list = [os.path.basename(x["DevicePath"]) for x in json_obj["Devices"]

                     if "SPDK" in x["ModelNumber"] or "Linux" in x["ModelNumber"]]

        return nvme_list



    def kernel_init_connect(self):

        self.log_print("Below connection attempts may result in error messages, this is expected!")

        for subsystem in self.subsystem_info_list:
@@ -1187,6 +1209,30 @@ class KernelInitiator(Initiator): 
                           "-s", subsystem[0], "-n", subsystem[1], "-a", subsystem[2], self.extra_params])

            time.sleep(2)



        if "io_uring" in self.ioengine:

            self.log_print("Setting block layer settings for io_uring.")



            # TODO: io_poll=1 and io_poll_delay=-1 params not set here, because

            #       apparently it's not possible for connected subsystems.

            #       Results in "error: Invalid argument"

            block_sysfs_settings = {

                "iostats": "0",

                "rq_affinity": "0",

                "nomerges": "2"

            }



            for disk in self.get_connected_nvme_list():

                sysfs = os.path.join("/sys/block", disk, "queue")

                for k, v in block_sysfs_settings.items():

                    sysfs_opt_path = os.path.join(sysfs, k)

                    try:

                        self.exec_cmd(["sudo", "bash", "-c", "echo %s > %s" % (v, sysfs_opt_path)], stderr_redirect=True)

                    except subprocess.CalledProcessError as e:

                        self.log_print("Warning: command %s failed due to error %s. %s was not set!" % (e.cmd, e.output, v))

                    finally:

                        _ = self.exec_cmd(["sudo", "cat", "%s" % (sysfs_opt_path)])

                        self.log_print("%s=%s" % (sysfs_opt_path, _))



    def kernel_init_disconnect(self):

        for subsystem in self.subsystem_info_list:

            self.exec_cmd(["sudo", self.nvmecli_bin, "disconnect", "-n", subsystem[1]])