Commit 1c7b4b7b authored by Jaroslaw Chachulski's avatar Jaroslaw Chachulski Committed by Konrad Sztyber
Browse files

scripts/nvmf_perf: add roce_enable option to configuration file 



Parse the config file for "irdma_roce_enable" flag to load irdma
driver with ROCE network protocol enabled on SPDK Target and
SPDK Initiator machines.

Signed-off-by: default avatarJaroslaw Chachulski <jaroslawx.chachulski@intel.com>
Change-Id: If81c0ec494c13a561c6c780105ee4caba053001a
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15343


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarPawel Piatek <pawelx.piatek@intel.com>
Reviewed-by: default avatarKonrad Sztyber <konrad.sztyber@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
parent 9579995c

scripts/perf/nvmf/README.md

+5 −1
Original line number Diff line number Diff line
@@ -59,7 +59,8 @@ The following sub-chapters describe each configuration section in more detail. 
    "username": "user",

    "password": "password",

    "transport": "transport_type",

    "skip_spdk_install": bool

    "skip_spdk_install": bool,

    "irdma_roce_enable": bool

}

```
@@ -76,6 +77,9 @@ Optional: 
  is already in place on Initiator systems and there's no need to re-build it,

  then set this option to true.

  Default: false.

- irdma_roce_enable - loads irdma driver with RoCEv2 network protocol enabled on Target and

  Initiator machines. This option applies only to system with Intel E810 NICs.

  Default: false



### Target System Configuration

scripts/perf/nvmf/run_nvmf.py

+33 −2
Original line number Diff line number Diff line
@@ -37,6 +37,7 @@ class Server: 
        self.transport = general_config["transport"].lower()

        self.nic_ips = server_config["nic_ips"]

        self.mode = server_config["mode"]

        self.irdma_roce_enable = False



        self.log = logging.getLogger(self.name)
@@ -63,6 +64,8 @@ class Server: 
            self.irq_settings.update(server_config["irq_settings"])

        if "tuned_profile" in server_config:

            self.tuned_profile = server_config["tuned_profile"]

        if "irdma_roce_enable" in general_config:

            self.irdma_roce_enable = general_config["irdma_roce_enable"]



        if not re.match(r'^[A-Za-z0-9\-]+$', name):

            self.log.info("Please use a name which contains only letters, numbers or dashes")
@@ -126,11 +129,38 @@ class Server: 
        self.configure_cpu_governor()

        self.configure_irq_affinity(**self.irq_settings)



    RDMA_PROTOCOL_IWARP = 0

    RDMA_PROTOCOL_ROCE = 1

    RDMA_PROTOCOL_UNKNOWN = -1



    def check_rdma_protocol(self):

        try:

            roce_ena = self.exec_cmd(["cat", "/sys/module/irdma/parameters/roce_ena"])

            roce_ena = roce_ena.decode().strip()

            if roce_ena == "0":

                return self.RDMA_PROTOCOL_IWARP

            else:

                return self.RDMA_PROTOCOL_ROCE

        except CalledProcessError as e:

            self.log.error("ERROR: failed to check RDMA protocol!")

            self.log.error("%s resulted in error: %s" % (e.cmd, e.output))

            return self.RDMA_PROTOCOL_UNKNOWN



    def load_drivers(self):

        self.log.info("Loading drivers")

        self.exec_cmd(["sudo", "modprobe", "-a",

                       "nvme-%s" % self.transport,

                       "nvmet-%s" % self.transport])

        current_mode = self.check_rdma_protocol()

        if current_mode == self.RDMA_PROTOCOL_UNKNOWN:

            self.log.error("ERROR: failed to check RDMA protocol mode")

            return

        if self.irdma_roce_enable and current_mode == self.RDMA_PROTOCOL_IWARP:

            self.reload_driver("irdma", "roce_ena=1")

        elif self.irdma_roce_enable and current_mode == self.RDMA_PROTOCOL_ROCE:

            self.log.info("Leaving irdma driver with RoCE enabled")

        else:

            self.reload_driver("irdma", "roce_ena=0")



    def configure_adq(self):

        self.adq_load_modules()
@@ -200,10 +230,11 @@ class Server: 
            self.log.info(xps_cmd)

            self.exec_cmd(xps_cmd)



    def reload_driver(self, driver):

    def reload_driver(self, driver, *modprobe_args):



        try:

            self.exec_cmd(["sudo", "rmmod", driver])

            self.exec_cmd(["sudo", "modprobe", driver])

            self.exec_cmd(["sudo", "modprobe", driver, *modprobe_args])

        except CalledProcessError as e:

            self.log.error("ERROR: failed to reload %s module!" % driver)

            self.log.error("%s resulted in error: %s" % (e.cmd, e.output))