fio_plugin: add NVMe-oF host support in fio_plugin. (13fc774b) · Commits · Public Repositories / spdk

autotest.sh

+1 −0

Original line number	Diff line number	Diff line
		@@ -166,6 +166,7 @@ if [ $SPDK_TEST_NVMF -eq 1 ]; then
		run_test test/nvmf/host/identify.sh
		run_test test/nvmf/host/perf.sh
		run_test test/nvmf/host/identify_kernel_nvmf.sh
		run_test test/nvmf/host/fio.sh

		timing_exit host

examples/nvme/fio_plugin/README.md

+6 −3

Original line number	Diff line number	Diff line
		@@ -34,10 +34,13 @@ To use the SPDK fio plugin with fio, simply set the following in the fio configu

		To select NVMe devices, you simply pass an identifier as the filename in the format

		domain.bus.slot.func/namespace
		'key=value [key=value] ... ns=value'

		Remember that NVMe namespaces start at 1, not 0! Also, the notation uses '.' throughout,
		not ':'. For example - 0000.04.00.0/1.
		Do not have any ':' in filename, otherwise it will be spilt into several file names. Also the
		NVMe namespaces start at 1, not 0! And it should be put on the end. For example,
		1. For local PCIe NVMe device - 'trtype=PCIe traddr=0000.04.00.0 ns=1'. traddr for local
		NVMe device should use this format: domain.bus.slot.func
		2. For devices exported by NVMe-oF target, 'trtype=RDMA adrfam=IPv4 traddr=192.168.100.8 trsvcid=4420 ns=1'

		Currently the SPDK fio plugin is limited to a single thread, so only one job is supported.
		fio jobs must also specify thread=1 when using the SPDK fio plugin.

examples/nvme/fio_plugin/fio_plugin.c

+134 −76

Original line number	Diff line number	Diff line
		@@ -40,6 +40,7 @@
		#include "spdk/nvme.h"
		#include "spdk/env.h"
		#include "spdk/string.h"
		#include "spdk/log.h"

		#include "config-host.h"
		#include "fio.h"
		@@ -63,6 +64,7 @@ struct spdk_fio_ns {
		};

		struct spdk_fio_ctrlr {
		struct spdk_nvme_transport_id tr_id;
		struct spdk_nvme_ctrlr *ctrlr;
		struct spdk_fio_ctrlr *next;

		@@ -79,6 +81,7 @@ struct spdk_fio_thread {
		struct io_u **iocq; // io completion queue
		unsigned int iocq_count; // number of iocq entries filled by last getevents
		unsigned int iocq_size; // number of iocq entries allocated
		struct fio_file *current_f; // fio_file given by user

		};

		@@ -86,33 +89,8 @@ static bool
		probe_cb(void cb_ctx, const struct spdk_nvme_transport_id trid,
		struct spdk_nvme_ctrlr_opts *opts)
		{
		struct fio_file *f;
		unsigned int i;
		struct thread_data *td = cb_ctx;
		int rc;
		struct spdk_pci_addr pci_addr;

		if (spdk_pci_addr_parse(&pci_addr, trid->traddr)) {
		return false;
		}

		/* Check if we want to claim this device */
		for_each_file(td, f, i) {
		int domain, bus, slot, func, nsid;
		rc = sscanf(f->file_name, "%x.%x.%x.%x/%x", &domain, &bus, &slot, &func, &nsid);
		if (rc != 5) {
		fprintf(stderr, "Invalid filename: %s\n", f->file_name);
		continue;
		}
		if (bus == pci_addr.bus &&
		slot == pci_addr.dev &&
		func == pci_addr.func) {
		return true;
		}
		}

		return false;
		}

		static void
		attach_cb(void cb_ctx, const struct spdk_nvme_transport_id trid,
		@@ -122,44 +100,74 @@ attach_cb(void cb_ctx, const struct spdk_nvme_transport_id trid,
		struct spdk_fio_thread *fio_thread = td->io_ops_data;
		struct spdk_fio_ctrlr *fio_ctrlr;
		struct spdk_fio_ns *fio_ns;
		struct fio_file *f;
		unsigned int i;
		struct spdk_pci_addr pci_addr;
		struct fio_file *f = fio_thread->current_f;
		uint32_t ns_id;
		bool ctrlr_is_added = false;
		char *p;

		p = strstr(f->file_name, "ns=");
		assert(p != NULL);
		ns_id = atoi(p + 3);
		if (!ns_id) {
		SPDK_ERRLOG("namespace id should be >=1, but current value=0\n");
		return;
		}

		/* check whether this trid is already added */
		fio_ctrlr = fio_thread->ctrlr_list;
		while (fio_ctrlr) {
		if (spdk_nvme_transport_id_compare(trid, &fio_ctrlr->tr_id) == 0) {
		ctrlr_is_added = true;
		break;
		}

		spdk_pci_addr_parse(&pci_addr, trid->traddr);
		fio_ctrlr = fio_ctrlr->next;
		}

		/* it is a new ctrlr and needs to be added */
		if (!ctrlr_is_added) {
		/* Create an fio_ctrlr and add it to the list */
		fio_ctrlr = calloc(1, sizeof(*fio_ctrlr));
		if (!fio_ctrlr) {
		SPDK_ERRLOG("Cannot allocate space for fio_ctrlr\n");
		return;
		}
		fio_ctrlr->ctrlr = ctrlr;
		fio_ctrlr->tr_id = *trid;
		fio_ctrlr->qpair = spdk_nvme_ctrlr_alloc_io_qpair(ctrlr, 0);
		fio_ctrlr->ns_list = NULL;
		fio_ctrlr->next = fio_thread->ctrlr_list;
		fio_thread->ctrlr_list = fio_ctrlr;
		}

		/* check whether this name space is existing or not */
		fio_ns = fio_ctrlr->ns_list;
		while (fio_ns) {
		if (spdk_nvme_ns_get_id(fio_ns->ns) == ns_id) {
		return;
		}
		fio_ns = fio_ns->next;
		}

		/* Loop through all of the file names provided and grab the matching namespaces */
		for_each_file(fio_thread->td, f, i) {
		int domain, bus, slot, func, nsid, rc;
		rc = sscanf(f->file_name, "%x.%x.%x.%x/%x", &domain, &bus, &slot, &func, &nsid);
		if (rc == 5 &&
		bus == pci_addr.bus &&
		slot == pci_addr.dev &&
		func == pci_addr.func) {
		/* create a new namespace */
		fio_ns = calloc(1, sizeof(*fio_ns));
		if (fio_ns == NULL) {
		continue;
		SPDK_ERRLOG("Cannot allocate space for fio_ns\n");
		return;
		}
		fio_ns->f = f;
		fio_ns->ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid);
		fio_ns->ns = spdk_nvme_ctrlr_get_ns(ctrlr, ns_id);
		if (fio_ns->ns == NULL) {
		SPDK_ERRLOG("Cannot get namespace by ns_id=%d\n", ns_id);
		free(fio_ns);
		continue;
		return;
		}


		f->real_file_size = spdk_nvme_ns_get_size(fio_ns->ns);
		if (f->real_file_size <= 0) {
		SPDK_ERRLOG("Cannot get namespace size by ns=%p\n", fio_ns->ns);
		free(fio_ns);
		continue;
		return;
		}

		f->filetype = FIO_TYPE_BLOCK;
		@@ -168,16 +176,22 @@ attach_cb(void cb_ctx, const struct spdk_nvme_transport_id trid,
		fio_ns->next = fio_ctrlr->ns_list;
		fio_ctrlr->ns_list = fio_ns;
		}
		}
		}

		/* Called once at initialization. This is responsible for gathering the size of
		* each "file", which in our case are in the form
		* "05:00.0/0" (PCI bus:device.function/NVMe NSID) */
		* 'key=value [key=value] ... ns=value'
		* For example, For local PCIe NVMe device - 'trtype=PCIe traddr=0000.04.00.0 ns=1'
		* For remote exported by NVMe-oF target, 'trtype=RDMA adrfam=IPv4 traddr=192.168.100.8 trsvcid=4420 ns=1' */
		static int spdk_fio_setup(struct thread_data *td)
		{
		struct spdk_fio_thread *fio_thread;
		struct spdk_env_opts opts;
		struct fio_file *f;
		char *p;
		int rc;
		struct spdk_nvme_transport_id trid;
		char *trid_info;
		unsigned int i;

		if (!td->o.use_thread) {
		log_err("spdk: must set thread=1 when using spdk plugin\n");
		@@ -196,12 +210,56 @@ static int spdk_fio_setup(struct thread_data *td)

		spdk_env_opts_init(&opts);
		opts.name = "fio";
		opts.dpdk_mem_size = 512;
		spdk_env_init(&opts);

		for_each_file(td, f, i) {
		memset(&trid, 0, sizeof(trid));
		trid_info = NULL;

		trid.trtype = SPDK_NVME_TRANSPORT_PCIE;

		p = strstr(f->file_name, " ns=");
		if (p == NULL) {
		SPDK_ERRLOG("Failed to find namespace 'ns=X'\n");
		continue;
		}

		trid_info = strndup(f->file_name, p - f->file_name);
		if (!trid_info) {
		SPDK_ERRLOG("Failed to allocate space for trid_info\n");
		continue;
		}

		rc = spdk_nvme_transport_id_parse(&trid, trid_info);
		if (rc < 0) {
		SPDK_ERRLOG("Failed to parse given str: %s\n", trid_info);
		free(trid_info);
		continue;
		}
		free(trid_info);

		if (trid.trtype == SPDK_NVME_TRANSPORT_PCIE) {
		struct spdk_pci_addr pci_addr;
		if (spdk_pci_addr_parse(&pci_addr, trid.traddr) < 0) {
		SPDK_ERRLOG("Invaild traddr=%s\n", trid.traddr);
		continue;
		}
		spdk_pci_addr_fmt(trid.traddr, sizeof(trid.traddr), &pci_addr);
		} else if (trid.trtype == SPDK_NVME_TRANSPORT_RDMA) {
		if (trid.subnqn[0] == '\0') {
		snprintf(trid.subnqn, sizeof(trid.subnqn), "%s",
		SPDK_NVMF_DISCOVERY_NQN);
		}
		}

		fio_thread->current_f = f;

		/* Enumerate all of the controllers */
		if (spdk_nvme_probe(NULL, td, probe_cb, attach_cb, NULL) != 0) {
		fprintf(stderr, "spdk_nvme_probe() failed\n");
		return 1;
		if (spdk_nvme_probe(&trid, td, probe_cb, attach_cb, NULL) != 0) {
		SPDK_ERRLOG("spdk_nvme_probe() failed\n");
		continue;
		}
		}

		return 0;

test/lib/nvme/nvme.sh

+1 −1

Original line number	Diff line number	Diff line
		@@ -48,7 +48,7 @@ timing_exit overhead
		if [ -d /usr/src/fio ]; then
		timing_enter fio_plugin
		for bdf in $(linux_iter_pci 0108); do
		/usr/src/fio/fio $rootdir/examples/nvme/fio_plugin/example_config.fio --filename=${bdf//:/.}/1
		/usr/src/fio/fio $rootdir/examples/nvme/fio_plugin/example_config.fio --filename="trtype=PCIe traddr=${bdf//:/.} ns=1"
		break
		done

test/nvmf/host/fio.sh

0 → 100755

+39 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/env bash

		testdir=$(readlink -f $(dirname $0))
		rootdir=$(readlink -f $testdir/../../..)
		source $rootdir/scripts/autotest_common.sh
		source $rootdir/test/nvmf/common.sh

		rpc_py="python $rootdir/scripts/rpc.py"

		set -e

		if ! rdma_nic_available; then
		echo "no NIC for nvmf test"
		exit 0
		fi

		timing_enter fio

		# Start up the NVMf target in another process
		$rootdir/app/nvmf_tgt/nvmf_tgt -c $testdir/../nvmf.conf -m 0x2 -p 1 -s 512 &
		nvmfpid=$!

		trap "killprocess $nvmfpid; exit 1" SIGINT SIGTERM EXIT

		waitforlisten $nvmfpid ${RPC_PORT}

		$rpc_py construct_nvmf_subsystem Direct nqn.2016-06.io.spdk:cnode1 'transport:RDMA traddr:192.168.100.8 trsvcid:4420' '' -p "*"

		if [ -d /usr/src/fio ]; then
		/usr/src/fio/fio $rootdir/examples/nvme/fio_plugin/example_config.fio --filename="trtype=RDMA adrfam=IPv4 traddr=192.168.100.8 trsvcid=4420 ns=1"
		fi
		sync

		$rpc_py delete_nvmf_subsystem nqn.2016-06.io.spdk:cnode1

		trap - SIGINT SIGTERM EXIT

		killprocess $nvmfpid
		timing_exit fio