Commit 17ef8ec9 authored by Niklas Cassel's avatar Niklas Cassel Committed by Tomasz Zawadzki
Browse files

examples/nvme_fio_plugin: add support for zone append 



Now when we have support for spdk_nvme_zns_zone_append() and
spdk_nvme_zns_zone_appendv(), hook them up in the nvme fio plugin.

Note that fio itself does not have support for zone append,
since unlike SPDK, there is no user facing zone append API in
Linux. Therefore, this new option simply replaces writes with
zone appends in the SPDK fio backend.

This is however still useful for the following reasons:
-Provides a way to test zone append in SPDK.
-By using zone append, we can test with iodepth > 1.

With regular writes, the user can only specify iodepth=1.
This is because for zone namespaces, writes have to target
the write pointer. Having more than one write in flight, per
zone, will lead to I/O errors.

In Linux, it is possible to use fio with iodepth > 1
on zoned namespaces, simply because of the mq-deadline
scheduler, which throttles writes such that there is only
one write in flight, per zone, even if user space has
queued up more.

Since a user might not want to use zone append unconditionally,
even on a namespace that supports it, make this an option
rather than enabling it unconditionally.

Signed-off-by: default avatarNiklas Cassel <niklas.cassel@wdc.com>
Change-Id: I028b79f6445bc63b68c97d1370c6f8139779666d
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/6330


Community-CI: Broadcom CI
Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
parent 014baeb8

examples/nvme/fio_plugin/README.md

+11 −0
Original line number Diff line number Diff line
@@ -139,6 +139,17 @@ then you can reset all zones before fio start running its jobs by using the engi 


    --initial_zone_reset=1



## Zone Append



When running FIO against a Zoned Namespace you need to specify --iodepth=1 to avoid

"Zone Invalid Write: The write to a zone was not at the write pointer." I/O errors.

However, if your controller supports Zone Append, you can use the engine option:



    --zone_append=1



To send zone append commands instead of write commands to the controller.

When using zone append, you will be able to specify a --iodepth greater than 1.



## Shared Memory Increase



If your device has a lot of zones, fio can give you errors such as:

examples/nvme/fio_plugin/fio_plugin.c

+54 −8
Original line number Diff line number Diff line
@@ -93,6 +93,7 @@ struct spdk_fio_options { 
	char	*digest_enable;

	int	enable_vmd;

	int	initial_zone_reset;

	int	zone_append;

};



struct spdk_fio_request {
@@ -130,6 +131,7 @@ struct spdk_fio_qpair { 
	struct spdk_nvme_qpair		*qpair;

	struct spdk_nvme_ns		*ns;

	uint32_t			io_flags;

	bool				zone_append_enabled;

	bool				nvme_pi_enabled;

	/* True for DIF and false for DIX, and this is valid only if nvme_pi_enabled is true. */

	bool				extended_lba;
@@ -408,6 +410,18 @@ attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 
		return;

	}



	if (fio_options->zone_append) {

		if (spdk_nvme_ns_get_csi(ns) == SPDK_NVME_CSI_ZNS &&

		    spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_ZONE_APPEND_SUPPORTED) {

			fprintf(stdout, "Using zone append instead of write\n");

			fio_qpair->zone_append_enabled = true;

		} else {

			SPDK_ERRLOG("zone_append=1 requested, but namespace lacks support\n");

			g_error = true;

			return;

		}

	}



	f->real_file_size = spdk_nvme_ns_get_size(fio_qpair->ns);

	if (f->real_file_size <= 0) {

		g_error = true;
@@ -688,6 +702,12 @@ static void spdk_fio_io_u_free(struct thread_data *td, struct io_u *io_u) 
	}

}



static inline uint64_t

fio_offset_to_zslba(unsigned long long offset, struct spdk_nvme_ns *ns)

{

	return (offset / spdk_nvme_zns_ns_get_zone_size(ns)) * spdk_nvme_zns_ns_get_zone_size_sectors(ns);

}



static int

fio_extended_lba_setup_pi(struct spdk_fio_qpair *fio_qpair, struct io_u *io_u)

{
@@ -982,15 +1002,31 @@ spdk_fio_queue(struct thread_data *td, struct io_u *io_u) 
		break;

	case DDIR_WRITE:

		if (!g_spdk_enable_sgl) {

			if (!fio_qpair->zone_append_enabled) {

				rc = spdk_nvme_ns_cmd_write_with_md(ns, fio_qpair->qpair, io_u->buf, md_buf, lba,

								    lba_count,

								    spdk_fio_completion_cb, fio_req,

								    fio_qpair->io_flags, dif_ctx->apptag_mask, dif_ctx->app_tag);

			} else {

				uint64_t zslba = fio_offset_to_zslba(io_u->offset, fio_qpair->ns);

				rc = spdk_nvme_zns_zone_append_with_md(ns, fio_qpair->qpair, io_u->buf, md_buf, zslba,

								       lba_count,

								       spdk_fio_completion_cb, fio_req,

								       fio_qpair->io_flags, dif_ctx->apptag_mask, dif_ctx->app_tag);

			}

		} else {

			if (!fio_qpair->zone_append_enabled) {

				rc = spdk_nvme_ns_cmd_writev_with_md(ns, fio_qpair->qpair, lba,

								     lba_count, spdk_fio_completion_cb, fio_req, fio_qpair->io_flags,

								     spdk_nvme_io_reset_sgl, spdk_nvme_io_next_sge, md_buf,

								     dif_ctx->apptag_mask, dif_ctx->app_tag);

			} else {

				uint64_t zslba = fio_offset_to_zslba(io_u->offset, fio_qpair->ns);

				rc = spdk_nvme_zns_zone_appendv_with_md(ns, fio_qpair->qpair, zslba,

									lba_count, spdk_fio_completion_cb, fio_req, fio_qpair->io_flags,

									spdk_nvme_io_reset_sgl, spdk_nvme_io_next_sge, md_buf,

									dif_ctx->apptag_mask, dif_ctx->app_tag);

			}

		}

		break;

	default:
@@ -1545,6 +1581,16 @@ static struct fio_option options[] = { 
		.category	= FIO_OPT_C_ENGINE,

		.group		= FIO_OPT_G_INVALID,

	},

	{

		.name		= "zone_append",

		.lname		= "Use zone append instead of write",

		.type		= FIO_OPT_INT,

		.off1		= offsetof(struct spdk_fio_options, zone_append),

		.def		= "0",

		.help		= "Use zone append instead of write (zone_append=1 or zone_append=0)",

		.category	= FIO_OPT_C_ENGINE,

		.group		= FIO_OPT_G_INVALID,

	},

	{

		.name		= NULL,

	},