Commit 07fa7327 authored by Shuhei Matsumoto's avatar Shuhei Matsumoto Committed by Jim Harris
Browse files

nvme_perf: Introduce function pointer table for IO type dependent operations 



Introduce a function pointer table to ns_entry to remove if-else
sequence in every operation depending on the type, AIO or NVMe.

This will simplify upcoming DIF and DIX support in the Perf tool.

Change-Id: Ibbd9a9ac3a0b5df529d5b60706ce750a746114c3
Signed-off-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-on: https://review.gerrithub.io/c/439630


Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarChangpeng Liu <changpeng.liu@intel.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Chandler-Test-Pool: SPDK Automated Test System <sys_sgsw@intel.com>
parent f75128a9

examples/nvme/perf/perf.c

+189 −105
Original line number Diff line number Diff line
@@ -59,8 +59,11 @@ enum entry_type { 
	ENTRY_TYPE_AIO_FILE,

};



struct ns_fn_table;



struct ns_entry {

	enum entry_type		type;

	const struct ns_fn_table	*fn_table;



	union {

		struct {
@@ -151,6 +154,21 @@ struct worker_thread { 
	unsigned		lcore;

};



struct ns_fn_table {

	void	(*setup_payload)(struct perf_task *task, uint8_t pattern);



	int	(*submit_io)(struct perf_task *task, struct ns_worker_ctx *ns_ctx,

			     struct ns_entry *entry, uint64_t offset_in_ios);



	void	(*check_io)(struct ns_worker_ctx *ns_ctx);



	void	(*verify_io)(struct perf_task *task, struct ns_entry *entry);



	int	(*init_ns_worker_ctx)(struct ns_worker_ctx *ns_ctx);



	void	(*cleanup_ns_worker_ctx)(struct ns_worker_ctx *ns_ctx);

};



static int g_outstanding_commands;



static bool g_latency_ssd_tracking_enable = false;
@@ -200,6 +218,17 @@ static void 
task_complete(struct perf_task *task);



#if HAVE_LIBAIO

static void

aio_setup_payload(struct perf_task *task, uint8_t pattern)

{

	task->buf = spdk_dma_zmalloc(g_io_size_bytes, g_io_align, NULL);

	if (task->buf == NULL) {

		fprintf(stderr, "spdk_dma_zmalloc() for task->buf failed\n");

		exit(1);

	}

	memset(task->buf, pattern, g_io_size_bytes);

}



static int

aio_submit(io_context_t aio_ctx, struct iocb *iocb, int fd, enum io_iocb_cmd cmd, void *buf,

	   unsigned long nbytes, uint64_t offset, void *cb_ctx)
@@ -220,6 +249,19 @@ aio_submit(io_context_t aio_ctx, struct iocb *iocb, int fd, enum io_iocb_cmd cmd 
	return 0;

}



static int

aio_submit_io(struct perf_task *task, struct ns_worker_ctx *ns_ctx,

	      struct ns_entry *entry, uint64_t offset_in_ios)

{

	if (task->is_read) {

		return aio_submit(ns_ctx->u.aio.ctx, &task->iocb, entry->u.aio.fd, IO_CMD_PREAD, task->buf,

				  g_io_size_bytes, offset_in_ios * g_io_size_bytes, task);

	} else {

		return aio_submit(ns_ctx->u.aio.ctx, &task->iocb, entry->u.aio.fd, IO_CMD_PWRITE, task->buf,

				  g_io_size_bytes, offset_in_ios * g_io_size_bytes, task);

	}

}



static void

aio_check_io(struct ns_worker_ctx *ns_ctx)

{
@@ -240,6 +282,43 @@ aio_check_io(struct ns_worker_ctx *ns_ctx) 
	}

}



static void

aio_verify_io(struct perf_task *task, struct ns_entry *entry)

{

}



static int

aio_init_ns_worker_ctx(struct ns_worker_ctx *ns_ctx)

{

	ns_ctx->u.aio.events = calloc(g_queue_depth, sizeof(struct io_event));

	if (!ns_ctx->u.aio.events) {

		return -1;

	}

	ns_ctx->u.aio.ctx = 0;

	if (io_setup(g_queue_depth, &ns_ctx->u.aio.ctx) < 0) {

		free(ns_ctx->u.aio.events);

		perror("io_setup");

		return -1;

	}

	return 0;

}



static void

aio_cleanup_ns_worker_ctx(struct ns_worker_ctx *ns_ctx)

{

	io_destroy(ns_ctx->u.aio.ctx);

	free(ns_ctx->u.aio.events);

}



static const struct ns_fn_table aio_fn_table = {

	.setup_payload		= aio_setup_payload,

	.submit_io		= aio_submit_io,

	.check_io		= aio_check_io,

	.verify_io		= aio_verify_io,

	.init_ns_worker_ctx	= aio_init_ns_worker_ctx,

	.cleanup_ns_worker_ctx	= aio_cleanup_ns_worker_ctx,

};



static int

register_aio_file(const char *path)

{
@@ -295,6 +374,7 @@ register_aio_file(const char *path) 
	}



	entry->type = ENTRY_TYPE_AIO_FILE;

	entry->fn_table = &aio_fn_table;

	entry->u.aio.fd = fd;

	entry->size_in_ios = size / g_io_size_bytes;

	entry->io_size_blocks = g_io_size_bytes / blklen;
@@ -459,6 +539,105 @@ task_extended_lba_pi_verify(struct ns_entry *entry, struct perf_task *task, 


static void io_complete(void *ctx, const struct spdk_nvme_cpl *cpl);



static void

nvme_setup_payload(struct perf_task *task, uint8_t pattern)

{

	uint32_t max_io_size_bytes;



	/* maximum extended lba format size from all active namespace,

	 * it's same with g_io_size_bytes for namespace without metadata.

	 */

	max_io_size_bytes = g_io_size_bytes + g_max_io_md_size * g_max_io_size_blocks;

	task->buf = spdk_dma_zmalloc(max_io_size_bytes, g_io_align, NULL);

	if (task->buf == NULL) {

		fprintf(stderr, "task->buf spdk_dma_zmalloc failed\n");

		exit(1);

	}

	memset(task->buf, pattern, max_io_size_bytes);

}



static int

nvme_submit_io(struct perf_task *task, struct ns_worker_ctx *ns_ctx,

	       struct ns_entry *entry, uint64_t offset_in_ios)

{

	task->lba = offset_in_ios * entry->io_size_blocks;



	task_extended_lba_setup_pi(entry, task, task->lba,

				   entry->io_size_blocks, !task->is_read);



	if (task->is_read) {

		return spdk_nvme_ns_cmd_read_with_md(entry->u.nvme.ns, ns_ctx->u.nvme.qpair,

						     task->buf, NULL,

						     task->lba,

						     entry->io_size_blocks, io_complete,

						     task, entry->io_flags,

						     task->appmask, task->apptag);

	} else {

		return spdk_nvme_ns_cmd_write_with_md(entry->u.nvme.ns, ns_ctx->u.nvme.qpair,

						      task->buf, NULL,

						      task->lba,

						      entry->io_size_blocks, io_complete,

						      task, entry->io_flags,

						      task->appmask, task->apptag);

	}

}



static void

nvme_check_io(struct ns_worker_ctx *ns_ctx)

{

	spdk_nvme_qpair_process_completions(ns_ctx->u.nvme.qpair, g_max_completions);

}



static void

nvme_verify_io(struct perf_task *task, struct ns_entry *entry)

{

	if (spdk_nvme_ns_supports_extended_lba(entry->u.nvme.ns) &&

	    task->is_read && !g_metacfg_pract_flag) {

		task_extended_lba_pi_verify(entry, task, task->lba,

					    entry->io_size_blocks);

	}

}



/*

 * TODO: If a controller has multiple namespaces, they could all use the same queue.

 *  For now, give each namespace/thread combination its own queue.

 */

static int

nvme_init_ns_worker_ctx(struct ns_worker_ctx *ns_ctx)

{

	struct spdk_nvme_io_qpair_opts opts;

	struct ns_entry *entry = ns_ctx->entry;



	spdk_nvme_ctrlr_get_default_io_qpair_opts(entry->u.nvme.ctrlr, &opts, sizeof(opts));

	if (opts.io_queue_requests < entry->num_io_requests) {

		opts.io_queue_requests = entry->num_io_requests;

	}



	ns_ctx->u.nvme.qpair = spdk_nvme_ctrlr_alloc_io_qpair(entry->u.nvme.ctrlr, &opts,

			       sizeof(opts));

	if (!ns_ctx->u.nvme.qpair) {

		printf("ERROR: spdk_nvme_ctrlr_alloc_io_qpair failed\n");

		return -1;

	}



	return 0;

}



static void

nvme_cleanup_ns_worker_ctx(struct ns_worker_ctx *ns_ctx)

{

	spdk_nvme_ctrlr_free_io_qpair(ns_ctx->u.nvme.qpair);

}



static const struct ns_fn_table nvme_fn_table = {

	.setup_payload		= nvme_setup_payload,

	.submit_io		= nvme_submit_io,

	.check_io		= nvme_check_io,

	.verify_io		= nvme_verify_io,

	.init_ns_worker_ctx	= nvme_init_ns_worker_ctx,

	.cleanup_ns_worker_ctx	= nvme_cleanup_ns_worker_ctx,

};



static void

register_ns(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns *ns)

{
@@ -509,6 +688,7 @@ register_ns(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns *ns) 
	}



	entry->type = ENTRY_TYPE_NVME_NS;

	entry->fn_table = &nvme_fn_table;

	entry->u.nvme.ctrlr = ctrlr;

	entry->u.nvme.ns = ns;

	entry->num_io_requests = entries;
@@ -655,50 +835,17 @@ submit_single_io(struct perf_task *task) 
		}

	}



	task->is_read = false;

	task->submit_tsc = spdk_get_ticks();

	task->lba = offset_in_ios * entry->io_size_blocks;



	if ((g_rw_percentage == 100) ||

	    (g_rw_percentage != 0 && ((rand_r(&seed) % 100) < g_rw_percentage))) {

#if HAVE_LIBAIO

		if (entry->type == ENTRY_TYPE_AIO_FILE) {

			rc = aio_submit(ns_ctx->u.aio.ctx, &task->iocb, entry->u.aio.fd, IO_CMD_PREAD, task->buf,

					g_io_size_bytes, offset_in_ios * g_io_size_bytes, task);

		} else

#endif

		{

			task_extended_lba_setup_pi(entry, task, task->lba,

						   entry->io_size_blocks, false);

		task->is_read = true;



			rc = spdk_nvme_ns_cmd_read_with_md(entry->u.nvme.ns, ns_ctx->u.nvme.qpair,

							   task->buf, NULL,

							   task->lba,

							   entry->io_size_blocks, io_complete,

							   task, entry->io_flags,

							   task->appmask, task->apptag);

		}

	} else {

#if HAVE_LIBAIO

		if (entry->type == ENTRY_TYPE_AIO_FILE) {

			rc = aio_submit(ns_ctx->u.aio.ctx, &task->iocb, entry->u.aio.fd, IO_CMD_PWRITE, task->buf,

					g_io_size_bytes, offset_in_ios * g_io_size_bytes, task);

		} else

#endif

		{

			task_extended_lba_setup_pi(entry, task, task->lba,

						   entry->io_size_blocks, true);



			rc = spdk_nvme_ns_cmd_write_with_md(entry->u.nvme.ns, ns_ctx->u.nvme.qpair,

							    task->buf, NULL,

							    task->lba,

							    entry->io_size_blocks, io_complete,

							    task, entry->io_flags,

							    task->appmask, task->apptag);

		}

		task->is_read = false;

	}



	rc = entry->fn_table->submit_io(task, ns_ctx, entry, offset_in_ios);



	if (rc != 0) {

		fprintf(stderr, "starting I/O failed\n");

	} else {
@@ -730,13 +877,7 @@ task_complete(struct perf_task *task) 
	}



	/* add application level verification for end-to-end data protection */

	if (entry->type == ENTRY_TYPE_NVME_NS) {

		if (spdk_nvme_ns_supports_extended_lba(entry->u.nvme.ns) &&

		    task->is_read && !g_metacfg_pract_flag) {

			task_extended_lba_pi_verify(entry, task, task->lba,

						    entry->io_size_blocks);

		}

	}

	entry->fn_table->verify_io(task, entry);



	/*

	 * is_draining indicates when time has expired for the test run
@@ -769,21 +910,13 @@ io_complete(void *ctx, const struct spdk_nvme_cpl *cpl) 
static void

check_io(struct ns_worker_ctx *ns_ctx)

{

#if HAVE_LIBAIO

	if (ns_ctx->entry->type == ENTRY_TYPE_AIO_FILE) {

		aio_check_io(ns_ctx);

	} else

#endif

	{

		spdk_nvme_qpair_process_completions(ns_ctx->u.nvme.qpair, g_max_completions);

	}

	ns_ctx->entry->fn_table->check_io(ns_ctx);

}



static struct perf_task *

allocate_task(struct ns_worker_ctx *ns_ctx, int queue_depth)

{

	struct perf_task *task;

	uint32_t max_io_size_bytes;



	task = calloc(1, sizeof(*task));

	if (task == NULL) {
@@ -791,16 +924,7 @@ allocate_task(struct ns_worker_ctx *ns_ctx, int queue_depth) 
		exit(1);

	}



	/* maximum extended lba format size from all active namespace,

	 * it's same with g_io_size_bytes for namespace without metadata.

	 */

	max_io_size_bytes = g_io_size_bytes + g_max_io_md_size * g_max_io_size_blocks;

	task->buf = spdk_dma_zmalloc(max_io_size_bytes, g_io_align, NULL);

	if (task->buf == NULL) {

		fprintf(stderr, "task->buf spdk_dma_zmalloc failed\n");

		exit(1);

	}

	memset(task->buf, queue_depth % 8 + 1, max_io_size_bytes);

	ns_ctx->entry->fn_table->setup_payload(task, queue_depth % 8 + 1);



	task->ns_ctx = ns_ctx;
@@ -821,53 +945,13 @@ submit_io(struct ns_worker_ctx *ns_ctx, int queue_depth) 
static int

init_ns_worker_ctx(struct ns_worker_ctx *ns_ctx)

{

	if (ns_ctx->entry->type == ENTRY_TYPE_AIO_FILE) {

#ifdef HAVE_LIBAIO

		ns_ctx->u.aio.events = calloc(g_queue_depth, sizeof(struct io_event));

		if (!ns_ctx->u.aio.events) {

			return -1;

		}

		ns_ctx->u.aio.ctx = 0;

		if (io_setup(g_queue_depth, &ns_ctx->u.aio.ctx) < 0) {

			free(ns_ctx->u.aio.events);

			perror("io_setup");

			return -1;

		}

#endif

	} else {

		/*

		 * TODO: If a controller has multiple namespaces, they could all use the same queue.

		 *  For now, give each namespace/thread combination its own queue.

		 */

		struct spdk_nvme_io_qpair_opts opts;



		spdk_nvme_ctrlr_get_default_io_qpair_opts(ns_ctx->entry->u.nvme.ctrlr, &opts, sizeof(opts));

		if (opts.io_queue_requests < ns_ctx->entry->num_io_requests) {

			opts.io_queue_requests = ns_ctx->entry->num_io_requests;

		}



		ns_ctx->u.nvme.qpair = spdk_nvme_ctrlr_alloc_io_qpair(ns_ctx->entry->u.nvme.ctrlr, &opts,

				       sizeof(opts));

		if (!ns_ctx->u.nvme.qpair) {

			printf("ERROR: spdk_nvme_ctrlr_alloc_io_qpair failed\n");

			return -1;

		}

	}



	return 0;

	return ns_ctx->entry->fn_table->init_ns_worker_ctx(ns_ctx);

}



static void

cleanup_ns_worker_ctx(struct ns_worker_ctx *ns_ctx)

{

	if (ns_ctx->entry->type == ENTRY_TYPE_AIO_FILE) {

#ifdef HAVE_LIBAIO

		io_destroy(ns_ctx->u.aio.ctx);

		free(ns_ctx->u.aio.events);

#endif

	} else {

		spdk_nvme_ctrlr_free_io_qpair(ns_ctx->u.nvme.qpair);

	}

	ns_ctx->entry->fn_table->cleanup_ns_worker_ctx(ns_ctx);

}



static int