Commit 3272320c authored by Daniel Verkamp's avatar Daniel Verkamp
Browse files

nvme: make I/O queue allocation explicit 



The previous method for registering I/O queues did not allow the user
to specify queue priority for weighted round robin arbitration, and it
limited the application to one queue per controller per thread.

Change the API to require explicit allocation of each queue for each
controller using the new function spdk_nvme_ctrlr_alloc_io_qpair().

Each function that submits a command on an I/O queue now takes an
explicit qpair parameter rather than implicitly using the thread-local
queue.

This also allows the application to allocate different numbers of
threads per controller; previously, the number of queues was capped at
the smallest value supported by any attached controller.

Weighted round robin arbitration is not supported yet; additional
changes to the controller startup process are required to enable
alternate arbitration methods.

Change-Id: Ia33be1050a6953bc5a3cca9284aefcd95b01116e
Signed-off-by: default avatarDaniel Verkamp <daniel.verkamp@intel.com>
parent 9f67a07f

CHANGELOG.md

+7 −0
Original line number Diff line number Diff line
@@ -13,6 +13,13 @@ user code. 
  moves device detection into the NVMe library.  The new API also allows

  parallel initialization of NVMe controllers, providing a major reduction in

  startup time when using multiple controllers.

  - I/O queue allocation was changed to be explicit in the API.  Each function

  that generates I/O requests now takes a queue pair (`spdk_nvme_qpair *`)

  argument, and I/O queues may be allocated using

  `spdk_nvme_ctrlr_alloc_io_qpair()`.  This allows more flexible assignment of

  queue pairs than the previous model, which only allowed a single queue

  per thread and limited the total number of I/O queues to the lowest number

  supported on any attached controller.

  - Added support for the Write Zeroes command.

  - `examples/nvme/perf` can now report I/O command latency from the

   the controller's viewpoint using the Intel vendor-specific read/write latency

doc/nvme.index.txt

+1 −1
Original line number Diff line number Diff line
@@ -46,7 +46,7 @@ 
- spdk_nvme_ns_cmd_write() \copybrief spdk_nvme_ns_cmd_write()

- spdk_nvme_ns_cmd_deallocate() \copybrief spdk_nvme_ns_cmd_deallocate()

- spdk_nvme_ns_cmd_flush() \copybrief spdk_nvme_ns_cmd_flush()

- spdk_nvme_ctrlr_process_io_completions() \copybrief spdk_nvme_ctrlr_process_io_completions()

- spdk_nvme_qpair_process_completions() \copybrief spdk_nvme_qpair_process_completions()



\section key_concepts Key Concepts

examples/nvme/nvme_manage/nvme_manage.c

+0 −9
Original line number Diff line number Diff line
@@ -462,12 +462,6 @@ int main(int argc, char **argv) 


	qsort(devs, num_devs, sizeof(devs[0]), cmp_devs);



	if (num_devs) {

		rc = spdk_nvme_register_io_thread();

		if (rc != 0)

			return rc;

	}



	usage();



	while (1) {
@@ -519,8 +513,5 @@ int main(int argc, char **argv) 
		spdk_nvme_detach(dev->ctrlr);

	}



	if (num_devs)

		spdk_nvme_unregister_io_thread();



	return rc;

}

examples/nvme/perf/perf.c

+72 −29
Original line number Diff line number Diff line
@@ -94,10 +94,19 @@ struct ns_worker_ctx { 
	uint64_t		offset_in_ios;

	bool			is_draining;



	union {

		struct {

			struct spdk_nvme_qpair	*qpair;

		} nvme;



#if HAVE_LIBAIO

		struct {

			struct io_event		*events;

			io_context_t		ctx;

		} aio;

#endif

	} u;



	struct ns_worker_ctx	*next;

};
@@ -177,6 +186,7 @@ register_ns(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns *ns) 
	entry->type = ENTRY_TYPE_NVME_NS;

	entry->u.nvme.ctrlr = ctrlr;

	entry->u.nvme.ns = ns;



	entry->size_in_ios = spdk_nvme_ns_get_size(ns) /

			     g_io_size_bytes;

	entry->io_size_blocks = g_io_size_bytes / spdk_nvme_ns_get_sector_size(ns);
@@ -346,14 +356,14 @@ aio_check_io(struct ns_worker_ctx *ns_ctx) 
	timeout.tv_sec = 0;

	timeout.tv_nsec = 0;



	count = io_getevents(ns_ctx->ctx, 1, g_queue_depth, ns_ctx->events, &timeout);

	count = io_getevents(ns_ctx->u.aio.ctx, 1, g_queue_depth, ns_ctx->u.aio.events, &timeout);

	if (count < 0) {

		fprintf(stderr, "io_getevents error\n");

		exit(1);

	}



	for (i = 0; i < count; i++) {

		task_complete(ns_ctx->events[i].data);

		task_complete(ns_ctx->u.aio.events[i].data);

	}

}

#endif /* HAVE_LIBAIO */
@@ -400,23 +410,25 @@ submit_single_io(struct ns_worker_ctx *ns_ctx) 
	    (g_rw_percentage != 0 && ((rand_r(&seed) % 100) < g_rw_percentage))) {

#if HAVE_LIBAIO

		if (entry->type == ENTRY_TYPE_AIO_FILE) {

			rc = aio_submit(ns_ctx->ctx, &task->iocb, entry->u.aio.fd, IO_CMD_PREAD, task->buf,

			rc = aio_submit(ns_ctx->u.aio.ctx, &task->iocb, entry->u.aio.fd, IO_CMD_PREAD, task->buf,

					g_io_size_bytes, offset_in_ios * g_io_size_bytes, task);

		} else

#endif

		{

			rc = spdk_nvme_ns_cmd_read(entry->u.nvme.ns, task->buf, offset_in_ios * entry->io_size_blocks,

			rc = spdk_nvme_ns_cmd_read(entry->u.nvme.ns, ns_ctx->u.nvme.qpair, task->buf,

						   offset_in_ios * entry->io_size_blocks,

						   entry->io_size_blocks, io_complete, task, 0);

		}

	} else {

#if HAVE_LIBAIO

		if (entry->type == ENTRY_TYPE_AIO_FILE) {

			rc = aio_submit(ns_ctx->ctx, &task->iocb, entry->u.aio.fd, IO_CMD_PWRITE, task->buf,

			rc = aio_submit(ns_ctx->u.aio.ctx, &task->iocb, entry->u.aio.fd, IO_CMD_PWRITE, task->buf,

					g_io_size_bytes, offset_in_ios * g_io_size_bytes, task);

		} else

#endif

		{

			rc = spdk_nvme_ns_cmd_write(entry->u.nvme.ns, task->buf, offset_in_ios * entry->io_size_blocks,

			rc = spdk_nvme_ns_cmd_write(entry->u.nvme.ns, ns_ctx->u.nvme.qpair, task->buf,

						    offset_in_ios * entry->io_size_blocks,

						    entry->io_size_blocks, io_complete, task, 0);

		}

	}
@@ -465,7 +477,7 @@ check_io(struct ns_worker_ctx *ns_ctx) 
	} else

#endif

	{

		spdk_nvme_ctrlr_process_io_completions(ns_ctx->entry->u.nvme.ctrlr, g_max_completions);

		spdk_nvme_qpair_process_completions(ns_ctx->u.nvme.qpair, g_max_completions);

	}

}
@@ -486,20 +498,66 @@ drain_io(struct ns_worker_ctx *ns_ctx) 
	}

}



static int

init_ns_worker_ctx(struct ns_worker_ctx *ns_ctx)

{

	if (ns_ctx->entry->type == ENTRY_TYPE_AIO_FILE) {

#ifdef HAVE_LIBAIO

		ns_ctx->u.aio.events = calloc(g_queue_depth, sizeof(struct io_event));

		if (!ns_ctx->u.aio.events) {

			return -1;

		}

		ns_ctx->u.aio.ctx = 0;

		if (io_setup(g_queue_depth, &ns_ctx->u.aio.ctx) < 0) {

			free(ns_ctx->u.aio.events);

			perror("io_setup");

			return -1;

		}

#endif

	} else {

		/*

		 * TODO: If a controller has multiple namespaces, they could all use the same queue.

		 *  For now, give each namespace/thread combination its own queue.

		 */

		ns_ctx->u.nvme.qpair = spdk_nvme_ctrlr_alloc_io_qpair(ns_ctx->entry->u.nvme.ctrlr, 0);

		if (!ns_ctx->u.nvme.qpair) {

			printf("ERROR: spdk_nvme_ctrlr_alloc_io_qpair failed\n");

			return -1;

		}

	}



	return 0;

}



static void

cleanup_ns_worker_ctx(struct ns_worker_ctx *ns_ctx)

{

	if (ns_ctx->entry->type == ENTRY_TYPE_NVME_NS) {

		spdk_nvme_ctrlr_free_io_qpair(ns_ctx->u.nvme.qpair);

	}

}



static int

work_fn(void *arg)

{

	uint64_t tsc_end = rte_get_timer_cycles() + g_time_in_sec * g_tsc_rate;

	uint64_t tsc_end;

	struct worker_thread *worker = (struct worker_thread *)arg;

	struct ns_worker_ctx *ns_ctx = NULL;



	printf("Starting thread on core %u\n", worker->lcore);



	if (spdk_nvme_register_io_thread() != 0) {

		fprintf(stderr, "spdk_nvme_register_io_thread() failed on core %u\n", worker->lcore);

		return -1;

	/* Allocate a queue pair for each namespace. */

	ns_ctx = worker->ns_ctx;

	while (ns_ctx != NULL) {

		if (init_ns_worker_ctx(ns_ctx) != 0) {

			printf("ERROR: init_ns_worker_ctx() failed\n");

			return 1;

		}

		ns_ctx = ns_ctx->next;

	}



	tsc_end = rte_get_timer_cycles() + g_time_in_sec * g_tsc_rate;



	/* Submit initial I/O for each namespace. */

	ns_ctx = worker->ns_ctx;

	while (ns_ctx != NULL) {
@@ -527,11 +585,10 @@ work_fn(void *arg) 
	ns_ctx = worker->ns_ctx;

	while (ns_ctx != NULL) {

		drain_io(ns_ctx);

		cleanup_ns_worker_ctx(ns_ctx);

		ns_ctx = ns_ctx->next;

	}



	spdk_nvme_unregister_io_thread();



	return 0;

}
@@ -928,20 +985,6 @@ associate_workers_with_ns(void) 
			return -1;

		}

		memset(ns_ctx, 0, sizeof(*ns_ctx));

#ifdef HAVE_LIBAIO

		ns_ctx->events = calloc(g_queue_depth, sizeof(struct io_event));

		if (!ns_ctx->events) {

			free(ns_ctx);

			return -1;

		}

		ns_ctx->ctx = 0;

		if (io_setup(g_queue_depth, &ns_ctx->ctx) < 0) {

			free(ns_ctx->events);

			free(ns_ctx);

			perror("io_setup");

			return -1;

		}

#endif



		printf("Associating %s with lcore %d\n", entry->name, worker->lcore);

		ns_ctx->entry = entry;

examples/nvme/reserve/reservation.c

+29 −26
Original line number Diff line number Diff line
@@ -187,7 +187,8 @@ reservation_ns_completion(void *cb_arg, const struct spdk_nvme_cpl *cpl) 
}



static int

reservation_ns_register(struct spdk_nvme_ctrlr *ctrlr, uint16_t ns_id)

reservation_ns_register(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair,

			uint16_t ns_id)

{

	int ret;

	struct spdk_nvme_reservation_register_data *rr_data;
@@ -202,7 +203,7 @@ reservation_ns_register(struct spdk_nvme_ctrlr *ctrlr, uint16_t ns_id) 
	outstanding_commands = 0;

	reserve_command_result = -1;



	ret = spdk_nvme_ns_cmd_reservation_register(ns, rr_data, 1,

	ret = spdk_nvme_ns_cmd_reservation_register(ns, qpair, rr_data, 1,

			SPDK_NVME_RESERVE_REGISTER_KEY,

			SPDK_NVME_RESERVE_PTPL_NO_CHANGES,

			reservation_ns_completion, NULL);
@@ -214,7 +215,7 @@ reservation_ns_register(struct spdk_nvme_ctrlr *ctrlr, uint16_t ns_id) 


	outstanding_commands++;

	while (outstanding_commands) {

		spdk_nvme_ctrlr_process_io_completions(ctrlr, 100);

		spdk_nvme_qpair_process_completions(qpair, 100);

	}



	if (reserve_command_result)
@@ -225,7 +226,7 @@ reservation_ns_register(struct spdk_nvme_ctrlr *ctrlr, uint16_t ns_id) 
}



static int

reservation_ns_report(struct spdk_nvme_ctrlr *ctrlr, uint16_t ns_id)

reservation_ns_report(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair, uint16_t ns_id)

{

	int ret, i;

	uint8_t *payload;
@@ -239,7 +240,7 @@ reservation_ns_report(struct spdk_nvme_ctrlr *ctrlr, uint16_t ns_id) 
	outstanding_commands = 0;

	reserve_command_result = -1;



	ret = spdk_nvme_ns_cmd_reservation_report(ns, payload, 0x1000,

	ret = spdk_nvme_ns_cmd_reservation_report(ns, qpair, payload, 0x1000,

			reservation_ns_completion, NULL);

	if (ret) {

		fprintf(stderr, "Reservation Report Failed\n");
@@ -249,7 +250,7 @@ reservation_ns_report(struct spdk_nvme_ctrlr *ctrlr, uint16_t ns_id) 


	outstanding_commands++;

	while (outstanding_commands) {

		spdk_nvme_ctrlr_process_io_completions(ctrlr, 100);

		spdk_nvme_qpair_process_completions(qpair, 100);

	}



	if (reserve_command_result) {
@@ -277,7 +278,7 @@ reservation_ns_report(struct spdk_nvme_ctrlr *ctrlr, uint16_t ns_id) 
}



static int

reservation_ns_acquire(struct spdk_nvme_ctrlr *ctrlr, uint16_t ns_id)

reservation_ns_acquire(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair, uint16_t ns_id)

{

	int ret;

	struct spdk_nvme_reservation_acquire_data *cdata;
@@ -290,7 +291,7 @@ reservation_ns_acquire(struct spdk_nvme_ctrlr *ctrlr, uint16_t ns_id) 
	outstanding_commands = 0;

	reserve_command_result = -1;



	ret = spdk_nvme_ns_cmd_reservation_acquire(ns, cdata,

	ret = spdk_nvme_ns_cmd_reservation_acquire(ns, qpair, cdata,

			0,

			SPDK_NVME_RESERVE_ACQUIRE,

			SPDK_NVME_RESERVE_WRITE_EXCLUSIVE,
@@ -303,7 +304,7 @@ reservation_ns_acquire(struct spdk_nvme_ctrlr *ctrlr, uint16_t ns_id) 


	outstanding_commands++;

	while (outstanding_commands) {

		spdk_nvme_ctrlr_process_io_completions(ctrlr, 100);

		spdk_nvme_qpair_process_completions(qpair, 100);

	}



	if (reserve_command_result)
@@ -314,7 +315,7 @@ reservation_ns_acquire(struct spdk_nvme_ctrlr *ctrlr, uint16_t ns_id) 
}



static int

reservation_ns_release(struct spdk_nvme_ctrlr *ctrlr, uint16_t ns_id)

reservation_ns_release(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair, uint16_t ns_id)

{

	int ret;

	struct spdk_nvme_reservation_key_data *cdata;
@@ -327,7 +328,7 @@ reservation_ns_release(struct spdk_nvme_ctrlr *ctrlr, uint16_t ns_id) 
	outstanding_commands = 0;

	reserve_command_result = -1;



	ret = spdk_nvme_ns_cmd_reservation_release(ns, cdata,

	ret = spdk_nvme_ns_cmd_reservation_release(ns, qpair, cdata,

			0,

			SPDK_NVME_RESERVE_RELEASE,

			SPDK_NVME_RESERVE_WRITE_EXCLUSIVE,
@@ -340,7 +341,7 @@ reservation_ns_release(struct spdk_nvme_ctrlr *ctrlr, uint16_t ns_id) 


	outstanding_commands++;

	while (outstanding_commands) {

		spdk_nvme_ctrlr_process_io_completions(ctrlr, 100);

		spdk_nvme_qpair_process_completions(qpair, 100);

	}



	if (reserve_command_result)
@@ -351,7 +352,8 @@ reservation_ns_release(struct spdk_nvme_ctrlr *ctrlr, uint16_t ns_id) 
}



static void

reserve_controller(struct spdk_nvme_ctrlr *ctrlr, struct spdk_pci_device *pci_dev)

reserve_controller(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair,

		   struct spdk_pci_device *pci_dev)

{

	const struct spdk_nvme_ctrlr_data	*cdata;
@@ -373,10 +375,10 @@ reserve_controller(struct spdk_nvme_ctrlr *ctrlr, struct spdk_pci_device *pci_de 
	get_host_identifier(ctrlr);



	/* tested 1 namespace */

	reservation_ns_register(ctrlr, 1);

	reservation_ns_acquire(ctrlr, 1);

	reservation_ns_report(ctrlr, 1);

	reservation_ns_release(ctrlr, 1);

	reservation_ns_register(ctrlr, qpair, 1);

	reservation_ns_acquire(ctrlr, qpair, 1);

	reservation_ns_report(ctrlr, qpair, 1);

	reservation_ns_release(ctrlr, qpair, 1);

}



static bool
@@ -441,14 +443,18 @@ int main(int argc, char **argv) 
		return 1;

	}



	if (num_devs) {

		rc = spdk_nvme_register_io_thread();

		if (rc != 0)

			return rc;

	}

	rc = 0;



	foreach_dev(iter) {

		reserve_controller(iter->ctrlr, iter->pci_dev);

		struct spdk_nvme_qpair *qpair;



		qpair = spdk_nvme_ctrlr_alloc_io_qpair(iter->ctrlr, 0);

		if (!qpair) {

			fprintf(stderr, "spdk_nvme_ctrlr_alloc_io_qpair() failed\n");

			rc = 1;

		} else {

			reserve_controller(iter->ctrlr, qpair, iter->pci_dev);

		}

	}



	printf("Cleaning up...\n");
@@ -458,8 +464,5 @@ int main(int argc, char **argv) 
		spdk_nvme_detach(dev->ctrlr);

	}



	if (num_devs)

		spdk_nvme_unregister_io_thread();



	return rc;

}