nvme: Add support for hotplug. (e8c63cdd) · Commits · Public Repositories / spdk

lib/nvme/nvme.c

+130 −43

Original line number	Diff line number	Diff line
		@@ -33,9 +33,11 @@

		#include "spdk/nvmf_spec.h"
		#include "nvme_internal.h"
		#include "nvme_uevent.h"

		struct nvme_driver _g_nvme_driver = {
		.lock = PTHREAD_MUTEX_INITIALIZER,
		.hotplug_fd = -1,
		.init_ctrlrs = TAILQ_HEAD_INITIALIZER(_g_nvme_driver.init_ctrlrs),
		.attached_ctrlrs = TAILQ_HEAD_INITIALIZER(_g_nvme_driver.attached_ctrlrs),
		.request_mempool = NULL,
		@@ -254,51 +256,14 @@ nvme_probe_one(enum spdk_nvme_transport transport, spdk_nvme_probe_cb probe_cb,
		}

		static int
		_spdk_nvme_probe(const struct spdk_nvme_discover_info info, void cb_ctx,
		spdk_nvme_probe_cb probe_cb, spdk_nvme_attach_cb attach_cb,
		spdk_nvme_remove_cb remove_cb)
		nvme_init_controllers(void *cb_ctx, spdk_nvme_attach_cb attach_cb)
		{
		int rc, start_rc;
		int rc = 0;
		int start_rc;
		struct spdk_nvme_ctrlr ctrlr, ctrlr_tmp;
		enum spdk_nvme_transport transport;

		if (!spdk_process_is_primary()) {
		while (g_spdk_nvme_driver->initialized == false) {
		usleep(200 * 1000);
		}
		}

		pthread_mutex_lock(&g_spdk_nvme_driver->lock);

		if (g_spdk_nvme_driver->request_mempool == NULL) {
		g_spdk_nvme_driver->request_mempool = spdk_mempool_create("nvme_request", 8192,
		sizeof(struct nvme_request), -1);
		if (g_spdk_nvme_driver->request_mempool == NULL) {
		SPDK_ERRLOG("Unable to allocate pool of requests\n");
		pthread_mutex_unlock(&g_spdk_nvme_driver->lock);
		return -1;
		}
		}

		if (!info) {
		transport = SPDK_NVME_TRANSPORT_PCIE;
		} else {
		if (!spdk_nvme_transport_available(info->trtype)) {
		SPDK_ERRLOG("NVMe over Fabrics trtype %u not available\n", info->trtype);
		pthread_mutex_unlock(&g_spdk_nvme_driver->lock);
		return -1;
		}

		transport = (uint8_t)info->trtype;
		}

		rc = nvme_transport_ctrlr_scan(transport, probe_cb, cb_ctx, (void *)info, NULL);

		/*
		* Keep going even if one or more nvme_attach() calls failed,
		* but maintain the value of rc to signal errors when we return.
		*/

		/* Initialize all new controllers in the init_ctrlrs list in parallel. */
		while (!TAILQ_EMPTY(&g_spdk_nvme_driver->init_ctrlrs)) {
		TAILQ_FOREACH_SAFE(ctrlr, &g_spdk_nvme_driver->init_ctrlrs, tailq, ctrlr_tmp) {
		@@ -349,8 +314,75 @@ _spdk_nvme_probe(const struct spdk_nvme_discover_info info, void cb_ctx,
		}
		}

		g_spdk_nvme_driver->initialized = true;
		pthread_mutex_unlock(&g_spdk_nvme_driver->lock);
		return rc;
		}

		static int
		nvme_attach_one(void *cb_ctx, spdk_nvme_probe_cb probe_cb, spdk_nvme_attach_cb attach_cb,
		struct spdk_pci_addr *pci_address)
		{
		nvme_transport_ctrlr_scan(SPDK_NVME_TRANSPORT_PCIE, probe_cb, cb_ctx, NULL, pci_address);
		return nvme_init_controllers(cb_ctx, attach_cb);
		}

		static int
		_spdk_nvme_probe(const struct spdk_nvme_discover_info info, void cb_ctx,
		spdk_nvme_probe_cb probe_cb, spdk_nvme_attach_cb attach_cb,
		spdk_nvme_remove_cb remove_cb)
		{
		int rc;
		enum spdk_nvme_transport transport;

		if (!spdk_process_is_primary()) {
		while (g_spdk_nvme_driver->initialized == false) {
		usleep(200 * 1000);
		}
		}

		pthread_mutex_lock(&g_spdk_nvme_driver->lock);

		if (g_spdk_nvme_driver->hotplug_fd < 0) {
		g_spdk_nvme_driver->hotplug_fd = spdk_uevent_connect();
		if (g_spdk_nvme_driver->hotplug_fd < 0) {
		SPDK_ERRLOG("Failed to open uevent netlink socket\n");
		}
		}

		if (g_spdk_nvme_driver->request_mempool == NULL) {
		g_spdk_nvme_driver->request_mempool = spdk_mempool_create("nvme_request", 8192,
		sizeof(struct nvme_request), -1);
		if (g_spdk_nvme_driver->request_mempool == NULL) {
		SPDK_ERRLOG("Unable to allocate pool of requests\n");
		pthread_mutex_unlock(&g_spdk_nvme_driver->lock);
		return -1;
		}
		}

		if (!info) {
		transport = SPDK_NVME_TRANSPORT_PCIE;
		} else {
		if (!spdk_nvme_transport_available(info->trtype)) {
		SPDK_ERRLOG("NVMe over Fabrics trtype %u not available\n", info->trtype);
		pthread_mutex_unlock(&g_spdk_nvme_driver->lock);
		return -1;
		}

		transport = (uint8_t)info->trtype;
		}

		nvme_transport_ctrlr_scan(transport, probe_cb, cb_ctx, (void *)info, NULL);

		pthread_mutex_unlock(&g_spdk_nvme_driver->lock);
		/*
		* Keep going even if one or more nvme_attach() calls failed,
		* but maintain the value of rc to signal errors when we return.
		*/

		rc = nvme_init_controllers(cb_ctx, attach_cb);

		pthread_mutex_lock(&g_spdk_nvme_driver->lock);
		g_spdk_nvme_driver->initialized = true;
		pthread_mutex_unlock(&g_spdk_nvme_driver->lock);
		return rc;
		}
		@@ -367,11 +399,66 @@ int spdk_nvme_discover(const struct spdk_nvme_discover_info info, void cb_ctx,
		return _spdk_nvme_probe(info, cb_ctx, probe_cb, attach_cb, remove_cb);
		}

		static int
		nvme_hotplug_monitor(void *cb_ctx, spdk_nvme_probe_cb probe_cb, spdk_nvme_attach_cb attach_cb,
		spdk_nvme_remove_cb remove_cb)
		{
		int rc = 0;
		struct spdk_nvme_ctrlr *ctrlr;
		struct spdk_uevent event;
		struct nvme_driver *nvme_driver = g_spdk_nvme_driver;

		while (spdk_get_uevent(nvme_driver->hotplug_fd, &event) > 0) {
		if (event.subsystem == SPDK_NVME_UEVENT_SUBSYSTEM_UIO) {
		if (event.action == SPDK_NVME_UEVENT_ADD) {
		SPDK_TRACELOG(SPDK_TRACE_NVME, "add nvme address: %04x:%02x:%02x.%u\n",
		event.pci_addr.domain, event.pci_addr.bus, event.pci_addr.dev, event.pci_addr.func);
		if (spdk_process_is_primary()) {
		nvme_attach_one(cb_ctx, probe_cb, attach_cb, &event.pci_addr);
		}
		} else if (event.action == SPDK_NVME_UEVENT_REMOVE) {
		bool in_list = false;

		TAILQ_FOREACH(ctrlr, &g_spdk_nvme_driver->attached_ctrlrs, tailq) {
		if (spdk_pci_addr_compare(&event.pci_addr, &ctrlr->probe_info.pci_addr) == 0) {
		in_list = true;
		break;
		}
		}
		if (in_list == false) {
		return 0;
		}
		SPDK_TRACELOG(SPDK_TRACE_NVME, "remove nvme address: %04x:%02x:%02x.%u\n",
		event.pci_addr.domain, event.pci_addr.bus, event.pci_addr.dev, event.pci_addr.func);

		nvme_ctrlr_fail(ctrlr, true);

		/* get the user app to clean up and stop I/O */
		if (remove_cb) {
		remove_cb(cb_ctx, ctrlr);
		}
		if (spdk_process_is_primary()) {
		rc = spdk_nvme_detach(ctrlr);
		if (rc) {
		SPDK_ERRLOG("Failed to hot detach nvme address: %04x:%04x:%04x.%u\n",
		event.pci_addr.domain, event.pci_addr.bus, event.pci_addr.dev,
		event.pci_addr.func);
		}
		}
		}
		}
		}
		return 0;
		}

		int
		spdk_nvme_probe(void *cb_ctx, spdk_nvme_probe_cb probe_cb, spdk_nvme_attach_cb attach_cb,
		spdk_nvme_remove_cb remove_cb)
		{
		if (g_spdk_nvme_driver->hotplug_fd < 0) {
		return _spdk_nvme_probe(NULL, cb_ctx, probe_cb, attach_cb, remove_cb);
		} else {
		return nvme_hotplug_monitor(cb_ctx, probe_cb, attach_cb, remove_cb);
		}
		}

		SPDK_LOG_REGISTER_TRACE_FLAG("nvme", SPDK_TRACE_NVME)

lib/nvme/nvme_ctrlr.c

+16 −9

Original line number	Diff line number	Diff line
		@@ -289,11 +289,14 @@ nvme_ctrlr_set_supported_features(struct spdk_nvme_ctrlr *ctrlr)
		}
		}

		static void
		nvme_ctrlr_fail(struct spdk_nvme_ctrlr *ctrlr)
		void
		nvme_ctrlr_fail(struct spdk_nvme_ctrlr *ctrlr, bool hot_remove)
		{
		struct spdk_nvme_qpair *qpair;

		if (hot_remove) {
		ctrlr->is_removed = true;
		}
		ctrlr->is_failed = true;
		nvme_qpair_fail(ctrlr->adminq);
		TAILQ_FOREACH(qpair, &ctrlr->active_io_qpairs, tailq) {
		@@ -308,6 +311,10 @@ nvme_ctrlr_shutdown(struct spdk_nvme_ctrlr *ctrlr)
		union spdk_nvme_csts_register csts;
		int ms_waited = 0;

		if (ctrlr->is_removed) {
		return;
		}

		if (nvme_ctrlr_get_cc(ctrlr, &cc)) {
		SPDK_ERRLOG("get_cc() failed\n");
		return;
		@@ -471,7 +478,7 @@ spdk_nvme_ctrlr_reset(struct spdk_nvme_ctrlr *ctrlr)
		while (ctrlr->state != NVME_CTRLR_STATE_READY) {
		if (nvme_ctrlr_process_init(ctrlr) != 0) {
		SPDK_ERRLOG("%s: controller reinitialization failed\n", __func__);
		nvme_ctrlr_fail(ctrlr);
		nvme_ctrlr_fail(ctrlr, false);
		rc = -1;
		break;
		}
		@@ -481,7 +488,7 @@ spdk_nvme_ctrlr_reset(struct spdk_nvme_ctrlr *ctrlr)
		/* Reinitialize qpairs */
		TAILQ_FOREACH(qpair, &ctrlr->active_io_qpairs, tailq) {
		if (nvme_transport_ctrlr_reinit_io_qpair(ctrlr, qpair) != 0) {
		nvme_ctrlr_fail(ctrlr);
		nvme_ctrlr_fail(ctrlr, false);
		rc = -1;
		}
		}
		@@ -1013,7 +1020,7 @@ nvme_ctrlr_process_init(struct spdk_nvme_ctrlr *ctrlr)
		if (nvme_ctrlr_get_cc(ctrlr, &cc) \|\|
		nvme_ctrlr_get_csts(ctrlr, &csts)) {
		SPDK_ERRLOG("get registers failed\n");
		nvme_ctrlr_fail(ctrlr);
		nvme_ctrlr_fail(ctrlr, false);
		return -EIO;
		}

		@@ -1044,7 +1051,7 @@ nvme_ctrlr_process_init(struct spdk_nvme_ctrlr *ctrlr)
		cc.bits.en = 0;
		if (nvme_ctrlr_set_cc(ctrlr, &cc)) {
		SPDK_ERRLOG("set_cc() failed\n");
		nvme_ctrlr_fail(ctrlr);
		nvme_ctrlr_fail(ctrlr, false);
		return -EIO;
		}
		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0, ready_timeout_in_ms);
		@@ -1087,7 +1094,7 @@ nvme_ctrlr_process_init(struct spdk_nvme_ctrlr *ctrlr)
		cc.bits.en = 0;
		if (nvme_ctrlr_set_cc(ctrlr, &cc)) {
		SPDK_ERRLOG("set_cc() failed\n");
		nvme_ctrlr_fail(ctrlr);
		nvme_ctrlr_fail(ctrlr, false);
		return -EIO;
		}
		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0, ready_timeout_in_ms);
		@@ -1121,14 +1128,14 @@ nvme_ctrlr_process_init(struct spdk_nvme_ctrlr *ctrlr)

		default:
		assert(0);
		nvme_ctrlr_fail(ctrlr);
		nvme_ctrlr_fail(ctrlr, false);
		return -1;
		}

		if (ctrlr->state_timeout_tsc != NVME_TIMEOUT_INFINITE &&
		spdk_get_ticks() > ctrlr->state_timeout_tsc) {
		SPDK_ERRLOG("Initialization timed out in state %d\n", ctrlr->state);
		nvme_ctrlr_fail(ctrlr);
		nvme_ctrlr_fail(ctrlr, false);
		return -1;
		}

lib/nvme/nvme_internal.h

+4 −0

Original line number	Diff line number	Diff line
		@@ -341,6 +341,8 @@ struct spdk_nvme_ctrlr {

		uint32_t num_ns;

		bool is_removed;

		bool is_resetting;

		bool is_failed;
		@@ -413,6 +415,7 @@ struct spdk_nvme_ctrlr {

		struct nvme_driver {
		pthread_mutex_t lock;
		int hotplug_fd;
		TAILQ_HEAD(, spdk_nvme_ctrlr) init_ctrlrs;
		TAILQ_HEAD(, spdk_nvme_ctrlr) attached_ctrlrs;
		struct spdk_mempool *request_mempool;
		@@ -496,6 +499,7 @@ int nvme_probe_one(enum spdk_nvme_transport type, spdk_nvme_probe_cb probe_cb, v

		int nvme_ctrlr_construct(struct spdk_nvme_ctrlr *ctrlr);
		void nvme_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr);
		void nvme_ctrlr_fail(struct spdk_nvme_ctrlr *ctrlr, bool hot_remove);
		int nvme_ctrlr_process_init(struct spdk_nvme_ctrlr *ctrlr);
		int nvme_ctrlr_start(struct spdk_nvme_ctrlr *ctrlr);

lib/nvme/nvme_pcie.c

+10 −1

Original line number	Diff line number	Diff line
		@@ -483,6 +483,10 @@ nvme_pcie_ctrlr_free_bars(struct nvme_pcie_ctrlr *pctrlr)
		int rc = 0;
		void addr = (void )pctrlr->regs;

		if (pctrlr->ctrlr.is_removed) {
		return rc;
		}

		rc = nvme_pcie_ctrlr_unmap_cmb(pctrlr);
		if (rc != 0) {
		SPDK_ERRLOG("nvme_ctrlr_unmap_cmb failed with error code %d\n", rc);
		@@ -577,6 +581,7 @@ struct spdk_nvme_ctrlr *nvme_pcie_ctrlr_construct(enum spdk_nvme_transport trans
		}

		pctrlr->is_remapped = false;
		pctrlr->ctrlr.is_removed = false;
		pctrlr->ctrlr.transport = SPDK_NVME_TRANSPORT_PCIE;
		pctrlr->devhandle = devhandle;
		pctrlr->ctrlr.opts = *opts;
		@@ -1343,6 +1348,10 @@ nvme_pcie_ctrlr_delete_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_

		assert(ctrlr != NULL);

		if (ctrlr->is_removed) {
		goto free;
		}

		/* Delete the I/O submission queue and then the completion queue */

		status.done = false;
		@@ -1369,8 +1378,8 @@ nvme_pcie_ctrlr_delete_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_
		return -1;
		}

		free:
		nvme_pcie_qpair_destroy(qpair);

		return 0;
		}

test/lib/nvme/unit/nvme_c/nvme_ut.c

+17 −0

Original line number	Diff line number	Diff line
		@@ -101,6 +101,23 @@ nvme_ctrlr_start(struct spdk_nvme_ctrlr *ctrlr)
		return 0;
		}

		void
		nvme_ctrlr_fail(struct spdk_nvme_ctrlr *ctrlr, bool hot_remove)
		{
		}

		int
		spdk_uevent_connect(void)
		{
		return 0;
		}

		int
		spdk_get_uevent(int fd, struct spdk_uevent *uevent)
		{
		return 0;
		}

		void
		spdk_nvme_ctrlr_opts_set_defaults(struct spdk_nvme_ctrlr_opts *opts)
		{