Commit 0c6979e2 authored by Seth Howell's avatar Seth Howell Committed by Jim Harris
Browse files

bdev/nvme: add a function for specifying a multipath trid. 



This is part of a larger series enabling failover at the bdev
layer for NVMe.

Signed-off-by: default avatarSeth Howell <seth.howell@intel.com>
Change-Id: Ic5c128244699c1a47275145ca7e41aa5f1366259
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/3044


Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarAleksey Marchuk <alexeymar@mellanox.com>
parent d88915f5

module/bdev/nvme/bdev_nvme.c

+96 −12
Original line number Diff line number Diff line
@@ -1361,6 +1361,7 @@ create_ctrlr(struct spdk_nvme_ctrlr *ctrlr, 
	     uint32_t prchk_flags)

{

	struct nvme_bdev_ctrlr *nvme_bdev_ctrlr;

	struct nvme_bdev_ctrlr_trid *trid_entry;

	uint32_t i;

	int rc;
@@ -1370,22 +1371,25 @@ create_ctrlr(struct spdk_nvme_ctrlr *ctrlr, 
		return -ENOMEM;

	}



	nvme_bdev_ctrlr->connected_trid = calloc(1, sizeof(*nvme_bdev_ctrlr->connected_trid));

	if (nvme_bdev_ctrlr->connected_trid == NULL) {

		SPDK_ERRLOG("Failed to allocate device trid struct\n");

		free(nvme_bdev_ctrlr);

		return -ENOMEM;

	}



	TAILQ_INIT(&nvme_bdev_ctrlr->trids);

	nvme_bdev_ctrlr->num_ns = spdk_nvme_ctrlr_get_num_ns(ctrlr);

	nvme_bdev_ctrlr->namespaces = calloc(nvme_bdev_ctrlr->num_ns, sizeof(struct nvme_bdev_ns *));

	if (!nvme_bdev_ctrlr->namespaces) {

		SPDK_ERRLOG("Failed to allocate block namespaces pointer\n");

		free(nvme_bdev_ctrlr->connected_trid);

		free(nvme_bdev_ctrlr);

		return -ENOMEM;

	}



	trid_entry = calloc(1, sizeof(*trid_entry));

	if (trid_entry == NULL) {

		SPDK_ERRLOG("Failed to allocate trid entry pointer\n");

		free(nvme_bdev_ctrlr->namespaces);

		free(nvme_bdev_ctrlr);

		return -ENOMEM;

	}



	trid_entry->trid = *trid;



	for (i = 0; i < nvme_bdev_ctrlr->num_ns; i++) {

		nvme_bdev_ctrlr->namespaces[i] = calloc(1, sizeof(struct nvme_bdev_ns));

		if (nvme_bdev_ctrlr->namespaces[i] == NULL) {
@@ -1393,8 +1397,8 @@ create_ctrlr(struct spdk_nvme_ctrlr *ctrlr, 
			for (; i > 0; i--) {

				free(nvme_bdev_ctrlr->namespaces[i - 1]);

			}

			free(trid_entry);

			free(nvme_bdev_ctrlr->namespaces);

			free(nvme_bdev_ctrlr->connected_trid);

			free(nvme_bdev_ctrlr);

			return -ENOMEM;

		}
@@ -1404,11 +1408,11 @@ create_ctrlr(struct spdk_nvme_ctrlr *ctrlr, 
	nvme_bdev_ctrlr->adminq_timer_poller = NULL;

	nvme_bdev_ctrlr->ctrlr = ctrlr;

	nvme_bdev_ctrlr->ref = 0;

	*nvme_bdev_ctrlr->connected_trid = *trid;

	nvme_bdev_ctrlr->connected_trid = &trid_entry->trid;

	nvme_bdev_ctrlr->name = strdup(name);

	if (nvme_bdev_ctrlr->name == NULL) {

		free(trid_entry);

		free(nvme_bdev_ctrlr->namespaces);

		free(nvme_bdev_ctrlr->connected_trid);

		free(nvme_bdev_ctrlr);

		return -ENOMEM;

	}
@@ -1417,9 +1421,9 @@ create_ctrlr(struct spdk_nvme_ctrlr *ctrlr, 
		rc = bdev_ocssd_init_ctrlr(nvme_bdev_ctrlr);

		if (spdk_unlikely(rc != 0)) {

			SPDK_ERRLOG("Unable to initialize OCSSD controller\n");

			free(trid_entry);

			free(nvme_bdev_ctrlr->name);

			free(nvme_bdev_ctrlr->namespaces);

			free(nvme_bdev_ctrlr->connected_trid);

			free(nvme_bdev_ctrlr);

			return rc;

		}
@@ -1450,6 +1454,8 @@ create_ctrlr(struct spdk_nvme_ctrlr *ctrlr, 
			SPDK_ERRLOG("Failed to initialize Opal\n");

		}

	}



	TAILQ_INSERT_HEAD(&nvme_bdev_ctrlr->trids, trid_entry, link);

	return 0;

}
@@ -1722,6 +1728,84 @@ bdev_nvme_async_poll(void *arg) 
	return SPDK_POLLER_BUSY;

}



int

bdev_nvme_add_trid(const char *name, struct spdk_nvme_transport_id *trid)

{

	struct nvme_bdev_ctrlr		*nvme_bdev_ctrlr;

	struct spdk_nvme_ctrlr		*new_ctrlr;

	struct spdk_nvme_ctrlr_opts	opts;

	uint32_t			i;

	struct spdk_nvme_ns		*ns, *new_ns;

	const struct spdk_nvme_ns_data	*ns_data, *new_ns_data;

	struct nvme_bdev_ctrlr_trid	*new_trid;

	int				rc = 0;



	assert(name != NULL);



	nvme_bdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name);

	if (nvme_bdev_ctrlr == NULL) {

		SPDK_ERRLOG("Failed to find NVMe controller\n");

		return -ENODEV;

	}



	/* Currently we only support failover to the same transport type. */

	if (nvme_bdev_ctrlr->connected_trid->trtype != trid->trtype) {

		return -EINVAL;

	}



	/* Currently we only support failover to the same NQN. */

	if (strncmp(trid->subnqn, nvme_bdev_ctrlr->connected_trid->subnqn, SPDK_NVMF_NQN_MAX_LEN)) {

		return -EINVAL;

	}



	/* Skip all the other checks if we've already registered this path. */

	TAILQ_FOREACH(new_trid, &nvme_bdev_ctrlr->trids, link) {

		if (!spdk_nvme_transport_id_compare(&new_trid->trid, trid)) {

			return -EEXIST;

		}

	}



	spdk_nvme_ctrlr_get_default_ctrlr_opts(&opts, sizeof(opts));

	opts.transport_retry_count = g_opts.retry_count;



	new_ctrlr = spdk_nvme_connect(trid, &opts, sizeof(opts));



	if (new_ctrlr == NULL) {

		return -ENODEV;

	}



	if (spdk_nvme_ctrlr_get_num_ns(new_ctrlr) != nvme_bdev_ctrlr->num_ns) {

		rc = -EINVAL;

		goto out;

	}



	for (i = 1; i <= nvme_bdev_ctrlr->num_ns; i++) {

		ns = spdk_nvme_ctrlr_get_ns(nvme_bdev_ctrlr->ctrlr, i);

		new_ns = spdk_nvme_ctrlr_get_ns(new_ctrlr, i);

		assert(ns != NULL);

		assert(new_ns != NULL);



		ns_data = spdk_nvme_ns_get_data(ns);

		new_ns_data = spdk_nvme_ns_get_data(new_ns);

		if (memcmp(ns_data->nguid, new_ns_data->nguid, sizeof(ns_data->nguid))) {

			rc = -EINVAL;

			goto out;

		}

	}



	new_trid = calloc(1, sizeof(*new_trid));

	if (new_trid == NULL) {

		rc = -ENOMEM;

		goto out;

	}

	new_trid->trid = *trid;

	TAILQ_INSERT_TAIL(&nvme_bdev_ctrlr->trids, new_trid, link);



out:

	spdk_nvme_detach(new_ctrlr);

	return rc;

}



int

bdev_nvme_create(struct spdk_nvme_transport_id *trid,

		 struct spdk_nvme_host_id *hostid,

module/bdev/nvme/bdev_nvme.h

+1 −0
Original line number Diff line number Diff line
@@ -66,6 +66,7 @@ struct spdk_nvme_qpair *bdev_nvme_get_io_qpair(struct spdk_io_channel *ctrlr_io_ 
void bdev_nvme_get_opts(struct spdk_bdev_nvme_opts *opts);

int bdev_nvme_set_opts(const struct spdk_bdev_nvme_opts *opts);

int bdev_nvme_set_hotplug(bool enabled, uint64_t period_us, spdk_msg_fn cb, void *cb_ctx);

int bdev_nvme_add_trid(const char *name, struct spdk_nvme_transport_id *trid);



int bdev_nvme_create(struct spdk_nvme_transport_id *trid,

		     struct spdk_nvme_host_id *hostid,

module/bdev/nvme/common.c

+7 −1
Original line number Diff line number Diff line
@@ -116,6 +116,7 @@ static void 
nvme_bdev_unregister_cb(void *io_device)

{

	struct nvme_bdev_ctrlr *nvme_bdev_ctrlr = io_device;

	struct nvme_bdev_ctrlr_trid *trid, *tmp_trid;

	uint32_t i;



	pthread_mutex_lock(&g_bdev_nvme_mutex);
@@ -127,8 +128,13 @@ nvme_bdev_unregister_cb(void *io_device) 
	for (i = 0; i < nvme_bdev_ctrlr->num_ns; i++) {

		free(nvme_bdev_ctrlr->namespaces[i]);

	}



	TAILQ_FOREACH_SAFE(trid, &nvme_bdev_ctrlr->trids, link, tmp_trid) {

		TAILQ_REMOVE(&nvme_bdev_ctrlr->trids, trid, link);

		free(trid);

	}



	free(nvme_bdev_ctrlr->namespaces);

	free(nvme_bdev_ctrlr->connected_trid);

	free(nvme_bdev_ctrlr);



	pthread_mutex_lock(&g_bdev_nvme_mutex);

module/bdev/nvme/common.h

+22 −15
Original line number Diff line number Diff line
@@ -69,6 +69,11 @@ struct nvme_bdev_ns { 


struct ocssd_bdev_ctrlr;



struct nvme_bdev_ctrlr_trid {

	struct spdk_nvme_transport_id		trid;

	TAILQ_ENTRY(nvme_bdev_ctrlr_trid)	link;

};



struct nvme_bdev_ctrlr {

	/**

	 * points to pinned, physically contiguous memory region;
@@ -101,6 +106,8 @@ struct nvme_bdev_ctrlr { 


	/** linked list pointer for device list */

	TAILQ_ENTRY(nvme_bdev_ctrlr)		tailq;



	TAILQ_HEAD(, nvme_bdev_ctrlr_trid)	trids;

};



struct nvme_bdev {

test/unit/lib/bdev/bdev_ocssd.c/bdev_ocssd_ut.c

+8 −4
Original line number Diff line number Diff line
@@ -198,6 +198,7 @@ create_nvme_bdev_controller(const struct spdk_nvme_transport_id *trid, const cha 
{

	struct spdk_nvme_ctrlr *ctrlr;

	struct nvme_bdev_ctrlr *nvme_bdev_ctrlr;

	struct nvme_bdev_ctrlr_trid *trid_entry;

	uint32_t nsid;



	ctrlr = find_controller(trid);
@@ -211,15 +212,15 @@ create_nvme_bdev_controller(const struct spdk_nvme_transport_id *trid, const cha 
	nvme_bdev_ctrlr->namespaces = calloc(ctrlr->ns_count, sizeof(struct nvme_bdev_ns *));

	SPDK_CU_ASSERT_FATAL(nvme_bdev_ctrlr->namespaces != NULL);



	nvme_bdev_ctrlr->connected_trid = calloc(1, sizeof(struct spdk_nvme_transport_id));

	SPDK_CU_ASSERT_FATAL(nvme_bdev_ctrlr->connected_trid != NULL);

	trid_entry = calloc(1, sizeof(struct nvme_bdev_ctrlr_trid));

	SPDK_CU_ASSERT_FATAL(trid_entry != NULL);

	trid_entry->trid = *trid;



	nvme_bdev_ctrlr->ctrlr = ctrlr;

	nvme_bdev_ctrlr->num_ns = ctrlr->ns_count;

	nvme_bdev_ctrlr->ref = 0;

	*nvme_bdev_ctrlr->connected_trid = *trid;

	nvme_bdev_ctrlr->connected_trid = &trid_entry->trid;

	nvme_bdev_ctrlr->name = strdup(name);



	for (nsid = 0; nsid < ctrlr->ns_count; ++nsid) {

		nvme_bdev_ctrlr->namespaces[nsid] = calloc(1, sizeof(struct nvme_bdev_ns));

		SPDK_CU_ASSERT_FATAL(nvme_bdev_ctrlr->namespaces[nsid] != NULL);
@@ -239,6 +240,9 @@ create_nvme_bdev_controller(const struct spdk_nvme_transport_id *trid, const cha 


	TAILQ_INSERT_TAIL(&g_nvme_bdev_ctrlrs, nvme_bdev_ctrlr, tailq);



	TAILQ_INIT(&nvme_bdev_ctrlr->trids);

	TAILQ_INSERT_HEAD(&nvme_bdev_ctrlr->trids, trid_entry, link);



	return nvme_bdev_ctrlr;

}