Commit 5f13e0f6 authored by Seth Howell's avatar Seth Howell Committed by Tomasz Zawadzki
Browse files

bdev/nvme: add a function for specifying a multipath trid. 



This is part of a larger series enabling failover at the bdev
layer for NVMe.

Signed-off-by: default avatarSeth Howell <seth.howell@intel.com>
Change-Id: I0d89e1afab8aeaa90237d0ba780d708154f6e3be
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/2881


Community-CI: Mellanox Build Bot
Community-CI: Broadcom CI
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarAleksey Marchuk <alexeymar@mellanox.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
parent af9e7330

module/bdev/nvme/bdev_nvme.c

+99 −13
Original line number Diff line number Diff line
@@ -1350,6 +1350,7 @@ create_ctrlr(struct spdk_nvme_ctrlr *ctrlr, 
	     uint32_t prchk_flags)

{

	struct nvme_bdev_ctrlr *nvme_bdev_ctrlr;

	struct nvme_bdev_ctrlr_trid *trid_entry;

	uint32_t i;

	int rc;
@@ -1358,23 +1359,25 @@ create_ctrlr(struct spdk_nvme_ctrlr *ctrlr, 
		SPDK_ERRLOG("Failed to allocate device struct\n");

		return -ENOMEM;

	}



	nvme_bdev_ctrlr->trid = calloc(1, sizeof(*nvme_bdev_ctrlr->trid));

	if (nvme_bdev_ctrlr->trid == NULL) {

		SPDK_ERRLOG("Failed to allocate device trid struct\n");

		free(nvme_bdev_ctrlr);

		return -ENOMEM;

	}



	TAILQ_INIT(&nvme_bdev_ctrlr->multipath_trids);

	nvme_bdev_ctrlr->num_ns = spdk_nvme_ctrlr_get_num_ns(ctrlr);

	nvme_bdev_ctrlr->namespaces = calloc(nvme_bdev_ctrlr->num_ns, sizeof(struct nvme_bdev_ns *));

	if (!nvme_bdev_ctrlr->namespaces) {

		SPDK_ERRLOG("Failed to allocate block namespaces pointer\n");

		free(nvme_bdev_ctrlr->trid);

		free(nvme_bdev_ctrlr);

		return -ENOMEM;

	}



	trid_entry = calloc(1, sizeof(*trid_entry));

	if (trid_entry == NULL) {

		SPDK_ERRLOG("Failed to allocate trid entry pointer\n");

		free(nvme_bdev_ctrlr->namespaces);

		free(nvme_bdev_ctrlr);

		return -ENOMEM;

	}



	trid_entry->trid = *trid;



	for (i = 0; i < nvme_bdev_ctrlr->num_ns; i++) {

		nvme_bdev_ctrlr->namespaces[i] = calloc(1, sizeof(struct nvme_bdev_ns));

		if (nvme_bdev_ctrlr->namespaces[i] == NULL) {
@@ -1382,8 +1385,8 @@ create_ctrlr(struct spdk_nvme_ctrlr *ctrlr, 
			for (; i > 0; i--) {

				free(nvme_bdev_ctrlr->namespaces[i - 1]);

			}

			free(trid_entry);

			free(nvme_bdev_ctrlr->namespaces);

			free(nvme_bdev_ctrlr->trid);

			free(nvme_bdev_ctrlr);

			return -ENOMEM;

		}
@@ -1392,11 +1395,11 @@ create_ctrlr(struct spdk_nvme_ctrlr *ctrlr, 
	nvme_bdev_ctrlr->adminq_timer_poller = NULL;

	nvme_bdev_ctrlr->ctrlr = ctrlr;

	nvme_bdev_ctrlr->ref = 0;

	*nvme_bdev_ctrlr->trid = *trid;

	nvme_bdev_ctrlr->trid = &trid_entry->trid;

	nvme_bdev_ctrlr->name = strdup(name);

	if (nvme_bdev_ctrlr->name == NULL) {

		free(trid_entry);

		free(nvme_bdev_ctrlr->namespaces);

		free(nvme_bdev_ctrlr->trid);

		free(nvme_bdev_ctrlr);

		return -ENOMEM;

	}
@@ -1405,9 +1408,9 @@ create_ctrlr(struct spdk_nvme_ctrlr *ctrlr, 
		rc = bdev_ocssd_init_ctrlr(nvme_bdev_ctrlr);

		if (spdk_unlikely(rc != 0)) {

			SPDK_ERRLOG("Unable to initialize OCSSD controller\n");

			free(trid_entry);

			free(nvme_bdev_ctrlr->name);

			free(nvme_bdev_ctrlr->namespaces);

			free(nvme_bdev_ctrlr->trid);

			free(nvme_bdev_ctrlr);

			return rc;

		}
@@ -1438,6 +1441,8 @@ create_ctrlr(struct spdk_nvme_ctrlr *ctrlr, 
			SPDK_ERRLOG("Failed to initialize Opal\n");

		}

	}



	TAILQ_INSERT_HEAD(&nvme_bdev_ctrlr->multipath_trids, trid_entry, link);

	return 0;

}
@@ -1711,6 +1716,87 @@ bdev_nvme_async_poll(void *arg) 
	return 1;

}



int

bdev_nvme_add_multipath_trid(const char *name, struct spdk_nvme_transport_id *trid)

{

	struct nvme_bdev_ctrlr		*nvme_bdev_ctrlr;

	struct spdk_nvme_ctrlr		*multipath_ctrlr;

	struct spdk_nvme_ctrlr_opts	opts;

	uint32_t			i;

	struct spdk_nvme_ns		*ns, *multipath_ns;

	const struct spdk_nvme_ns_data	*ns_data, *multipath_ns_data;

	struct nvme_bdev_ctrlr_trid	*multipath_trid;

	int				rc = 0;



	if (name == NULL) {

		return -EINVAL;

	}



	nvme_bdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name);

	if (nvme_bdev_ctrlr == NULL) {

		SPDK_ERRLOG("Failed to find NVMe controller\n");

		return -ENODEV;

	}



	/* Currently we only support failover to the same transport type. */

	if (nvme_bdev_ctrlr->trid->trtype != trid->trtype) {

		return -EINVAL;

	}



	/* Currently we only support failover to the same NQN. */

	if (strncmp(trid->subnqn, nvme_bdev_ctrlr->trid->subnqn, SPDK_NVMF_NQN_MAX_LEN)) {

		return -EINVAL;

	}



	/* Skip all the other checks if we've already registered this path. */

	TAILQ_FOREACH(multipath_trid, &nvme_bdev_ctrlr->multipath_trids, link) {

		if (!memcmp(&multipath_trid->trid, trid, sizeof(struct spdk_nvme_transport_id))) {

			return 0;

		}

	}



	spdk_nvme_ctrlr_get_default_ctrlr_opts(&opts, sizeof(opts));

	opts.transport_retry_count = g_opts.retry_count;



	multipath_ctrlr = spdk_nvme_connect(trid, &opts, sizeof(opts));



	if (multipath_ctrlr == NULL) {

		return -ENODEV;

	}



	if (spdk_nvme_ctrlr_get_num_ns(multipath_ctrlr) != nvme_bdev_ctrlr->num_ns) {

		rc = -EINVAL;

		goto out;

	}



	for (i = 1; i <= nvme_bdev_ctrlr->num_ns; i++) {

		ns = spdk_nvme_ctrlr_get_ns(nvme_bdev_ctrlr->ctrlr, i);

		multipath_ns = spdk_nvme_ctrlr_get_ns(multipath_ctrlr, i);

		assert(ns != NULL);

		assert(multipath_ns != NULL);



		ns_data = spdk_nvme_ns_get_data(ns);

		multipath_ns_data = spdk_nvme_ns_get_data(multipath_ns);

		if (memcmp(ns_data->nguid, multipath_ns_data->nguid, sizeof(ns_data->nguid))) {

			rc = -EINVAL;

			goto out;

		}

	}



out:

	spdk_nvme_detach(multipath_ctrlr);

	if (rc == 0) {

		multipath_trid = calloc(1, sizeof(*multipath_trid));

		if (multipath_trid == NULL) {

			return -ENOMEM;

		}

		multipath_trid->trid = *trid;

		TAILQ_INSERT_TAIL(&nvme_bdev_ctrlr->multipath_trids, multipath_trid, link);

	}



	return rc;

}



int

bdev_nvme_create(struct spdk_nvme_transport_id *trid,

		 struct spdk_nvme_host_id *hostid,

module/bdev/nvme/bdev_nvme.h

+1 −0
Original line number Diff line number Diff line
@@ -66,6 +66,7 @@ struct spdk_nvme_qpair *bdev_nvme_get_io_qpair(struct spdk_io_channel *ctrlr_io_ 
void bdev_nvme_get_opts(struct spdk_bdev_nvme_opts *opts);

int bdev_nvme_set_opts(const struct spdk_bdev_nvme_opts *opts);

int bdev_nvme_set_hotplug(bool enabled, uint64_t period_us, spdk_msg_fn cb, void *cb_ctx);

int bdev_nvme_add_multipath_trid(const char *name, struct spdk_nvme_transport_id *trid);



int bdev_nvme_create(struct spdk_nvme_transport_id *trid,

		     struct spdk_nvme_host_id *hostid,

module/bdev/nvme/common.c

+7 −1
Original line number Diff line number Diff line
@@ -116,6 +116,7 @@ static void 
nvme_bdev_unregister_cb(void *io_device)

{

	struct nvme_bdev_ctrlr *nvme_bdev_ctrlr = io_device;

	struct nvme_bdev_ctrlr_trid *multipath_trid, *tmp_trid;

	uint32_t i;



	pthread_mutex_lock(&g_bdev_nvme_mutex);
@@ -127,8 +128,13 @@ nvme_bdev_unregister_cb(void *io_device) 
	for (i = 0; i < nvme_bdev_ctrlr->num_ns; i++) {

		free(nvme_bdev_ctrlr->namespaces[i]);

	}



	TAILQ_FOREACH_SAFE(multipath_trid, &nvme_bdev_ctrlr->multipath_trids, link, tmp_trid) {

		TAILQ_REMOVE(&nvme_bdev_ctrlr->multipath_trids, multipath_trid, link);

		free(multipath_trid);

	}



	free(nvme_bdev_ctrlr->namespaces);

	free(nvme_bdev_ctrlr->trid);

	free(nvme_bdev_ctrlr);



	pthread_mutex_lock(&g_bdev_nvme_mutex);

module/bdev/nvme/common.h

+21 −14
Original line number Diff line number Diff line
@@ -69,6 +69,11 @@ struct nvme_bdev_ns { 


struct ocssd_bdev_ctrlr;



struct nvme_bdev_ctrlr_trid {

	struct spdk_nvme_transport_id		trid;

	TAILQ_ENTRY(nvme_bdev_ctrlr_trid)	link;

};



struct nvme_bdev_ctrlr {

	/**

	 * points to pinned, physically contiguous memory region;
@@ -100,6 +105,8 @@ struct nvme_bdev_ctrlr { 


	/** linked list pointer for device list */

	TAILQ_ENTRY(nvme_bdev_ctrlr)		tailq;



	TAILQ_HEAD(, nvme_bdev_ctrlr_trid)	multipath_trids;

};



struct nvme_bdev {

test/unit/lib/bdev/bdev_ocssd.c/bdev_ocssd_ut.c

+8 −4
Original line number Diff line number Diff line
@@ -198,6 +198,7 @@ create_nvme_bdev_controller(const struct spdk_nvme_transport_id *trid, const cha 
{

	struct spdk_nvme_ctrlr *ctrlr;

	struct nvme_bdev_ctrlr *nvme_bdev_ctrlr;

	struct nvme_bdev_ctrlr_trid *trid_entry;

	uint32_t nsid;



	ctrlr = find_controller(trid);
@@ -211,15 +212,15 @@ create_nvme_bdev_controller(const struct spdk_nvme_transport_id *trid, const cha 
	nvme_bdev_ctrlr->namespaces = calloc(ctrlr->ns_count, sizeof(struct nvme_bdev_ns *));

	SPDK_CU_ASSERT_FATAL(nvme_bdev_ctrlr->namespaces != NULL);



	nvme_bdev_ctrlr->trid = calloc(1, sizeof(struct spdk_nvme_transport_id));

	SPDK_CU_ASSERT_FATAL(nvme_bdev_ctrlr->trid != NULL);

	trid_entry = calloc(1, sizeof(struct nvme_bdev_ctrlr_trid));

	SPDK_CU_ASSERT_FATAL(trid_entry != NULL);

	trid_entry->trid = *trid;



	nvme_bdev_ctrlr->ctrlr = ctrlr;

	nvme_bdev_ctrlr->num_ns = ctrlr->ns_count;

	nvme_bdev_ctrlr->ref = 0;

	*nvme_bdev_ctrlr->trid = *trid;

	nvme_bdev_ctrlr->trid = &trid_entry->trid;

	nvme_bdev_ctrlr->name = strdup(name);



	for (nsid = 0; nsid < ctrlr->ns_count; ++nsid) {

		nvme_bdev_ctrlr->namespaces[nsid] = calloc(1, sizeof(struct nvme_bdev_ns));

		SPDK_CU_ASSERT_FATAL(nvme_bdev_ctrlr->namespaces[nsid] != NULL);
@@ -239,6 +240,9 @@ create_nvme_bdev_controller(const struct spdk_nvme_transport_id *trid, const cha 


	TAILQ_INSERT_TAIL(&g_nvme_bdev_ctrlrs, nvme_bdev_ctrlr, tailq);



	TAILQ_INIT(&nvme_bdev_ctrlr->multipath_trids);

	TAILQ_INSERT_HEAD(&nvme_bdev_ctrlr->multipath_trids, trid_entry, link);



	return nvme_bdev_ctrlr;

}