Commit 62ded32f authored by Seth Howell's avatar Seth Howell Committed by Tomasz Zawadzki
Browse files

bdev/nvme: add failover option to bdev_nvme_reset 



When we fail to process admin completions on a controller
attempt to failover to a previously registered trid

Signed-off-by: default avatarSeth Howell <seth.howell@intel.com>
Change-Id: Icdb43a726969fede4665f2fe59723dd90c5dad0d
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/2883


Community-CI: Mellanox Build Bot
Community-CI: Broadcom CI
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarAleksey Marchuk <alexeymar@mellanox.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
parent 5f13e0f6

module/bdev/nvme/bdev_nvme.c

+29 −7
Original line number Diff line number Diff line
@@ -162,7 +162,8 @@ static int bdev_nvme_io_passthru(struct nvme_bdev *nbdev, struct spdk_io_channel 
static int bdev_nvme_io_passthru_md(struct nvme_bdev *nbdev, struct spdk_io_channel *ch,

				    struct nvme_bdev_io *bio,

				    struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes, void *md_buf, size_t md_len);

static int bdev_nvme_reset(struct nvme_bdev_ctrlr *nvme_bdev_ctrlr, struct nvme_bdev_io *bio);

static int bdev_nvme_reset(struct nvme_bdev_ctrlr *nvme_bdev_ctrlr, struct nvme_bdev_io *bio,

			   bool failover);



typedef void (*populate_namespace_fn)(struct nvme_bdev_ctrlr *nvme_bdev_ctrlr,

				      struct nvme_bdev_ns *nvme_ns, struct nvme_async_probe_ctx *ctx);
@@ -272,7 +273,7 @@ bdev_nvme_poll_adminq(void *arg) 
	if (rc < 0) {

		nvme_bdev_ctrlr = nvme_bdev_ctrlr_get(spdk_nvme_ctrlr_get_transport_id(ctrlr));

		assert(nvme_bdev_ctrlr != NULL);

		bdev_nvme_reset(nvme_bdev_ctrlr, NULL);

		bdev_nvme_reset(nvme_bdev_ctrlr, NULL, true);

	}



	return rc;
@@ -447,10 +448,12 @@ _bdev_nvme_reset_destroy_qpair(struct spdk_io_channel_iter *i) 
}



static int

bdev_nvme_reset(struct nvme_bdev_ctrlr *nvme_bdev_ctrlr, struct nvme_bdev_io *bio)

bdev_nvme_reset(struct nvme_bdev_ctrlr *nvme_bdev_ctrlr, struct nvme_bdev_io *bio, bool failover)

{

	struct spdk_io_channel *ch;

	struct nvme_io_channel *nvme_ch;

	struct nvme_bdev_ctrlr_trid *multipath_trid, *tmp_trid;

	int rc;



	pthread_mutex_lock(&g_bdev_nvme_mutex);

	if (nvme_bdev_ctrlr->destruct) {
@@ -482,6 +485,25 @@ bdev_nvme_reset(struct nvme_bdev_ctrlr *nvme_bdev_ctrlr, struct nvme_bdev_io *bi 
		return 0;

	}



	if (failover) {

		tmp_trid = TAILQ_FIRST(&nvme_bdev_ctrlr->multipath_trids);

		assert(tmp_trid);

		assert(&tmp_trid->trid == nvme_bdev_ctrlr->trid);

		multipath_trid = TAILQ_NEXT(tmp_trid, link);



		if (multipath_trid) {

			spdk_nvme_ctrlr_fail(nvme_bdev_ctrlr->ctrlr);

			nvme_bdev_ctrlr->trid = &multipath_trid->trid;

			/** Shuffle the old trid to the end of the list and use the new one.

			 * Allows for round robin through multiple connections.

			 */

			rc = spdk_nvme_ctrlr_set_trid(nvme_bdev_ctrlr->ctrlr, &multipath_trid->trid);

			assert(rc == 0);

			TAILQ_REMOVE(&nvme_bdev_ctrlr->multipath_trids, tmp_trid, link);

			TAILQ_INSERT_TAIL(&nvme_bdev_ctrlr->multipath_trids, tmp_trid, link);

		}

	}



	pthread_mutex_unlock(&g_bdev_nvme_mutex);

	/* First, delete all NVMe I/O queue pairs. */

	spdk_for_each_channel(nvme_bdev_ctrlr,
@@ -592,7 +614,7 @@ _bdev_nvme_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_ 
				       bdev_io->u.bdev.num_blocks);



	case SPDK_BDEV_IO_TYPE_RESET:

		return bdev_nvme_reset(nbdev->nvme_bdev_ctrlr, nbdev_io);

		return bdev_nvme_reset(nbdev->nvme_bdev_ctrlr, nbdev_io, false);



	case SPDK_BDEV_IO_TYPE_FLUSH:

		return bdev_nvme_flush(nbdev,
@@ -1129,7 +1151,7 @@ nvme_abort_cpl(void *ctx, const struct spdk_nvme_cpl *cpl) 
		SPDK_WARNLOG("Abort failed. Resetting controller.\n");

		nvme_bdev_ctrlr = nvme_bdev_ctrlr_get(spdk_nvme_ctrlr_get_transport_id(ctrlr));

		assert(nvme_bdev_ctrlr != NULL);

		bdev_nvme_reset(nvme_bdev_ctrlr, NULL);

		bdev_nvme_reset(nvme_bdev_ctrlr, NULL, false);

	}

}
@@ -1148,7 +1170,7 @@ timeout_cb(void *cb_arg, struct spdk_nvme_ctrlr *ctrlr, 
		SPDK_ERRLOG("Controller Fatal Status, reset required\n");

		nvme_bdev_ctrlr = nvme_bdev_ctrlr_get(spdk_nvme_ctrlr_get_transport_id(ctrlr));

		assert(nvme_bdev_ctrlr != NULL);

		bdev_nvme_reset(nvme_bdev_ctrlr, NULL);

		bdev_nvme_reset(nvme_bdev_ctrlr, NULL, false);

		return;

	}
@@ -1168,7 +1190,7 @@ timeout_cb(void *cb_arg, struct spdk_nvme_ctrlr *ctrlr, 
	case SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET:

		nvme_bdev_ctrlr = nvme_bdev_ctrlr_get(spdk_nvme_ctrlr_get_transport_id(ctrlr));

		assert(nvme_bdev_ctrlr != NULL);

		bdev_nvme_reset(nvme_bdev_ctrlr, NULL);

		bdev_nvme_reset(nvme_bdev_ctrlr, NULL, false);

		break;

	case SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE:

		SPDK_DEBUGLOG(SPDK_LOG_BDEV_NVME, "No action for nvme controller timeout.\n");