Commit 00a79982 authored by Shuhei Matsumoto's avatar Shuhei Matsumoto Committed by Tomasz Zawadzki
Browse files

bdev/nvme: Move per controller settings into a option structure 



The following patches will enable us to specify I/O error resiliency
options per nvme_ctrlr as global options. To do it easier, move
per controller options about I/O error resiliency into struct nvme_ctrlr_opts.

prchk_flags is not exactly for resiliency but move it into struct
nvme_ctrlr_opts too.

Signed-off-by: default avatarShuhei Matsumoto <smatsumoto@nvidia.com>
Change-Id: I85fd1738bb6e293cd804b086ade82274485f213d
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/11829


Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
parent df32eb11

module/bdev/nvme/bdev_nvme.c

+28 −31
Original line number Diff line number Diff line
@@ -747,7 +747,7 @@ nvme_io_path_is_failed(struct nvme_io_path *io_path) 
	}



	if (nvme_ctrlr->resetting) {

		if (nvme_ctrlr->reconnect_delay_sec != 0) {

		if (nvme_ctrlr->opts.reconnect_delay_sec != 0) {

			return false;

		} else {

			return true;
@@ -1319,13 +1319,13 @@ bdev_nvme_check_ctrlr_loss_timeout(struct nvme_ctrlr *nvme_ctrlr) 
{

	int32_t elapsed;



	if (nvme_ctrlr->ctrlr_loss_timeout_sec == 0 ||

	    nvme_ctrlr->ctrlr_loss_timeout_sec == -1) {

	if (nvme_ctrlr->opts.ctrlr_loss_timeout_sec == 0 ||

	    nvme_ctrlr->opts.ctrlr_loss_timeout_sec == -1) {

		return false;

	}



	elapsed = (spdk_get_ticks() - nvme_ctrlr->reset_start_tsc) / spdk_get_ticks_hz();

	if (elapsed >= nvme_ctrlr->ctrlr_loss_timeout_sec) {

	if (elapsed >= nvme_ctrlr->opts.ctrlr_loss_timeout_sec) {

		return true;

	} else {

		return false;
@@ -1337,12 +1337,12 @@ bdev_nvme_check_fast_io_fail_timeout(struct nvme_ctrlr *nvme_ctrlr) 
{

	uint32_t elapsed;



	if (nvme_ctrlr->fast_io_fail_timeout_sec == 0) {

	if (nvme_ctrlr->opts.fast_io_fail_timeout_sec == 0) {

		return false;

	}



	elapsed = (spdk_get_ticks() - nvme_ctrlr->reset_start_tsc) / spdk_get_ticks_hz();

	if (elapsed >= nvme_ctrlr->fast_io_fail_timeout_sec) {

	if (elapsed >= nvme_ctrlr->opts.fast_io_fail_timeout_sec) {

		return true;

	} else {

		return false;
@@ -1364,7 +1364,7 @@ bdev_nvme_check_op_after_reset(struct nvme_ctrlr *nvme_ctrlr, bool success) 
	if (nvme_ctrlr_can_be_unregistered(nvme_ctrlr)) {

		/* Complete pending destruct after reset completes. */

		return OP_COMPLETE_PENDING_DESTRUCT;

	} else if (success || nvme_ctrlr->reconnect_delay_sec == 0) {

	} else if (success || nvme_ctrlr->opts.reconnect_delay_sec == 0) {

		nvme_ctrlr->reset_start_tsc = 0;

		return OP_NONE;

	} else if (bdev_nvme_check_ctrlr_loss_timeout(nvme_ctrlr)) {
@@ -1420,7 +1420,7 @@ bdev_nvme_start_reconnect_delay_timer(struct nvme_ctrlr *nvme_ctrlr) 
	assert(nvme_ctrlr->reconnect_delay_timer == NULL);

	nvme_ctrlr->reconnect_delay_timer = SPDK_POLLER_REGISTER(bdev_nvme_reconnect_delay_timer_expired,

					    nvme_ctrlr,

					    nvme_ctrlr->reconnect_delay_sec * SPDK_SEC_TO_USEC);

					    nvme_ctrlr->opts.reconnect_delay_sec * SPDK_SEC_TO_USEC);

}



static void
@@ -2582,7 +2582,7 @@ nvme_bdev_create(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns) 
	bdev->opal = nvme_ctrlr->opal_dev != NULL;



	rc = nvme_disk_create(&bdev->disk, nvme_ctrlr->nbdev_ctrlr->name, nvme_ctrlr->ctrlr,

			      nvme_ns->ns, nvme_ctrlr->prchk_flags, bdev);

			      nvme_ns->ns, nvme_ctrlr->opts.prchk_flags, bdev);

	if (rc != 0) {

		SPDK_ERRLOG("Failed to create NVMe disk\n");

		pthread_mutex_destroy(&bdev->mutex);
@@ -3429,10 +3429,7 @@ nvme_ctrlr_create(struct spdk_nvme_ctrlr *ctrlr, 
	}



	if (ctx != NULL) {

		nvme_ctrlr->prchk_flags = ctx->prchk_flags;

		nvme_ctrlr->ctrlr_loss_timeout_sec = ctx->ctrlr_loss_timeout_sec;

		nvme_ctrlr->reconnect_delay_sec = ctx->reconnect_delay_sec;

		nvme_ctrlr->fast_io_fail_timeout_sec = ctx->fast_io_fail_timeout_sec;

		memcpy(&nvme_ctrlr->opts, &ctx->bdev_opts, sizeof(ctx->bdev_opts));

	}



	nvme_ctrlr->adminq_timer_poller = SPDK_POLLER_REGISTER(bdev_nvme_poll_adminq, nvme_ctrlr,
@@ -3944,14 +3941,11 @@ bdev_nvme_create(struct spdk_nvme_transport_id *trid, 
		 const char *base_name,

		 const char **names,

		 uint32_t count,

		 uint32_t prchk_flags,

		 spdk_bdev_create_nvme_fn cb_fn,

		 void *cb_ctx,

		 struct spdk_nvme_ctrlr_opts *drv_opts,

		 bool multipath,

		 int32_t ctrlr_loss_timeout_sec,

		 uint32_t reconnect_delay_sec,

		 uint32_t fast_io_fail_timeout_sec)

		 struct nvme_ctrlr_opts *bdev_opts,

		 bool multipath)

{

	struct nvme_probe_skip_entry	*entry, *tmp;

	struct nvme_async_probe_ctx	*ctx;
@@ -3965,8 +3959,10 @@ bdev_nvme_create(struct spdk_nvme_transport_id *trid, 
		return -EEXIST;

	}



	if (!bdev_nvme_check_multipath_params(ctrlr_loss_timeout_sec, reconnect_delay_sec,

					      fast_io_fail_timeout_sec)) {

	if (bdev_opts != NULL &&

	    !bdev_nvme_check_multipath_params(bdev_opts->ctrlr_loss_timeout_sec,

					      bdev_opts->reconnect_delay_sec,

					      bdev_opts->fast_io_fail_timeout_sec)) {

		return -EINVAL;

	}
@@ -3979,11 +3975,11 @@ bdev_nvme_create(struct spdk_nvme_transport_id *trid, 
	ctx->count = count;

	ctx->cb_fn = cb_fn;

	ctx->cb_ctx = cb_ctx;

	ctx->prchk_flags = prchk_flags;

	ctx->trid = *trid;

	ctx->ctrlr_loss_timeout_sec = ctrlr_loss_timeout_sec;

	ctx->reconnect_delay_sec = reconnect_delay_sec;

	ctx->fast_io_fail_timeout_sec = fast_io_fail_timeout_sec;



	if (bdev_opts) {

		memcpy(&ctx->bdev_opts, bdev_opts, sizeof(*bdev_opts));

	}



	if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) {

		TAILQ_FOREACH_SAFE(entry, &g_skipped_nvme_ctrlrs, tailq, tmp) {
@@ -4365,9 +4361,9 @@ discovery_log_page_cb(void *cb_arg, int rc, const struct spdk_nvme_cpl *cpl, 
			}

			spdk_nvme_ctrlr_get_default_ctrlr_opts(&new_ctx->drv_opts, sizeof(new_ctx->drv_opts));

			snprintf(new_ctx->drv_opts.hostnqn, sizeof(new_ctx->drv_opts.hostnqn), "%s", ctx->hostnqn);

			rc = bdev_nvme_create(&new_ctx->trid, new_ctx->name, NULL, 0, 0,

			rc = bdev_nvme_create(&new_ctx->trid, new_ctx->name, NULL, 0,

					      discovery_attach_controller_done, new_ctx,

					      &new_ctx->drv_opts, true, 0, 0, 0);

					      &new_ctx->drv_opts, NULL, true);

			if (rc == 0) {

				TAILQ_INSERT_TAIL(&ctx->nvm_entry_ctxs, new_ctx, tailq);

				ctx->attach_in_progress++;
@@ -5703,12 +5699,13 @@ nvme_ctrlr_config_json(struct spdk_json_write_ctx *w, 
	spdk_json_write_named_string(w, "name", nvme_ctrlr->nbdev_ctrlr->name);

	nvme_bdev_dump_trid_json(trid, w);

	spdk_json_write_named_bool(w, "prchk_reftag",

				   (nvme_ctrlr->prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_REFTAG) != 0);

				   (nvme_ctrlr->opts.prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_REFTAG) != 0);

	spdk_json_write_named_bool(w, "prchk_guard",

				   (nvme_ctrlr->prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_GUARD) != 0);

	spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", nvme_ctrlr->ctrlr_loss_timeout_sec);

	spdk_json_write_named_uint32(w, "reconnect_delay_sec", nvme_ctrlr->reconnect_delay_sec);

	spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec", nvme_ctrlr->fast_io_fail_timeout_sec);

				   (nvme_ctrlr->opts.prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_GUARD) != 0);

	spdk_json_write_named_int32(w, "ctrlr_loss_timeout_sec", nvme_ctrlr->opts.ctrlr_loss_timeout_sec);

	spdk_json_write_named_uint32(w, "reconnect_delay_sec", nvme_ctrlr->opts.reconnect_delay_sec);

	spdk_json_write_named_uint32(w, "fast_io_fail_timeout_sec",

				     nvme_ctrlr->opts.fast_io_fail_timeout_sec);



	spdk_json_write_object_end(w);

module/bdev/nvme/bdev_nvme.h

+13 −19
Original line number Diff line number Diff line
@@ -52,17 +52,21 @@ typedef void (*spdk_bdev_create_nvme_fn)(void *ctx, size_t bdev_count, int rc); 
typedef void (*spdk_bdev_nvme_start_discovery_fn)(void *ctx, int rc);

typedef void (*spdk_bdev_nvme_stop_discovery_fn)(void *ctx);



struct nvme_ctrlr_opts {

	uint32_t prchk_flags;

	int32_t ctrlr_loss_timeout_sec;

	uint32_t reconnect_delay_sec;

	uint32_t fast_io_fail_timeout_sec;

};



struct nvme_async_probe_ctx {

	struct spdk_nvme_probe_ctx *probe_ctx;

	const char *base_name;

	const char **names;

	uint32_t count;

	uint32_t prchk_flags;

	int32_t ctrlr_loss_timeout_sec;

	uint32_t reconnect_delay_sec;

	uint32_t fast_io_fail_timeout_sec;

	struct spdk_poller *poller;

	struct spdk_nvme_transport_id trid;

	struct nvme_ctrlr_opts bdev_opts;

	struct spdk_nvme_ctrlr_opts drv_opts;

	spdk_bdev_create_nvme_fn cb_fn;

	void *cb_ctx;
@@ -114,12 +118,9 @@ struct nvme_ctrlr { 
	uint32_t				fast_io_fail_timedout : 1;

	uint32_t				destruct : 1;

	uint32_t				ana_log_page_updating : 1;

	/**

	 * PI check flags. This flags is set to NVMe controllers created only

	 * through bdev_nvme_attach_controller RPC or .INI config file. Hot added

	 * NVMe controllers are not included.

	 */

	uint32_t				prchk_flags;



	struct nvme_ctrlr_opts			opts;



	RB_HEAD(nvme_ns_tree, nvme_ns)		namespaces;



	struct spdk_opal_dev			*opal_dev;
@@ -148,10 +149,6 @@ struct nvme_ctrlr { 


	struct nvme_async_probe_ctx		*probe_ctx;



	uint32_t				reconnect_delay_sec;

	int32_t					ctrlr_loss_timeout_sec;

	uint32_t				fast_io_fail_timeout_sec;



	pthread_mutex_t				mutex;

};
@@ -265,14 +262,11 @@ int bdev_nvme_create(struct spdk_nvme_transport_id *trid, 
		     const char *base_name,

		     const char **names,

		     uint32_t count,

		     uint32_t prchk_flags,

		     spdk_bdev_create_nvme_fn cb_fn,

		     void *cb_ctx,

		     struct spdk_nvme_ctrlr_opts *drv_opts,

		     bool multipath,

		     int32_t ctrlr_loss_timeout_sec,

		     uint32_t reconnect_delay_sec,

		     uint32_t fast_io_fail_timeout_sec);

		     struct nvme_ctrlr_opts *bdev_opts,

		     bool multipath);



int bdev_nvme_start_discovery(struct spdk_nvme_transport_id *trid, const char *base_name,

			      struct spdk_nvme_ctrlr_opts *drv_opts,

module/bdev/nvme/bdev_nvme_rpc.c

+9 −13
Original line number Diff line number Diff line
@@ -181,11 +181,8 @@ struct rpc_bdev_nvme_attach_controller { 
	char *hostnqn;

	char *hostaddr;

	char *hostsvcid;

	uint32_t prchk_flags;

	char *multipath;

	int32_t ctrlr_loss_timeout_sec;

	uint32_t reconnect_delay_sec;

	uint32_t fast_io_fail_timeout_sec;

	struct nvme_ctrlr_opts bdev_opts;

	struct spdk_nvme_ctrlr_opts drv_opts;

};
@@ -248,16 +245,16 @@ static const struct spdk_json_object_decoder rpc_bdev_nvme_attach_controller_dec 
	{"hostaddr", offsetof(struct rpc_bdev_nvme_attach_controller, hostaddr), spdk_json_decode_string, true},

	{"hostsvcid", offsetof(struct rpc_bdev_nvme_attach_controller, hostsvcid), spdk_json_decode_string, true},



	{"prchk_reftag", offsetof(struct rpc_bdev_nvme_attach_controller, prchk_flags), bdev_nvme_decode_reftag, true},

	{"prchk_guard", offsetof(struct rpc_bdev_nvme_attach_controller, prchk_flags), bdev_nvme_decode_guard, true},

	{"prchk_reftag", offsetof(struct rpc_bdev_nvme_attach_controller, bdev_opts.prchk_flags), bdev_nvme_decode_reftag, true},

	{"prchk_guard", offsetof(struct rpc_bdev_nvme_attach_controller, bdev_opts.prchk_flags), bdev_nvme_decode_guard, true},

	{"hdgst", offsetof(struct rpc_bdev_nvme_attach_controller, drv_opts.header_digest), spdk_json_decode_bool, true},

	{"ddgst", offsetof(struct rpc_bdev_nvme_attach_controller, drv_opts.data_digest), spdk_json_decode_bool, true},

	{"fabrics_connect_timeout_us", offsetof(struct rpc_bdev_nvme_attach_controller, drv_opts.fabrics_connect_timeout_us), spdk_json_decode_uint64, true},

	{"multipath", offsetof(struct rpc_bdev_nvme_attach_controller, multipath), spdk_json_decode_string, true},

	{"num_io_queues", offsetof(struct rpc_bdev_nvme_attach_controller, drv_opts.num_io_queues), spdk_json_decode_uint32, true},

	{"ctrlr_loss_timeout_sec", offsetof(struct rpc_bdev_nvme_attach_controller, ctrlr_loss_timeout_sec), spdk_json_decode_int32, true},

	{"reconnect_delay_sec", offsetof(struct rpc_bdev_nvme_attach_controller, reconnect_delay_sec), spdk_json_decode_uint32, true},

	{"fast_io_fail_timeout_sec", offsetof(struct rpc_bdev_nvme_attach_controller, fast_io_fail_timeout_sec), spdk_json_decode_uint32, true},

	{"ctrlr_loss_timeout_sec", offsetof(struct rpc_bdev_nvme_attach_controller, bdev_opts.ctrlr_loss_timeout_sec), spdk_json_decode_int32, true},

	{"reconnect_delay_sec", offsetof(struct rpc_bdev_nvme_attach_controller, bdev_opts.reconnect_delay_sec), spdk_json_decode_uint32, true},

	{"fast_io_fail_timeout_sec", offsetof(struct rpc_bdev_nvme_attach_controller, bdev_opts.fast_io_fail_timeout_sec), spdk_json_decode_uint32, true},

};



#define NVME_MAX_BDEVS_PER_RPC 128
@@ -495,7 +492,7 @@ rpc_bdev_nvme_attach_controller(struct spdk_jsonrpc_request *request, 
			goto cleanup;

		}



		if (ctx->req.prchk_flags) {

		if (ctx->req.bdev_opts.prchk_flags) {

			spdk_jsonrpc_send_error_response_fmt(request, -EINVAL,

							     "A controller named %s already exists. To add a path, do not specify PI options.\n",

							     ctx->req.name);
@@ -516,10 +513,9 @@ rpc_bdev_nvme_attach_controller(struct spdk_jsonrpc_request *request, 


	ctx->request = request;

	ctx->count = NVME_MAX_BDEVS_PER_RPC;

	rc = bdev_nvme_create(&trid, ctx->req.name, ctx->names, ctx->count, ctx->req.prchk_flags,

	rc = bdev_nvme_create(&trid, ctx->req.name, ctx->names, ctx->count,

			      rpc_bdev_nvme_attach_controller_done, ctx, &ctx->req.drv_opts,

			      multipath, ctx->req.ctrlr_loss_timeout_sec,

			      ctx->req.reconnect_delay_sec, ctx->req.fast_io_fail_timeout_sec);

			      &ctx->req.bdev_opts, multipath);

	if (rc) {

		spdk_jsonrpc_send_error_response(request, rc, spdk_strerror(-rc));

		goto cleanup;

test/unit/lib/bdev/nvme/bdev_nvme.c/bdev_nvme_ut.c

+113 −92

File changed.

Preview size limit exceeded, changes collapsed.