Commit 2a6a6448 authored by Shuhei Matsumoto's avatar Shuhei Matsumoto Committed by Tomasz Zawadzki
Browse files

bdev/nvme: Add bdev_nvme_get_io_paths RPC to monitor I/O path states 



Add an new RPC bdev_nvme_get_io_paths to query all active I/O paths.

One io_path belongs to One nvme_bdev_channel.
Each nvme_bdev_channel is associated with one nvme_bdev.

If the RPC bdev_nvme_get_io_paths has a bdev name as a parameter
it can use spdk_for_each_channel() simply for the corresponding
nvme_bdev.

However, users will want to know I/O paths of all nvme_bdevs like
the RPC bdev_get_bdevs.

One io_path has one nvme_qpair. One nvme_qpair belongs to one
nvme_poll_group. By relying on these relationships, the RPC
bdev_nvme_get_io_paths traverses all nvme_poll_groups by using
spdk_for_each_channel() to g_bdev_nvme_ctrlrs.

The RPC bdev_nvme_get_io_paths has two modes, display all or
the specified NVMe bdev's active I/O paths.

The specified bdev name is used just for comparison and empty
array is returned if no matched io_path is found.

Signed-off-by: default avatarShuhei Matsumoto <smatsumoto@nvidia.com>
Change-Id: I4a0dbf3ef7aaa9a7b7345fc03dc493cc6d37bc99
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/12146


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Reviewed-by: default avatarAleksey Marchuk <alexeymar@mellanox.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
parent 2730f5ca

CHANGELOG.md

+2 −0
Original line number Diff line number Diff line
@@ -21,6 +21,8 @@ A new API `spdk_bdev_unregister_by_name` was added to handle race conditions cor 
New APIs, `spdk_for_each_bdev` and `spdk_for_each_bdev_leaf`, were added to provide iteration

safe for race conditions.



A new RPC `bdev_nvme_get_io_paths` was added to get all active I/O paths.



### idxd



A new parameter `flags` was added to all low level submission and preparation

doc/jsonrpc.md

+50 −0
Original line number Diff line number Diff line
@@ -3344,6 +3344,56 @@ Example response: 
}

~~~



### bdev_nvme_get_io_paths {#rpc_bdev_nvme_get_io_paths}



Display all or the specified NVMe bdev's active I/O paths.



#### Parameters



Name                    | Optional | Type        | Description

----------------------- | -------- | ----------- | -----------

name                    | Optional | string      | Name of the NVMe bdev



#### Example



Example request:



~~~json

{

  "jsonrpc": "2.0",

  "method": "bdev_nvme_get_io_paths",

  "id": 1,

  "params": {

    "name": "Nvme0n1"

  }

}

~~~



Example response:



~~~json

{

  "jsonrpc": "2.0",

  "id": 1,

  "result": {

    "poll_groups": [

      {

        "thread": "app_thread",

        "io_paths": [

          {

            "bdev_name": "Nvme0n1",

            "cntlid": 0,

            "current": true,

            "connected": true,

            "accessible": true

          }

        ]

      }

    ]

  }

}

~~~



### bdev_nvme_cuse_register {#rpc_bdev_nvme_cuse_register}



Register CUSE device on NVMe controller.

module/bdev/nvme/bdev_nvme.c

+24 −0
Original line number Diff line number Diff line
@@ -6160,4 +6160,28 @@ bdev_nvme_get_ctrlr(struct spdk_bdev *bdev) 
	return nvme_ns->ctrlr->ctrlr;

}



void

nvme_io_path_info_json(struct spdk_json_write_ctx *w, struct nvme_io_path *io_path)

{

	struct nvme_ns *nvme_ns = io_path->nvme_ns;

	struct nvme_ctrlr *nvme_ctrlr = io_path->qpair->ctrlr;

	const struct spdk_nvme_ctrlr_data *cdata;



	spdk_json_write_object_begin(w);



	spdk_json_write_named_string(w, "bdev_name", nvme_ns->bdev->disk.name);



	cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr);



	spdk_json_write_named_uint32(w, "cntlid", cdata->cntlid);



	spdk_json_write_named_bool(w, "current", io_path == io_path->nbdev_ch->current_io_path);



	spdk_json_write_named_bool(w, "connected", nvme_io_path_is_connected(io_path));



	spdk_json_write_named_bool(w, "accessible", nvme_ns_is_accessible(nvme_ns));



	spdk_json_write_object_end(w);

}



SPDK_LOG_REGISTER_COMPONENT(bdev_nvme)

module/bdev/nvme/bdev_nvme.h

+2 −0
Original line number Diff line number Diff line
@@ -223,6 +223,8 @@ struct nvme_poll_group { 
	TAILQ_HEAD(, nvme_qpair)		qpair_list;

};



void nvme_io_path_info_json(struct spdk_json_write_ctx *w, struct nvme_io_path *io_path);



struct nvme_ctrlr *nvme_ctrlr_get_by_name(const char *name);



struct nvme_bdev_ctrlr *nvme_bdev_ctrlr_get_by_name(const char *name);

module/bdev/nvme/bdev_nvme_rpc.c

+109 −0
Original line number Diff line number Diff line
@@ -2017,3 +2017,112 @@ cleanup: 
}

SPDK_RPC_REGISTER("bdev_nvme_remove_error_injection", rpc_bdev_nvme_remove_error_injection,

		  SPDK_RPC_RUNTIME)



struct rpc_get_io_paths {

	char *name;

};



static void

free_rpc_get_io_paths(struct rpc_get_io_paths *r)

{

	free(r->name);

}



static const struct spdk_json_object_decoder rpc_get_io_paths_decoders[] = {

	{"name", offsetof(struct rpc_get_io_paths, name), spdk_json_decode_string, true},

};



struct rpc_get_io_paths_ctx {

	struct rpc_get_io_paths req;

	struct spdk_jsonrpc_request *request;

	struct spdk_json_write_ctx *w;

};



static void

rpc_bdev_nvme_get_io_paths_done(struct spdk_io_channel_iter *i, int status)

{

	struct rpc_get_io_paths_ctx *ctx = spdk_io_channel_iter_get_ctx(i);



	spdk_json_write_array_end(ctx->w);



	spdk_json_write_object_end(ctx->w);



	spdk_jsonrpc_end_result(ctx->request, ctx->w);



	free_rpc_get_io_paths(&ctx->req);

	free(ctx);

}



static void

_rpc_bdev_nvme_get_io_paths(struct spdk_io_channel_iter *i)

{

	struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i);

	struct nvme_poll_group *group = spdk_io_channel_get_ctx(_ch);

	struct rpc_get_io_paths_ctx *ctx = spdk_io_channel_iter_get_ctx(i);

	struct nvme_qpair *qpair;

	struct nvme_io_path *io_path;

	struct nvme_bdev *nbdev;



	spdk_json_write_object_begin(ctx->w);



	spdk_json_write_named_string(ctx->w, "thread", spdk_thread_get_name(spdk_get_thread()));



	spdk_json_write_named_array_begin(ctx->w, "io_paths");



	TAILQ_FOREACH(qpair, &group->qpair_list, tailq) {

		TAILQ_FOREACH(io_path, &qpair->io_path_list, tailq) {

			nbdev = io_path->nvme_ns->bdev;



			if (ctx->req.name != NULL &&

			    strcmp(ctx->req.name, nbdev->disk.name) != 0) {

				continue;

			}



			nvme_io_path_info_json(ctx->w, io_path);

		}

	}



	spdk_json_write_array_end(ctx->w);



	spdk_json_write_object_end(ctx->w);



	spdk_for_each_channel_continue(i, 0);

}



static void

rpc_bdev_nvme_get_io_paths(struct spdk_jsonrpc_request *request,

			   const struct spdk_json_val *params)

{

	struct rpc_get_io_paths_ctx *ctx;



	ctx = calloc(1, sizeof(*ctx));

	if (ctx == NULL) {

		spdk_jsonrpc_send_error_response(request, -ENOMEM, spdk_strerror(ENOMEM));

		return;

	}



	if (params != NULL &&

	    spdk_json_decode_object(params, rpc_get_io_paths_decoders,

				    SPDK_COUNTOF(rpc_get_io_paths_decoders),

				    &ctx->req)) {

		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,

						 "bdev_nvme_get_io_paths requires no parameters");



		free_rpc_get_io_paths(&ctx->req);

		free(ctx);

		return;

	}



	ctx->request = request;

	ctx->w = spdk_jsonrpc_begin_result(request);



	spdk_json_write_object_begin(ctx->w);



	spdk_json_write_named_array_begin(ctx->w, "poll_groups");



	spdk_for_each_channel(&g_nvme_bdev_ctrlrs,

			      _rpc_bdev_nvme_get_io_paths,

			      ctx,

			      rpc_bdev_nvme_get_io_paths_done);

}

SPDK_RPC_REGISTER("bdev_nvme_get_io_paths", rpc_bdev_nvme_get_io_paths, SPDK_RPC_RUNTIME)