Commit b317d8f3 authored by Richael Zhuang's avatar Richael Zhuang Committed by Tomasz Zawadzki
Browse files

bdev_nvme: add support to display io path stat of the NVMe bdev 



Add RPC bdev_nvme_get_path_iostat to show I/O statistics for IO paths
of the NVMe bdev.

Change-Id: I22e5ad112d5cfa6d96cf246dcd0e511ae71dc839
Signed-off-by: default avatarRichael Zhuang <richael.zhuang@arm.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/14745


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <smatsumoto@nvidia.com>
parent 57cf8961

CHANGELOG.md

+2 −0
Original line number Diff line number Diff line
@@ -69,6 +69,8 @@ were added to process I/O statistics outside the generic bdev layer, especially 
Added I/O statistics per I/O path to the NVMe bdev module for NVMe bdev multipath. It can be

enabled by a new option io_path_stat of RPC bdev_nvme_set_options.



Added RPC bdev_nvme_get_path_iostat to get I/O statistics for IO paths of the NVMe bdev.



### event



Added core lock file mechanism to prevent the same CPU cores from being used by multiple

doc/jsonrpc.md

+96 −0
Original line number Diff line number Diff line
@@ -4211,6 +4211,102 @@ Example response: 
}

~~~



### bdev_nvme_get_path_iostat {#rpc_bdev_nvme_get_path_iostat}



Get I/O statistics for IO paths of the block device. Call RPC bdev_nvme_set_options to set enable_io_path_stat

true before using this RPC.



#### Parameters



Name                    | Optional | Type        | Description

----------------------- | -------- | ----------- | -----------

name                    | Required | string      | Name of the NVMe bdev



#### Example



Example request:



~~~json

{

  "jsonrpc": "2.0",

  "method": "bdev_nvme_get_path_iostat",

  "id": 1,

  "params": {

    "name": "NVMe0n1"

  }

}

~~~



Example response:



~~~json

{

  "jsonrpc": "2.0",

  "id": 1,

  "result": {

    "name": "NVMe0n1",

    "stats": [

      {

        "trid": {

          "trtype": "TCP",

          "adrfam": "IPv4",

          "traddr": "10.169.204.201",

          "trsvcid": "4420",

          "subnqn": "nqn.2016-06.io.spdk:cnode1"

        },

        "stat": {

          "bytes_read": 676691968,

          "num_read_ops": 165201,

          "bytes_written": 0,

          "num_write_ops": 0,

          "bytes_unmapped": 0,

          "num_unmap_ops": 0,

          "max_read_latency_ticks": 521487,

          "min_read_latency_ticks": 0,

          "write_latency_ticks": 0,

          "max_write_latency_ticks": 0,

          "min_write_latency_ticks": 0,

          "unmap_latency_ticks": 0,

          "max_unmap_latency_ticks": 0,

          "min_unmap_latency_ticks": 0,

          "copy_latency_ticks": 0,

          "max_copy_latency_ticks": 0,

          "min_copy_latency_ticks": 0

        }

      },

      {

        "trid": {

          "trtype": "TCP",

          "adrfam": "IPv4",

          "traddr": "8.8.8.6",

          "trsvcid": "4420",

          "subnqn": "nqn.2016-06.io.spdk:cnode1"

        },

        "stat": {

          "bytes_read": 677138432,

          "num_read_ops": 165317,

          "bytes_written": 0,

          "num_write_ops": 0,

          "bytes_unmapped": 0,

          "num_unmap_ops": 0,

          "max_read_latency_ticks": 108525,

          "min_read_latency_ticks": 0,

          "write_latency_ticks": 0,

          "max_write_latency_ticks": 0,

          "min_write_latency_ticks": 0,

          "unmap_latency_ticks": 0,

          "max_unmap_latency_ticks": 0,

          "min_unmap_latency_ticks": 0,

          "copy_latency_ticks": 0,

          "max_copy_latency_ticks": 0,

          "min_copy_latency_ticks": 0

        }

      }

    ]

  }

}

~~~



### bdev_nvme_cuse_register {#rpc_bdev_nvme_cuse_register}



Register CUSE device on NVMe controller.

module/bdev/nvme/bdev_nvme_rpc.c

+198 −0
Original line number Diff line number Diff line
@@ -2456,3 +2456,201 @@ rpc_bdev_nvme_get_mdns_discovery_info(struct spdk_jsonrpc_request *request, 


SPDK_RPC_REGISTER("bdev_nvme_get_mdns_discovery_info", rpc_bdev_nvme_get_mdns_discovery_info,

		  SPDK_RPC_RUNTIME)



struct rpc_get_path_stat {

	char	*name;

};



struct path_stat {

	struct spdk_bdev_io_stat	stat;

	struct spdk_nvme_transport_id	trid;

	struct nvme_ns			*ns;

};



struct rpc_bdev_nvme_path_stat_ctx {

	struct spdk_jsonrpc_request	*request;

	struct path_stat		*path_stat;

	uint32_t			num_paths;

	struct spdk_bdev_desc		*desc;

};



static void

free_rpc_get_path_stat(struct rpc_get_path_stat *req)

{

	free(req->name);

}



static const struct spdk_json_object_decoder rpc_get_path_stat_decoders[] = {

	{"name", offsetof(struct rpc_get_path_stat, name), spdk_json_decode_string},

};



static void

dummy_bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *ctx)

{

}



static void

rpc_bdev_nvme_path_stat_per_channel(struct spdk_io_channel_iter *i)

{

	struct rpc_bdev_nvme_path_stat_ctx *ctx = spdk_io_channel_iter_get_ctx(i);

	struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);

	struct nvme_bdev_channel *nbdev_ch = spdk_io_channel_get_ctx(ch);

	struct nvme_io_path *io_path;

	struct path_stat *path_stat;

	uint32_t j;



	assert(ctx->num_paths != 0);



	for (j = 0; j < ctx->num_paths; j++) {

		path_stat = &ctx->path_stat[j];



		STAILQ_FOREACH(io_path, &nbdev_ch->io_path_list, stailq) {

			if (path_stat->ns == io_path->nvme_ns) {

				assert(io_path->stat != NULL);

				spdk_bdev_add_io_stat(&path_stat->stat, io_path->stat);

			}

		}

	}



	spdk_for_each_channel_continue(i, 0);

}



static void

rpc_bdev_nvme_path_stat_done(struct spdk_io_channel_iter *i, int status)

{

	struct rpc_bdev_nvme_path_stat_ctx *ctx = spdk_io_channel_iter_get_ctx(i);

	struct nvme_bdev *nbdev = spdk_io_channel_iter_get_io_device(i);

	struct spdk_json_write_ctx *w;

	struct path_stat *path_stat;

	uint32_t j;



	assert(ctx->num_paths != 0);



	w = spdk_jsonrpc_begin_result(ctx->request);

	spdk_json_write_object_begin(w);

	spdk_json_write_named_string(w, "name", nbdev->disk.name);

	spdk_json_write_named_array_begin(w, "stats");



	for (j = 0; j < ctx->num_paths; j++) {

		path_stat = &ctx->path_stat[j];

		spdk_json_write_object_begin(w);



		spdk_json_write_named_object_begin(w, "trid");

		nvme_bdev_dump_trid_json(&path_stat->trid, w);

		spdk_json_write_object_end(w);



		spdk_json_write_named_object_begin(w, "stat");

		spdk_bdev_dump_io_stat_json(&path_stat->stat, w);

		spdk_json_write_object_end(w);



		spdk_json_write_object_end(w);

	}



	spdk_json_write_array_end(w);

	spdk_json_write_object_end(w);

	spdk_jsonrpc_end_result(ctx->request, w);



	spdk_bdev_close(ctx->desc);

	free(ctx->path_stat);

	free(ctx);

}



static void

rpc_bdev_nvme_get_path_iostat(struct spdk_jsonrpc_request *request,

			      const struct spdk_json_val *params)

{

	struct rpc_get_path_stat req = {};

	struct spdk_bdev_desc *desc = NULL;

	struct spdk_bdev *bdev;

	struct nvme_bdev *nbdev;

	struct nvme_ns *nvme_ns;

	struct path_stat *path_stat;

	struct rpc_bdev_nvme_path_stat_ctx *ctx;

	struct spdk_bdev_nvme_opts opts;

	uint32_t num_paths = 0, i = 0;

	int rc;



	bdev_nvme_get_opts(&opts);

	if (!opts.io_path_stat) {

		SPDK_ERRLOG("RPC not enabled if enable_io_path_stat is false\n");

		spdk_jsonrpc_send_error_response(request, -EPERM,

						 "RPC not enabled if enable_io_path_stat is false");

		return;

	}



	if (spdk_json_decode_object(params, rpc_get_path_stat_decoders,

				    SPDK_COUNTOF(rpc_get_path_stat_decoders),

				    &req)) {

		SPDK_ERRLOG("spdk_json_decode_object failed\n");

		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR,

						 "spdk_json_decode_object failed");

		free_rpc_get_path_stat(&req);

		return;

	}



	rc = spdk_bdev_open_ext(req.name, false, dummy_bdev_event_cb, NULL, &desc);

	if (rc != 0) {

		SPDK_ERRLOG("Failed to open bdev '%s': %d\n", req.name, rc);

		spdk_jsonrpc_send_error_response(request, rc, spdk_strerror(-rc));

		free_rpc_get_path_stat(&req);

		return;

	}



	free_rpc_get_path_stat(&req);



	ctx = calloc(1, sizeof(struct rpc_bdev_nvme_path_stat_ctx));

	if (ctx == NULL) {

		spdk_bdev_close(desc);

		SPDK_ERRLOG("Failed to allocate rpc_bdev_nvme_path_stat_ctx struct\n");

		spdk_jsonrpc_send_error_response(request, -ENOMEM, spdk_strerror(ENOMEM));

		return;

	}



	bdev = spdk_bdev_desc_get_bdev(desc);

	nbdev = bdev->ctxt;



	pthread_mutex_lock(&nbdev->mutex);

	if (nbdev->ref == 0) {

		rc = -ENOENT;

		goto err;

	}



	num_paths = nbdev->ref;

	path_stat = calloc(num_paths, sizeof(struct path_stat));

	if (path_stat == NULL) {

		rc = -ENOMEM;

		SPDK_ERRLOG("Failed to allocate memory for path_stat.\n");

		goto err;

	}



	/* store the history stat */

	TAILQ_FOREACH(nvme_ns, &nbdev->nvme_ns_list, tailq) {

		assert(i < num_paths);

		path_stat[i].ns = nvme_ns;

		path_stat[i].trid = nvme_ns->ctrlr->active_path_id->trid;



		assert(nvme_ns->stat != NULL);

		memcpy(&path_stat[i].stat, nvme_ns->stat, sizeof(struct spdk_bdev_io_stat));

		i++;

	}

	pthread_mutex_unlock(&nbdev->mutex);



	ctx->request = request;

	ctx->desc = desc;

	ctx->path_stat = path_stat;

	ctx->num_paths = num_paths;



	spdk_for_each_channel(nbdev,

			      rpc_bdev_nvme_path_stat_per_channel,

			      ctx,

			      rpc_bdev_nvme_path_stat_done);

	return;



err:

	pthread_mutex_unlock(&nbdev->mutex);

	spdk_jsonrpc_send_error_response(request, rc, spdk_strerror(-rc));

	spdk_bdev_close(desc);

	free(ctx);

}

SPDK_RPC_REGISTER("bdev_nvme_get_path_iostat", rpc_bdev_nvme_get_path_iostat,

		  SPDK_RPC_RUNTIME)

python/spdk/rpc/bdev.py

+14 −0
Original line number Diff line number Diff line
@@ -967,6 +967,20 @@ def bdev_nvme_set_multipath_policy(client, name, policy, selector, rr_min_io): 
    return client.call('bdev_nvme_set_multipath_policy', params)





def bdev_nvme_get_path_iostat(client, name):

    """Get I/O statistics for IO paths of the block device.



    Args:

        name: bdev name to query



    Returns:

        I/O statistics for IO paths of the requested block device.

    """

    params = {'name': name}



    return client.call('bdev_nvme_get_path_iostat', params)





def bdev_nvme_cuse_register(client, name):

    """Register CUSE devices on NVMe controller.

scripts/rpc.py

+10 −0
Original line number Diff line number Diff line
@@ -876,6 +876,16 @@ if __name__ == "__main__": 
    p.add_argument('-r', '--rr-min-io', help='Number of IO to route to a path before switching to another for round-robin', required=False)

    p.set_defaults(func=bdev_nvme_set_multipath_policy)



    def bdev_nvme_get_path_iostat(args):

        print_dict(rpc.bdev.bdev_nvme_get_path_iostat(args.client,

                                                      name=args.name))



    p = subparsers.add_parser('bdev_nvme_get_path_iostat',

                              help="""Display current I/O statistics of all the IO paths of the blockdev. It can be

                              called when io_path_stat is true.""")

    p.add_argument('-b', '--name', help="Name of the Blockdev. Example: NVMe0n1", required=True)

    p.set_defaults(func=bdev_nvme_get_path_iostat)



    def bdev_nvme_cuse_register(args):

        rpc.bdev.bdev_nvme_cuse_register(args.client,

                                         name=args.name)