bdev/qos: add RPC method to set QoS at runtime (ffba4fdb) · Commits · Public Repositories / spdk

doc/jsonrpc.md

+34 −0

Original line number	Diff line number	Diff line
		@@ -172,6 +172,40 @@ Name \| Optional \| Type \| Description
		----------------------- \| -------- \| ----------- \| -----------
		name \| Required \| string \| Block device name

		## set_bdev_qos_limit_iops {#rpc_set_bdev_qos_limit_iops}

		Set an IOPS-based quality of service rate limit on a bdev.

		### Parameters

		Name \| Optional \| Type \| Description
		----------------------- \| -------- \| ----------- \| -----------
		name \| Required \| string \| Block device name
		ios_per_sec \| Required \| number \| Number of I/Os per second to allow

		### Example

		Example request:
		~~~
		{
		"jsonrpc": "2.0",
		"id": 1,
		"method": "set_bdev_qos_limit_iops",
		"params": {
		"name": "Malloc0"
		"ios_per_sec": 20000
		}
		}
		~~~

		Example response:
		~~~
		{
		"jsonrpc": "2.0",
		"id": 1,
		"result": true
		}
		~~~

		# NVMe-oF Target {#jsonrpc_components_nvmf_tgt}

include/spdk/bdev.h

+11 −0

Original line number	Diff line number	Diff line
		@@ -659,6 +659,17 @@ int spdk_bdev_flush_blocks(struct spdk_bdev_desc desc, struct spdk_io_channel
		int spdk_bdev_reset(struct spdk_bdev_desc desc, struct spdk_io_channel ch,
		spdk_bdev_io_completion_cb cb, void *cb_arg);

		/**
		* Set an IOPS-based quality of service rate limit on a bdev.
		*
		* \param bdev Block device.
		* \param ios_per_sec I/O per second limit.
		* \param cb_fn Callback function to be called when the QoS limit has been updated.
		* \param cb_arg Argument to pass to cb_fn.
		*/
		void spdk_bdev_set_qos_limit_iops(struct spdk_bdev *bdev, uint64_t ios_per_sec,
		void (cb_fn)(void cb_arg, int status), void *cb_arg);

		/**
		* Submit an NVMe Admin command to the bdev. This passes directly through
		* the block layer to the device. Support for NVMe passthru is optional,

lib/bdev/bdev.c

+117 −10

Original line number	Diff line number	Diff line
		@@ -1109,6 +1109,23 @@ spdk_bdev_qos_channel_create(struct spdk_bdev *bdev)
		return 0;
		}

		/* Caller must hold bdev->mutex */
		static int
		_spdk_bdev_enable_qos(struct spdk_bdev bdev, struct spdk_bdev_channel ch)
		{
		/* Rate limiting on this bdev enabled */
		if (bdev->ios_per_sec) {
		if (bdev->qos_channel == NULL) {
		if (spdk_bdev_qos_channel_create(bdev) != 0) {
		return -1;
		}
		}
		ch->flags \|= BDEV_CH_QOS_ENABLED;
		}

		return 0;
		}

		static int
		spdk_bdev_channel_create(void io_device, void ctx_buf)
		{
		@@ -1138,17 +1155,11 @@ spdk_bdev_channel_create(void io_device, void ctx_buf)

		pthread_mutex_lock(&bdev->mutex);

		/* Rate limiting on this bdev enabled */
		if (bdev->ios_per_sec) {
		if (bdev->qos_channel == NULL) {
		if (spdk_bdev_qos_channel_create(bdev) != 0) {
		if (_spdk_bdev_enable_qos(bdev, ch)) {
		_spdk_bdev_channel_destroy_resource(ch);
		pthread_mutex_unlock(&bdev->mutex);
		return -1;
		}
		}
		ch->flags \|= BDEV_CH_QOS_ENABLED;
		}

		bdev->channel_count++;

		@@ -2864,4 +2875,100 @@ spdk_bdev_write_zeroes_split(struct spdk_bdev_io bdev_io, bool success, void c
		spdk_bdev_io_submit(bdev_io);
		}

		struct set_qos_limit_ctx {
		void (cb_fn)(void cb_arg, int status);
		void *cb_arg;
		struct spdk_bdev *bdev;
		};

		static void
		_spdk_bdev_set_qos_limit_done(struct set_qos_limit_ctx *ctx, int status)
		{
		ctx->cb_fn(ctx->cb_arg, status);
		free(ctx);
		}

		static void
		_spdk_bdev_update_qos_limit_iops_msg(void *cb_arg)
		{
		struct set_qos_limit_ctx *ctx = cb_arg;
		struct spdk_bdev *bdev = ctx->bdev;

		/*
		* There is possibility that the QoS channel has been destroyed
		* when processing this message. Have a check here as the QoS
		* channel is protected through the critical section.
		*/
		if (bdev->qos_channel) {
		spdk_bdev_qos_update_max_ios_per_timeslice(bdev->qos_channel);
		}

		_spdk_bdev_set_qos_limit_done(ctx, 0);
		}

		static void
		_spdk_bdev_enable_qos_msg(struct spdk_io_channel_iter *i)
		{
		void *io_device = spdk_io_channel_iter_get_io_device(i);
		struct spdk_bdev *bdev = __bdev_from_io_dev(io_device);
		struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
		struct spdk_bdev_channel *bdev_ch = spdk_io_channel_get_ctx(ch);
		int rc;

		pthread_mutex_lock(&bdev->mutex);
		rc = _spdk_bdev_enable_qos(bdev, bdev_ch);
		pthread_mutex_unlock(&bdev->mutex);

		spdk_for_each_channel_continue(i, rc);
		}

		static void
		_spdk_bdev_enable_qos_done(struct spdk_io_channel_iter *i, int status)
		{
		struct set_qos_limit_ctx *ctx = spdk_io_channel_iter_get_ctx(i);

		_spdk_bdev_set_qos_limit_done(ctx, status);
		}

		void
		spdk_bdev_set_qos_limit_iops(struct spdk_bdev *bdev, uint64_t ios_per_sec,
		void (cb_fn)(void cb_arg, int status), void *cb_arg)
		{
		struct set_qos_limit_ctx *ctx;

		if (ios_per_sec == 0 \|\| ios_per_sec % SPDK_BDEV_QOS_MIN_IOS_PER_SEC) {
		SPDK_ERRLOG("Requested ios_per_sec limit %" PRIu64 " is not a multiple of %u\n",
		ios_per_sec, SPDK_BDEV_QOS_MIN_IOS_PER_SEC);
		cb_fn(cb_arg, -EINVAL);
		return;
		}

		ctx = calloc(1, sizeof(*ctx));
		if (ctx == NULL) {
		cb_fn(cb_arg, -ENOMEM);
		return;
		}

		ctx->cb_fn = cb_fn;
		ctx->cb_arg = cb_arg;

		pthread_mutex_lock(&bdev->mutex);
		bdev->ios_per_sec = ios_per_sec;
		if (bdev->qos_thread) {
		/*
		* QoS is already enabled, so just update the limit information on the QoS thread.
		*/
		ctx->bdev = bdev;
		spdk_thread_send_msg(bdev->qos_thread, _spdk_bdev_update_qos_limit_iops_msg, ctx);
		pthread_mutex_unlock(&bdev->mutex);
		return;
		}
		pthread_mutex_unlock(&bdev->mutex);

		/* Enable QoS on all channels. */
		spdk_for_each_channel(__bdev_to_io_dev(bdev),
		_spdk_bdev_enable_qos_msg, ctx,
		_spdk_bdev_enable_qos_done);
		}

		SPDK_LOG_REGISTER_COMPONENT("bdev", SPDK_LOG_BDEV)

lib/bdev/rpc/bdev_rpc.c

+71 −0

Original line number	Diff line number	Diff line
		@@ -304,3 +304,74 @@ invalid:
		free_rpc_delete_bdev(&req);
		}
		SPDK_RPC_REGISTER("delete_bdev", spdk_rpc_delete_bdev)


		struct rpc_set_bdev_qos_limit_iops {
		char *name;
		uint64_t ios_per_sec;
		};

		static void
		free_rpc_set_bdev_qos_limit_iops(struct rpc_set_bdev_qos_limit_iops *r)
		{
		free(r->name);
		}

		static const struct spdk_json_object_decoder rpc_set_bdev_qos_limit_iops_decoders[] = {
		{"name", offsetof(struct rpc_set_bdev_qos_limit_iops, name), spdk_json_decode_string},
		{"ios_per_sec", offsetof(struct rpc_set_bdev_qos_limit_iops, ios_per_sec), spdk_json_decode_uint64},
		};

		static void
		spdk_rpc_set_bdev_qos_limit_iops_complete(void *cb_arg, int status)
		{
		struct spdk_jsonrpc_request *request = cb_arg;
		struct spdk_json_write_ctx *w;

		if (status != 0) {
		spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,
		"Failed to configure IOPS limit: %s",
		spdk_strerror(-status));
		return;
		}

		w = spdk_jsonrpc_begin_result(request);
		if (w == NULL) {
		return;
		}

		spdk_json_write_bool(w, true);
		spdk_jsonrpc_end_result(request, w);
		}

		static void
		spdk_rpc_set_bdev_qos_limit_iops(struct spdk_jsonrpc_request *request,
		const struct spdk_json_val *params)
		{
		struct rpc_set_bdev_qos_limit_iops req = {};
		struct spdk_bdev *bdev;

		if (spdk_json_decode_object(params, rpc_set_bdev_qos_limit_iops_decoders,
		SPDK_COUNTOF(rpc_set_bdev_qos_limit_iops_decoders),
		&req)) {
		SPDK_ERRLOG("spdk_json_decode_object failed\n");
		goto invalid;
		}

		bdev = spdk_bdev_get_by_name(req.name);
		if (bdev == NULL) {
		SPDK_ERRLOG("bdev '%s' does not exist\n", req.name);
		goto invalid;
		}

		free_rpc_set_bdev_qos_limit_iops(&req);
		spdk_bdev_set_qos_limit_iops(bdev, req.ios_per_sec,
		spdk_rpc_set_bdev_qos_limit_iops_complete, request);
		return;

		invalid:
		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, "Invalid parameters");
		free_rpc_set_bdev_qos_limit_iops(&req);
		}

		SPDK_RPC_REGISTER("set_bdev_qos_limit_iops", spdk_rpc_set_bdev_qos_limit_iops)

scripts/rpc.py

+9 −0

Original line number	Diff line number	Diff line
		@@ -198,6 +198,15 @@ if __name__ == "__main__":
		'bdev_name', help='Blockdev name to be deleted. Example: Malloc0.')
		p.set_defaults(func=delete_bdev)

		@call_cmd
		def set_bdev_qos_limit_iops(args):
		rpc.bdev.set_bdev_qos_limit_iops(args.client, args)

		p = subparsers.add_parser('set_bdev_qos_limit_iops', help='Set QoS IOPS limit on a blockdev')
		p.add_argument('name', help='Blockdev name to set QoS. Example: Malloc0')
		p.add_argument('ios_per_sec', help='IOs per second limit (>=10000). Example: 20000', type=int)
		p.set_defaults(func=set_bdev_qos_limit_iops)

		@call_cmd
		def bdev_inject_error(args):
		rpc.bdev.bdev_inject_error(args.client, args)