Commit 868c28cd authored by GangCao's avatar GangCao Committed by Ben Walker
Browse files

QoS/Bdev: add the RPC support for the bandwidth rate limit 



This patch added the support of RPC method to enable,
adjust, disable the bandwidth rate limit on the bdev.

And it can work together with the existing IOPS rate limit.

The RPC method has been consolidated to support both IOPS
and bandwidth rate limits as below:

usage:
rpc.py set_bdev_qos_limit [-h]
                          [--rw_ios_per_sec RW_IOS_PER_SEC]
                          [--rw_mbytes_per_sec RW_MBYTES_PER_SEC]
                          name

positional arguments:
  name       Blockdev name to set QoS. Example: Malloc0

optional arguments:
  -h, --help show this help message and exit
  --rw_ios_per_sec RW_IOS_PER_SEC
             R/W IOs per second limit (>=10000, example: 20000).
             0 means unlimited.
  --rw_mbytes_per_sec RW_MBYTES_PER_SEC
             R/W megabytes per second limit (>=10, example: 100).
             0 means unlimited.

Change-Id: I9c03cd635280add01801a81c6a6c02f0cf85bee1
Signed-off-by: default avatarGangCao <gang.cao@intel.com>
Reviewed-on: https://review.gerrithub.io/416511


Chandler-Test-Pool: SPDK Automated Test System <sys_sgsw@intel.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarZiye Yang <optimistyzy@gmail.com>
Reviewed-by: default avatarChangpeng Liu <changpeng.liu@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
parent d9ecb572

CHANGELOG.md

+6 −0
Original line number Diff line number Diff line
@@ -35,6 +35,12 @@ currently marked as experimental. Do not use in production. 
The RAID virtual bdev module is now always enabled by default.  The configure --with-raid and

--without-raid options are now ignored and deprecated and will be removed in the next release.



Enforcement of bandwidth limits for quality of service (QoS) has been added to the bdev layer.

See the new [set_bdev_qos_limit](http://www.spdk.io/doc/jsonrpc.html#rpc_set_bdev_qos_limit)

documentation for more details. The previous set_bdev_qos_limit_iops RPC method introduced at

18.04 release has been deprecated. The new set_bdev_qos_limit RPC method can support both

bandwidth and IOPS limits.



### Environment Abstraction Layer and Event Framework



The size parameter of spdk_mem_map_translate is now a pointer. This allows the

doc/jsonrpc.md

+8 −5
Original line number Diff line number Diff line
@@ -230,6 +230,7 @@ Example response: 
    "set_iscsi_options",

    "set_bdev_options",

    "set_bdev_qos_limit_iops",

    "set_bdev_qos_limit",

    "delete_bdev",

    "get_bdevs",

    "get_bdevs_iostat",
@@ -627,16 +628,17 @@ Example response: 
}

~~~



## set_bdev_qos_limit_iops {#rpc_set_bdev_qos_limit_iops}

## set_bdev_qos_limit {#rpc_set_bdev_qos_limit}



Set an IOPS-based quality of service rate limit on a bdev.

Set the quality of service rate limit on a bdev.



### Parameters



Name                    | Optional | Type        | Description

----------------------- | -------- | ----------- | -----------

name                    | Required | string      | Block device name

ios_per_sec             | Required | number      | Number of I/Os per second to allow. 0 means unlimited.

rw_ios_per_sec          | Optional | number      | Number of R/W I/Os per second to allow. 0 means unlimited.

rw_mbytes_per_sec       | Optional | number      | Number of R/W megabytes per second to allow. 0 means unlimited.



### Example
@@ -645,10 +647,11 @@ Example request: 
{

  "jsonrpc": "2.0",

  "id": 1,

  "method": "set_bdev_qos_limit_iops",

  "method": "set_bdev_qos_limit",

  "params": {

    "name": "Malloc0"

    "ios_per_sec": 20000

    "rw_ios_per_sec": 20000

    "rw_mbytes_per_sec": 100

  }

}

~~~

lib/bdev/bdev.c

+36 −1
Original line number Diff line number Diff line
@@ -80,7 +80,7 @@ int __itt_init_ittlib(const char *, __itt_group_id); 
#define SPDK_BDEV_QOS_LIMIT_NOT_DEFINED		UINT64_MAX



static const char *qos_conf_type[] = {"Limit_IOPS", "Limit_BPS"};

static const char *qos_rpc_type[] = {"qos_ios_per_sec"};

static const char *qos_rpc_type[] = {"rw_ios_per_sec", "rw_mbytes_per_sec"};



TAILQ_HEAD(spdk_bdev_list, spdk_bdev);
@@ -517,6 +517,35 @@ spdk_bdev_config_text(FILE *fp) 
	}

}



static void

spdk_bdev_qos_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w)

{

	int i;

	struct spdk_bdev_qos *qos = bdev->internal.qos;

	uint64_t limits[SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES];



	if (!qos) {

		return;

	}



	spdk_bdev_get_qos_rate_limits(bdev, limits);



	spdk_json_write_object_begin(w);

	spdk_json_write_named_string(w, "method", "set_bdev_qos_limit");

	spdk_json_write_name(w, "params");



	spdk_json_write_object_begin(w);

	spdk_json_write_named_string(w, "name", bdev->name);

	for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {

		if (limits[i] > 0) {

			spdk_json_write_named_uint64(w, qos_rpc_type[i], limits[i]);

		}

	}

	spdk_json_write_object_end(w);



	spdk_json_write_object_end(w);

}



void

spdk_bdev_subsystem_config_json(struct spdk_json_write_ctx *w)

{
@@ -543,6 +572,8 @@ spdk_bdev_subsystem_config_json(struct spdk_json_write_ctx *w) 
	}



	TAILQ_FOREACH(bdev, &g_bdev_mgr.bdevs, internal.link) {

		spdk_bdev_qos_config_json(bdev, w);



		if (bdev->fn_table->write_config_json) {

			bdev->fn_table->write_config_json(bdev, w);

		}
@@ -1953,6 +1984,10 @@ spdk_bdev_get_qos_rate_limits(struct spdk_bdev *bdev, uint64_t *limits) 
			if (bdev->internal.qos->rate_limits[i].limit !=

			    SPDK_BDEV_QOS_LIMIT_NOT_DEFINED) {

				limits[i] = bdev->internal.qos->rate_limits[i].limit;

				if (_spdk_bdev_qos_is_iops_rate_limit(i) == false) {

					/* Change from Byte to Megabyte which is user visible. */

					limits[i] = limits[i] / 1024 / 1024;

				}

			}

		}

	}

lib/bdev/rpc/bdev_rpc.c

+45 −25
Original line number Diff line number Diff line
@@ -220,6 +220,7 @@ spdk_rpc_dump_bdev_info(struct spdk_json_write_ctx *w, 
{

	struct spdk_bdev_alias *tmp;

	uint64_t qos_limits[SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES];

	int i;



	spdk_json_write_object_begin(w);
@@ -251,9 +252,14 @@ spdk_rpc_dump_bdev_info(struct spdk_json_write_ctx *w, 
		spdk_json_write_named_string(w, "uuid", uuid_str);

	}



	spdk_json_write_name(w, "assigned_rate_limits");

	spdk_json_write_object_begin(w);

	spdk_bdev_get_qos_rate_limits(bdev, qos_limits);

	spdk_json_write_name(w, spdk_bdev_get_qos_rpc_type(SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT));

	spdk_json_write_uint64(w, qos_limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT]);

	for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {

		spdk_json_write_name(w, spdk_bdev_get_qos_rpc_type(i));

		spdk_json_write_uint64(w, qos_limits[i]);

	}

	spdk_json_write_object_end(w);



	spdk_json_write_name(w, "claimed");

	spdk_json_write_bool(w, (bdev->internal.claim_module != NULL));
@@ -484,31 +490,40 @@ SPDK_RPC_REGISTER("set_bdev_qd_sampling_period", 
		  spdk_rpc_set_bdev_qd_sampling_period,

		  SPDK_RPC_RUNTIME)



struct rpc_set_bdev_qos_limit_iops {

struct rpc_set_bdev_qos_limit {

	char		*name;

	uint64_t	ios_per_sec;

	uint64_t	limits[SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES];

};



static void

free_rpc_set_bdev_qos_limit_iops(struct rpc_set_bdev_qos_limit_iops *r)

free_rpc_set_bdev_qos_limit(struct rpc_set_bdev_qos_limit *r)

{

	free(r->name);

}



static const struct spdk_json_object_decoder rpc_set_bdev_qos_limit_iops_decoders[] = {

	{"name", offsetof(struct rpc_set_bdev_qos_limit_iops, name), spdk_json_decode_string},

	{"ios_per_sec", offsetof(struct rpc_set_bdev_qos_limit_iops, ios_per_sec), spdk_json_decode_uint64},

static const struct spdk_json_object_decoder rpc_set_bdev_qos_limit_decoders[] = {

	{"name", offsetof(struct rpc_set_bdev_qos_limit, name), spdk_json_decode_string},

	{

		"rw_ios_per_sec", offsetof(struct rpc_set_bdev_qos_limit,

					   limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT]),

		spdk_json_decode_uint64, true

	},

	{

		"rw_mbytes_per_sec", offsetof(struct rpc_set_bdev_qos_limit,

					      limits[SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT]),

		spdk_json_decode_uint64, true

	},

};



static void

spdk_rpc_set_bdev_qos_limit_iops_complete(void *cb_arg, int status)

spdk_rpc_set_bdev_qos_limit_complete(void *cb_arg, int status)

{

	struct spdk_jsonrpc_request *request = cb_arg;

	struct spdk_json_write_ctx *w;



	if (status != 0) {

		spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,

						     "Failed to configure IOPS limit: %s",

						     "Failed to configure rate limit: %s",

						     spdk_strerror(-status));

		return;

	}
@@ -523,15 +538,16 @@ spdk_rpc_set_bdev_qos_limit_iops_complete(void *cb_arg, int status) 
}



static void

spdk_rpc_set_bdev_qos_limit_iops(struct spdk_jsonrpc_request *request,

spdk_rpc_set_bdev_qos_limit(struct spdk_jsonrpc_request *request,

			    const struct spdk_json_val *params)

{

	struct rpc_set_bdev_qos_limit_iops req = {NULL, UINT64_MAX};

	struct rpc_set_bdev_qos_limit req = {NULL, {UINT64_MAX, UINT64_MAX}};

	struct spdk_bdev *bdev;

	uint64_t limits[SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES] = {UINT64_MAX, UINT64_MAX};

	bool valid_limit = false;

	int i;



	if (spdk_json_decode_object(params, rpc_set_bdev_qos_limit_iops_decoders,

				    SPDK_COUNTOF(rpc_set_bdev_qos_limit_iops_decoders),

	if (spdk_json_decode_object(params, rpc_set_bdev_qos_limit_decoders,

				    SPDK_COUNTOF(rpc_set_bdev_qos_limit_decoders),

				    &req)) {

		SPDK_ERRLOG("spdk_json_decode_object failed\n");

		goto invalid;
@@ -545,23 +561,27 @@ spdk_rpc_set_bdev_qos_limit_iops(struct spdk_jsonrpc_request *request, 
		goto exit;

	}



	if (req.ios_per_sec == UINT64_MAX) {

		SPDK_ERRLOG("invalid rate limits set\n");

	for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {

		if (req.limits[i] != UINT64_MAX) {

			valid_limit = true;

		}

	}



	if (valid_limit == false) {

		SPDK_ERRLOG("no rate limits specified\n");

		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,

						 "Invalid rate limits");

						 "No rate limits specified");

		goto exit;

	}



	free_rpc_set_bdev_qos_limit_iops(&req);

	limits[SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT] = req.ios_per_sec;

	spdk_bdev_set_qos_rate_limits(bdev, limits, spdk_rpc_set_bdev_qos_limit_iops_complete,

				      request);

	free_rpc_set_bdev_qos_limit(&req);

	spdk_bdev_set_qos_rate_limits(bdev, req.limits, spdk_rpc_set_bdev_qos_limit_complete, request);

	return;



invalid:

	spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, "Invalid parameters");

exit:

	free_rpc_set_bdev_qos_limit_iops(&req);

	free_rpc_set_bdev_qos_limit(&req);

}



SPDK_RPC_REGISTER("set_bdev_qos_limit_iops", spdk_rpc_set_bdev_qos_limit_iops, SPDK_RPC_RUNTIME)
 
SPDK_RPC_REGISTER("set_bdev_qos_limit", spdk_rpc_set_bdev_qos_limit, SPDK_RPC_RUNTIME)

scripts/rpc.py

+13 −8
Original line number Diff line number Diff line
@@ -449,16 +449,21 @@ if __name__ == "__main__": 
    p.set_defaults(func=set_bdev_qd_sampling_period)



    @call_cmd

    def set_bdev_qos_limit_iops(args):

        rpc.bdev.set_bdev_qos_limit_iops(args.client,

    def set_bdev_qos_limit(args):

        rpc.bdev.set_bdev_qos_limit(args.client,

                                    name=args.name,

                                         ios_per_sec=args.ios_per_sec)

                                    rw_ios_per_sec=args.rw_ios_per_sec,

                                    rw_mbytes_per_sec=args.rw_mbytes_per_sec)



    p = subparsers.add_parser('set_bdev_qos_limit_iops', help='Set QoS IOPS limit on a blockdev')

    p = subparsers.add_parser('set_bdev_qos_limit', help='Set QoS rate limit on a blockdev')

    p.add_argument('name', help='Blockdev name to set QoS. Example: Malloc0')

    p.add_argument('ios_per_sec',

                   help='IOs per second limit (>=10000, example: 20000). 0 means unlimited.', type=int)

    p.set_defaults(func=set_bdev_qos_limit_iops)

    p.add_argument('--rw_ios_per_sec',

                   help='R/W IOs per second limit (>=10000, example: 20000). 0 means unlimited.',

                   type=int, required=False)

    p.add_argument('--rw_mbytes_per_sec',

                   help="R/W megabytes per second limit (>=10, example: 100). 0 means unlimited.",

                   type=int, required=False)

    p.set_defaults(func=set_bdev_qos_limit)



    @call_cmd

    def bdev_inject_error(args):