Commit 316cb3b1 authored by GangCao's avatar GangCao Committed by Ben Walker
Browse files

QoS/Bdev: add the Read/Write separate bandwidth rate limits 



This patch adds the support of read and write separate
bandwidth rate limits control with the configuration file.

Below is the example (in MiB) for the configuration section:

[QoS]
  Limit_Read_BPS Malloc0 100
  Limit_Write_BPS Nvme0n1 200

Change-Id: I0221516ce70c3fbb07b9e80c1c814ed5ba271c88
Signed-off-by: default avatarGangCao <gang.cao@intel.com>
Reviewed-on: https://review.gerrithub.io/c/416672


Chandler-Test-Pool: SPDK Automated Test System <sys_sgsw@intel.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
parent 1da6e2f5

include/spdk/bdev.h

+4 −0
Original line number Diff line number Diff line
@@ -112,6 +112,10 @@ enum spdk_bdev_qos_rate_limit_type { 
	SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT = 0,

	/** Byte per second rate limit for both read and write */

	SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT,

	/** Byte per second rate limit for read only */

	SPDK_BDEV_QOS_R_BPS_RATE_LIMIT,

	/** Byte per second rate limit for write only */

	SPDK_BDEV_QOS_W_BPS_RATE_LIMIT,

	/** Keep last */

	SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES

};

lib/bdev/bdev.c

+75 −2
Original line number Diff line number Diff line
@@ -81,8 +81,12 @@ int __itt_init_ittlib(const char *, __itt_group_id); 


#define SPDK_BDEV_POOL_ALIGNMENT 512



static const char *qos_conf_type[] = {"Limit_IOPS", "Limit_BPS"};

static const char *qos_rpc_type[] = {"rw_ios_per_sec", "rw_mbytes_per_sec"};

static const char *qos_conf_type[] = {"Limit_IOPS",

				      "Limit_BPS", "Limit_Read_BPS", "Limit_Write_BPS"

				     };

static const char *qos_rpc_type[] = {"rw_ios_per_sec",

				     "rw_mbytes_per_sec", "r_mbytes_per_sec", "w_mbytes_per_sec"

				    };



TAILQ_HEAD(spdk_bdev_list, spdk_bdev);
@@ -1214,6 +1218,8 @@ _spdk_bdev_qos_is_iops_rate_limit(enum spdk_bdev_qos_rate_limit_type limit) 
	case SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT:

		return true;

	case SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT:

	case SPDK_BDEV_QOS_R_BPS_RATE_LIMIT:

	case SPDK_BDEV_QOS_W_BPS_RATE_LIMIT:

		return false;

	case SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES:

	default:
@@ -1237,6 +1243,25 @@ _spdk_bdev_qos_io_to_limit(struct spdk_bdev_io *bdev_io) 
	}

}



static bool

_spdk_bdev_is_read_io(struct spdk_bdev_io *bdev_io)

{

	switch (bdev_io->type) {

	case SPDK_BDEV_IO_TYPE_NVME_IO:

	case SPDK_BDEV_IO_TYPE_NVME_IO_MD:

		/* Bit 1 (0x2) set for read operation */

		if (bdev_io->u.nvme_passthru.cmd.opc & SPDK_NVME_OPC_READ) {

			return true;

		} else {

			return false;

		}

	case SPDK_BDEV_IO_TYPE_READ:

		return true;

	default:

		return false;

	}

}



static uint64_t

_spdk_bdev_get_io_size_in_byte(struct spdk_bdev_io *bdev_io)

{
@@ -1266,6 +1291,26 @@ _spdk_bdev_qos_rw_queue_io(const struct spdk_bdev_qos_limit *limit, struct spdk_ 
	}

}



static bool

_spdk_bdev_qos_r_queue_io(const struct spdk_bdev_qos_limit *limit, struct spdk_bdev_io *io)

{

	if (_spdk_bdev_is_read_io(io) == false) {

		return false;

	}



	return _spdk_bdev_qos_rw_queue_io(limit, io);

}



static bool

_spdk_bdev_qos_w_queue_io(const struct spdk_bdev_qos_limit *limit, struct spdk_bdev_io *io)

{

	if (_spdk_bdev_is_read_io(io) == true) {

		return false;

	}



	return _spdk_bdev_qos_rw_queue_io(limit, io);

}



static void

_spdk_bdev_qos_rw_iops_update_quota(struct spdk_bdev_qos_limit *limit, struct spdk_bdev_io *io)

{
@@ -1278,6 +1323,26 @@ _spdk_bdev_qos_rw_bps_update_quota(struct spdk_bdev_qos_limit *limit, struct spd 
	limit->remaining_this_timeslice -= _spdk_bdev_get_io_size_in_byte(io);

}



static void

_spdk_bdev_qos_r_bps_update_quota(struct spdk_bdev_qos_limit *limit, struct spdk_bdev_io *io)

{

	if (_spdk_bdev_is_read_io(io) == false) {

		return;

	}



	return _spdk_bdev_qos_rw_bps_update_quota(limit, io);

}



static void

_spdk_bdev_qos_w_bps_update_quota(struct spdk_bdev_qos_limit *limit, struct spdk_bdev_io *io)

{

	if (_spdk_bdev_is_read_io(io) == true) {

		return;

	}



	return _spdk_bdev_qos_rw_bps_update_quota(limit, io);

}



static void

_spdk_bdev_qos_set_ops(struct spdk_bdev_qos *qos)

{
@@ -1299,6 +1364,14 @@ _spdk_bdev_qos_set_ops(struct spdk_bdev_qos *qos) 
			qos->rate_limits[i].queue_io = _spdk_bdev_qos_rw_queue_io;

			qos->rate_limits[i].update_quota = _spdk_bdev_qos_rw_bps_update_quota;

			break;

		case SPDK_BDEV_QOS_R_BPS_RATE_LIMIT:

			qos->rate_limits[i].queue_io = _spdk_bdev_qos_r_queue_io;

			qos->rate_limits[i].update_quota = _spdk_bdev_qos_r_bps_update_quota;

			break;

		case SPDK_BDEV_QOS_W_BPS_RATE_LIMIT:

			qos->rate_limits[i].queue_io = _spdk_bdev_qos_w_queue_io;

			qos->rate_limits[i].update_quota = _spdk_bdev_qos_w_bps_update_quota;

			break;

		default:

			break;

		}

lib/bdev/rpc/bdev_rpc.c

+3 −5
Original line number Diff line number Diff line
@@ -550,9 +550,8 @@ static void 
spdk_rpc_set_bdev_qos_limit(struct spdk_jsonrpc_request *request,

			    const struct spdk_json_val *params)

{

	struct rpc_set_bdev_qos_limit req = {NULL, {UINT64_MAX, UINT64_MAX}};

	struct rpc_set_bdev_qos_limit req = {NULL, {UINT64_MAX, UINT64_MAX, UINT64_MAX, UINT64_MAX}};

	struct spdk_bdev *bdev;

	bool valid_limit = false;

	int i;



	if (spdk_json_decode_object(params, rpc_set_bdev_qos_limit_decoders,
@@ -572,11 +571,10 @@ spdk_rpc_set_bdev_qos_limit(struct spdk_jsonrpc_request *request, 


	for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {

		if (req.limits[i] != UINT64_MAX) {

			valid_limit = true;

			break;

		}

	}



	if (valid_limit == false) {

	if (i == SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES) {

		SPDK_ERRLOG("no rate limits specified\n");

		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,

						 "No rate limits specified");

test/bdev/bdev.conf.in

+2 −0
Original line number Diff line number Diff line
@@ -35,6 +35,8 @@ 
  # Assign 100 (MB) bandwidth for the Malloc3 block

  # device

  Limit_BPS Malloc3 100

  # Assign 50 (MB) read only bandwidth for the AIO0 block device

  Limit_Read_BPS AIO0 50



[RAID0]

  Name raid0

test/spdkcli/match_files/spdkcli_details_vhost.test.match

+3 −1
Original line number Diff line number Diff line 
{

  "aliases": [],

  "assigned_rate_limits": {

    "r_mbytes_per_sec": $(N),

    "rw_ios_per_sec": $(N),

    "rw_mbytes_per_sec": $(N)

    "rw_mbytes_per_sec": $(N),

    "w_mbytes_per_sec": $(N)

  },

  "block_size": $(N),

  "claimed": false,