Commit 8da7772a authored by GangCao's avatar GangCao Committed by Jim Harris
Browse files

bdev/qos: add the configuration parse of bandwidth rate limiting 



To support the bandwidth rate limiting besides the IOPS rate
limiting, this patch is to add the support of the configuration
parse. The format will be as following to have a 10 (10M)
on the Malloc0 bdev. The default unit is in MB.

Limit_BWPS Malloc0 10

Change-Id: I62d70391ccad7804e6673ec56a3ed1cb0a4fbbd4
Signed-off-by: default avatarGangCao <gang.cao@intel.com>
Reviewed-on: https://review.gerrithub.io/413652


Tested-by: default avatarSPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: default avatarChangpeng Liu <changpeng.liu@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
parent 5f0c61a5

lib/bdev/bdev.c

+91 −39
Original line number Diff line number Diff line
@@ -66,6 +66,15 @@ int __itt_init_ittlib(const char *, __itt_group_id); 
#define SPDK_BDEV_SEC_TO_USEC			1000000ULL

#define SPDK_BDEV_QOS_MIN_IO_PER_TIMESLICE	1

#define SPDK_BDEV_QOS_MIN_IOS_PER_SEC		10000

#define SPDK_BDEV_QOS_MIN_BW_IN_MB_PER_SEC	10



enum spdk_bdev_qos_type {

	SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT = 0,

	SPDK_BDEV_QOS_RW_BYTEPS_RATE_LIMIT,

	SPDK_BDEV_QOS_NUM_TYPES /* Keep last */

};



static const char *qos_type_str[SPDK_BDEV_QOS_NUM_TYPES] = {"Limit_IOPS", "Limit_BWPS"};



struct spdk_bdev_mgr {

	struct spdk_mempool *bdev_io_pool;
@@ -103,7 +112,10 @@ static struct spdk_thread *g_fini_thread = NULL; 


struct spdk_bdev_qos {

	/** Rate limit, in I/O per second */

	uint64_t rate_limit;

	uint64_t iops_rate_limit;



	/** Rate limit, in byte per second */

	uint64_t byte_rate_limit;



	/** The channel that all I/O are funneled through */

	struct spdk_bdev_channel *ch;
@@ -992,7 +1004,7 @@ spdk_bdev_qos_update_max_ios_per_timeslice(struct spdk_bdev_qos *qos) 
{

	uint64_t max_ios_per_timeslice = 0;



	max_ios_per_timeslice = qos->rate_limit * SPDK_BDEV_QOS_TIMESLICE_IN_USEC /

	max_ios_per_timeslice = qos->iops_rate_limit * SPDK_BDEV_QOS_TIMESLICE_IN_USEC /

				SPDK_BDEV_SEC_TO_USEC;

	qos->max_ios_per_timeslice = spdk_max(max_ios_per_timeslice,

					      SPDK_BDEV_QOS_MIN_IO_PER_TIMESLICE);
@@ -1381,15 +1393,15 @@ spdk_bdev_get_num_blocks(const struct spdk_bdev *bdev) 
uint64_t

spdk_bdev_get_qos_ios_per_sec(struct spdk_bdev *bdev)

{

	uint64_t rate_limit = 0;

	uint64_t iops_rate_limit = 0;



	pthread_mutex_lock(&bdev->mutex);

	if (bdev->qos) {

		rate_limit = bdev->qos->rate_limit;

		iops_rate_limit = bdev->qos->iops_rate_limit;

	}

	pthread_mutex_unlock(&bdev->mutex);



	return rate_limit;

	return iops_rate_limit;

}



size_t
@@ -2425,21 +2437,73 @@ spdk_bdev_io_get_thread(struct spdk_bdev_io *bdev_io) 
	return spdk_io_channel_get_thread(bdev_io->ch->channel);

}



static void

_spdk_bdev_qos_config_type(struct spdk_bdev *bdev, uint64_t qos_set,

			   enum spdk_bdev_qos_type qos_type)

{

	uint64_t	min_qos_set = 0;



	switch (qos_type) {

	case SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT:

		min_qos_set = SPDK_BDEV_QOS_MIN_IOS_PER_SEC;

		break;

	case SPDK_BDEV_QOS_RW_BYTEPS_RATE_LIMIT:

		min_qos_set = SPDK_BDEV_QOS_MIN_BW_IN_MB_PER_SEC;

		break;

	default:

		SPDK_ERRLOG("Unsupported QoS type.\n");

		return;

	}



	if (qos_set % min_qos_set) {

		SPDK_ERRLOG("Assigned QoS %" PRIu64 " on bdev %s is not multiple of %lu\n",

			    qos_set, bdev->name, min_qos_set);

		SPDK_ERRLOG("Failed to enable QoS on this bdev %s\n", bdev->name);

		return;

	}



	if (!bdev->qos) {

		bdev->qos = calloc(1, sizeof(*bdev->qos));

		if (!bdev->qos) {

			SPDK_ERRLOG("Unable to allocate memory for QoS tracking\n");

			return;

		}

	}



	switch (qos_type) {

	case SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT:

		bdev->qos->iops_rate_limit = qos_set;

		break;

	case SPDK_BDEV_QOS_RW_BYTEPS_RATE_LIMIT:

		bdev->qos->byte_rate_limit = qos_set * 1024 * 1024;

		break;

	default:

		break;

	}



	SPDK_DEBUGLOG(SPDK_LOG_BDEV, "Bdev:%s QoS type:%d set:%lu\n",

		      bdev->name, qos_type, qos_set);



	return;

}



static void

_spdk_bdev_qos_config(struct spdk_bdev *bdev)

{

	struct spdk_conf_section	*sp = NULL;

	const char			*val = NULL;

	uint64_t			ios_per_sec = 0;

	int				i = 0;

	uint64_t			qos_set = 0;

	int				i = 0, j = 0;



	sp = spdk_conf_find_section(NULL, "QoS");

	if (!sp) {

		return;

	}



	while (j < SPDK_BDEV_QOS_NUM_TYPES) {

		i = 0;

		while (true) {

		val = spdk_conf_section_get_nmval(sp, "Limit_IOPS", i, 0);

			val = spdk_conf_section_get_nmval(sp, qos_type_str[j], i, 0);

			if (!val) {

				break;

			}
@@ -2449,32 +2513,20 @@ _spdk_bdev_qos_config(struct spdk_bdev *bdev) 
				continue;

			}



		val = spdk_conf_section_get_nmval(sp, "Limit_IOPS", i, 1);

		if (!val) {

			return;

			val = spdk_conf_section_get_nmval(sp, qos_type_str[j], i, 1);

			if (val) {

				qos_set = strtoull(val, NULL, 10);

				_spdk_bdev_qos_config_type(bdev, qos_set, j);

			}



		ios_per_sec = strtoull(val, NULL, 10);

		if (ios_per_sec > 0) {

			if (ios_per_sec % SPDK_BDEV_QOS_MIN_IOS_PER_SEC) {

				SPDK_ERRLOG("Assigned IOPS %" PRIu64 " on bdev %s is not multiple of %u\n",

					    ios_per_sec, bdev->name, SPDK_BDEV_QOS_MIN_IOS_PER_SEC);

				SPDK_ERRLOG("Failed to enable QoS on this bdev %s\n", bdev->name);

			} else {

				bdev->qos = calloc(1, sizeof(*bdev->qos));

				if (!bdev->qos) {

					SPDK_ERRLOG("Unable to allocate memory for QoS tracking\n");

					return;

				}

				bdev->qos->rate_limit = ios_per_sec;

				SPDK_DEBUGLOG(SPDK_LOG_BDEV, "Bdev:%s QoS:%lu\n",

					      bdev->name, bdev->qos->rate_limit);

			break;

		}



		j++;

	}



	return;

}

}



static int

spdk_bdev_init(struct spdk_bdev *bdev)
@@ -3098,13 +3150,13 @@ spdk_bdev_set_qos_limit_iops(struct spdk_bdev *bdev, uint64_t ios_per_sec, 
				return;

			}



			bdev->qos->rate_limit = ios_per_sec;

			bdev->qos->iops_rate_limit = ios_per_sec;

			spdk_for_each_channel(__bdev_to_io_dev(bdev),

					      _spdk_bdev_enable_qos_msg, ctx,

					      _spdk_bdev_enable_qos_done);

		} else {

			/* Updating */

			bdev->qos->rate_limit = ios_per_sec;

			bdev->qos->iops_rate_limit = ios_per_sec;

			spdk_thread_send_msg(bdev->qos->thread, _spdk_bdev_update_qos_limit_iops_msg, ctx);

		}

	} else {

test/unit/lib/bdev/mt/bdev.c/bdev_ut.c

+3 −3
Original line number Diff line number Diff line
@@ -629,7 +629,7 @@ basic_qos(void) 
	bdev->qos = calloc(1, sizeof(*bdev->qos));

	SPDK_CU_ASSERT_FATAL(bdev->qos != NULL);

	TAILQ_INIT(&bdev->qos->queued);

	bdev->qos->rate_limit = 2000; /* 2 I/O per millisecond */

	bdev->qos->iops_rate_limit = 2000; /* 2 I/O per millisecond */



	g_get_io_channel = true;
@@ -732,7 +732,7 @@ io_during_qos_queue(void) 
	bdev->qos = calloc(1, sizeof(*bdev->qos));

	SPDK_CU_ASSERT_FATAL(bdev->qos != NULL);

	TAILQ_INIT(&bdev->qos->queued);

	bdev->qos->rate_limit = 1000; /* 1000 I/O per second, or 1 per millisecond */

	bdev->qos->iops_rate_limit = 1000; /* 1000 I/O per second, or 1 per millisecond */



	g_get_io_channel = true;
@@ -815,7 +815,7 @@ io_during_qos_reset(void) 
	bdev->qos = calloc(1, sizeof(*bdev->qos));

	SPDK_CU_ASSERT_FATAL(bdev->qos != NULL);

	TAILQ_INIT(&bdev->qos->queued);

	bdev->qos->rate_limit = 1000; /* 1000 I/O per second, or 1 per millisecond */

	bdev->qos->iops_rate_limit = 1000; /* 1000 I/O per second, or 1 per millisecond */



	g_get_io_channel = true;