bdev/raid: add qos for raid process (89fd1730) · Commits · Public Repositories / spdk

doc/jsonrpc.md

+10 −4

Original line number	Diff line number	Diff line
		@@ -10902,12 +10902,17 @@ This RPC can be called at any time, but the new value will only take effect for
		The `process_window_size_kb` parameter defines the size of the "window" (LBA range of the raid bdev)
		in which a background process like rebuild performs its work. Any positive value is valid, but the value
		actually used by a raid bdev can be adjusted to the size of the raid bdev or the write unit size.
		`process_max_bandwidth_mb_sec` parameter defines the maximum bandwidth used by a background process like
		rebuild. Any positive value or zero is valid, zero means no bandwidth limitation for background process.
		It can only limit the process bandwidth but doesn't guarantee it can be reached. Changing this value will
		not affect existing processes, it will only take effect on new processes generated after the RPC is completed.

		#### Parameters

		Name \| Optional \| Type \| Description
		-------------------------- \| -------- \| ----------- \| -----------
		----------------------------- \| -------- \| ----------- \| -----------
		process_window_size_kb \| Optional \| number \| Background process (e.g. rebuild) window size in KiB
		process_max_bandwidth_mb_sec \| Optional \| number \| Background process (e.g. rebuild) maximum bandwidth in MiB/Sec

		#### Example

		@@ -10920,7 +10925,8 @@ request:
		"method": "bdev_raid_set_options",
		"id": 1,
		"params": {
		"process_window_size_kb": 512
		"process_window_size_kb": 512,
		"process_max_bandwidth_mb_sec": 100
		}
		}
		~~~

include/spdk/util.h

+1 −0

Original line number	Diff line number	Diff line
		@@ -41,6 +41,7 @@ extern "C" {
		*/
		#define SPDK_COUNTOF_MEMBER(type, member) (SPDK_COUNTOF(((type *)0)->member))

		#define SPDK_SEC_TO_MSEC 1000ULL
		#define SPDK_SEC_TO_USEC 1000000ULL
		#define SPDK_SEC_TO_NSEC 1000000000ULL

module/bdev/raid/bdev_raid.c

+87 −9

Original line number	Diff line number	Diff line
		@@ -17,6 +17,7 @@
		#define RAID_BDEV_PROCESS_MAX_QD 16

		#define RAID_BDEV_PROCESS_WINDOW_SIZE_KB_DEFAULT 1024
		#define RAID_BDEV_PROCESS_MAX_BANDWIDTH_MB_SEC_DEFAULT 0

		static bool g_shutdown_started = false;

		@@ -51,6 +52,15 @@ enum raid_bdev_process_state {
		RAID_PROCESS_STATE_STOPPED,
		};

		struct raid_process_qos {
		bool enable_qos;
		uint64_t last_tsc;
		double bytes_per_tsc;
		double bytes_available;
		double bytes_max;
		struct spdk_poller *process_continue_poller;
		};

		struct raid_bdev_process {
		struct raid_bdev *raid_bdev;
		enum raid_process_type type;
		@@ -67,6 +77,7 @@ struct raid_bdev_process {
		struct raid_base_bdev_info *target;
		int status;
		TAILQ_HEAD(, raid_process_finish_action) finish_actions;
		struct raid_process_qos qos;
		};

		struct raid_process_finish_action {
		@@ -77,6 +88,7 @@ struct raid_process_finish_action {

		static struct spdk_raid_bdev_opts g_opts = {
		.process_window_size_kb = RAID_BDEV_PROCESS_WINDOW_SIZE_KB_DEFAULT,
		.process_max_bandwidth_mb_sec = RAID_BDEV_PROCESS_MAX_BANDWIDTH_MB_SEC_DEFAULT,
		};

		void
		@@ -1400,6 +1412,8 @@ raid_bdev_opts_config_json(struct spdk_json_write_ctx *w)

		spdk_json_write_named_object_begin(w, "params");
		spdk_json_write_named_uint32(w, "process_window_size_kb", g_opts.process_window_size_kb);
		spdk_json_write_named_uint32(w, "process_max_bandwidth_mb_sec",
		g_opts.process_max_bandwidth_mb_sec);
		spdk_json_write_object_end(w);

		spdk_json_write_object_end(w);
		@@ -2449,6 +2463,8 @@ _raid_bdev_process_finish_done(void *ctx)
		free(finish_action);
		}

		spdk_poller_unregister(&process->qos.process_continue_poller);

		raid_bdev_process_free(process);

		spdk_thread_exit(spdk_get_thread());
		@@ -2782,11 +2798,64 @@ raid_bdev_process_window_range_locked(void *ctx, int status)
		_raid_bdev_process_thread_run(process);
		}

		static bool
		raid_bdev_process_consume_token(struct raid_bdev_process *process)
		{
		struct raid_bdev *raid_bdev = process->raid_bdev;
		uint64_t now = spdk_get_ticks();

		process->qos.bytes_available = spdk_min(process->qos.bytes_max,
		process->qos.bytes_available +
		(now - process->qos.last_tsc) * process->qos.bytes_per_tsc);
		process->qos.last_tsc = now;
		if (process->qos.bytes_available > 0.0) {
		process->qos.bytes_available -= process->window_size * raid_bdev->bdev.blocklen;
		return true;
		}
		return false;
		}

		static bool
		raid_bdev_process_lock_window_range(struct raid_bdev_process *process)
		{
		struct raid_bdev *raid_bdev = process->raid_bdev;
		int rc;

		assert(process->window_range_locked == false);

		if (process->qos.enable_qos) {
		if (raid_bdev_process_consume_token(process)) {
		spdk_poller_pause(process->qos.process_continue_poller);
		} else {
		spdk_poller_resume(process->qos.process_continue_poller);
		return false;
		}
		}

		rc = spdk_bdev_quiesce_range(&raid_bdev->bdev, &g_raid_if,
		process->window_offset, process->max_window_size,
		raid_bdev_process_window_range_locked, process);
		if (rc != 0) {
		raid_bdev_process_window_range_locked(process, rc);
		}
		return true;
		}

		static int
		raid_bdev_process_continue_poll(void *arg)
		{
		struct raid_bdev_process *process = arg;

		if (raid_bdev_process_lock_window_range(process)) {
		return SPDK_POLLER_BUSY;
		}
		return SPDK_POLLER_IDLE;
		}

		static void
		raid_bdev_process_thread_run(struct raid_bdev_process *process)
		{
		struct raid_bdev *raid_bdev = process->raid_bdev;
		int rc;

		assert(spdk_get_thread() == process->thread);
		assert(process->window_remaining == 0);
		@@ -2805,13 +2874,7 @@ raid_bdev_process_thread_run(struct raid_bdev_process *process)

		process->max_window_size = spdk_min(raid_bdev->bdev.blockcnt - process->window_offset,
		process->max_window_size);

		rc = spdk_bdev_quiesce_range(&raid_bdev->bdev, &g_raid_if,
		process->window_offset, process->max_window_size,
		raid_bdev_process_window_range_locked, process);
		if (rc != 0) {
		raid_bdev_process_window_range_locked(process, rc);
		}
		raid_bdev_process_lock_window_range(process);
		}

		static void
		@@ -2833,6 +2896,12 @@ raid_bdev_process_thread_init(void *ctx)
		process->raid_ch = spdk_io_channel_get_ctx(ch);
		process->state = RAID_PROCESS_STATE_RUNNING;

		if (process->qos.enable_qos) {
		process->qos.process_continue_poller = SPDK_POLLER_REGISTER(raid_bdev_process_continue_poll,
		process, 0);
		spdk_poller_pause(process->qos.process_continue_poller);
		}

		SPDK_NOTICELOG("Started %s on raid bdev %s\n",
		raid_bdev_process_to_str(process->type), raid_bdev->bdev.name);

		@@ -2999,6 +3068,15 @@ raid_bdev_process_alloc(struct raid_bdev *raid_bdev, enum raid_process_type type
		TAILQ_INIT(&process->requests);
		TAILQ_INIT(&process->finish_actions);

		if (g_opts.process_max_bandwidth_mb_sec != 0) {
		process->qos.enable_qos = true;
		process->qos.last_tsc = spdk_get_ticks();
		process->qos.bytes_per_tsc = g_opts.process_max_bandwidth_mb_sec * 1024 * 1024.0 /
		spdk_get_ticks_hz();
		process->qos.bytes_max = g_opts.process_max_bandwidth_mb_sec * 1024 * 1024.0 / SPDK_SEC_TO_MSEC;
		process->qos.bytes_available = 0.0;
		}

		for (i = 0; i < RAID_BDEV_PROCESS_MAX_QD; i++) {
		process_req = raid_bdev_process_alloc_request(process);
		if (process_req == NULL) {

module/bdev/raid/bdev_raid.h

+2 −0

Original line number	Diff line number	Diff line
		@@ -553,6 +553,8 @@ int raid_bdev_load_base_bdev_superblock(struct spdk_bdev_desc *desc, struct spdk
		struct spdk_raid_bdev_opts {
		/* Size of the background process window in KiB */
		uint32_t process_window_size_kb;
		/* Maximum bandwidth in MiB to process per second */
		uint32_t process_max_bandwidth_mb_sec;
		};

		void raid_bdev_get_opts(struct spdk_raid_bdev_opts *opts);

module/bdev/raid/bdev_raid_rpc.c

+1 −0

Original line number	Diff line number	Diff line
		@@ -607,6 +607,7 @@ SPDK_RPC_REGISTER("bdev_raid_remove_base_bdev", rpc_bdev_raid_remove_base_bdev,

		static const struct spdk_json_object_decoder rpc_bdev_raid_options_decoders[] = {
		{"process_window_size_kb", offsetof(struct spdk_raid_bdev_opts, process_window_size_kb), spdk_json_decode_uint32, true},
		{"process_max_bandwidth_mb_sec", offsetof(struct spdk_raid_bdev_opts, process_max_bandwidth_mb_sec), spdk_json_decode_uint32, true},
		};

		static void