Commit 3a02df0b authored by Jim Harris's avatar Jim Harris Committed by Konrad Sztyber
Browse files

event: add new 'mappings' parameter to static scheduler 



This is useful for testing purposes, to place specific spdk_threads
on specific cores. Specified core must be in the thread's cpumask.

Usage:

scripts/rpc.py framework_set_scheduler --mappings='2:0,5:1'

This will move thread 2 to core 0 and thread 5 to core 1.

Note: checking for failure when setting core-limit parameter on
static scheduler has been removed. The whole mechanism for determine
which optional parameters are accepted for different schedulers needs
to be rethought, the current mechanism is not really working.

Signed-off-by: default avatarJim Harris <jim.harris@samsung.com>
Change-Id: I16000805b8e2512f7ec6d84ee812d32bfcb2fd6d
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/24766


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarKonrad Sztyber <konrad.sztyber@intel.com>
Reviewed-by: default avatarChangpeng Liu <changpeliu@tencent.com>
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Community-CI: Mellanox Build Bot
Reviewed-by: default avatarAnisa Su <anisa.su@samsung.com>
Reviewed-by: default avatarShuhei Matsumoto <smatsumoto@nvidia.com>
parent 118c273a

lib/event/scheduler_static.c

+102 −0
Original line number Diff line number Diff line
@@ -9,6 +9,7 @@ 
#include "spdk/event.h"

#include "spdk/log.h"

#include "spdk/env.h"

#include "spdk/string.h"

#include "spdk/scheduler.h"



#include "spdk_internal/event.h"
@@ -66,10 +67,111 @@ balance_static(struct spdk_scheduler_core_info *cores, uint32_t core_count) 
	spdk_scheduler_set_period(0);

}



static const struct spdk_json_object_decoder static_sched_decoders[] = {

	{"mappings", 0, spdk_json_decode_string, true},

};



static int

set_opts_static(const struct spdk_json_val *opts)

{

	char *tok, *mappings = NULL, *copy;

	bool valid;



	if (opts != NULL) {

		if (spdk_json_decode_object_relaxed(opts, static_sched_decoders,

						    SPDK_COUNTOF(static_sched_decoders),

						    &mappings)) {

			SPDK_ERRLOG("Decoding scheduler opts JSON failed\n");

			return -EINVAL;

		}

	}



	if (mappings == NULL) {

		return 0;

	}



	copy = strdup(mappings);

	if (copy == NULL) {

		free(mappings);

		return -ENOMEM;

	}



	valid = true;

	tok = strtok(mappings, ":");

	while (tok) {

		struct spdk_lw_thread *lw_thread = NULL;

		struct spdk_thread *thread;

		int thread_id, core;



		thread_id = spdk_strtol(tok, 10);

		if (thread_id > 0) {

			thread = spdk_thread_get_by_id(thread_id);

			if (thread != NULL) {

				lw_thread = spdk_thread_get_ctx(thread);

			}

		}

		if (lw_thread == NULL) {

			SPDK_ERRLOG("invalid thread ID '%s' in mappings '%s'\n", tok, copy);

			valid = false;

			break;

		}



		tok = strtok(NULL, ",");

		core = spdk_strtol(tok, 10);

		if (core < 0 || spdk_reactor_get(core) == NULL) {

			SPDK_ERRLOG("invalid core number '%s' in mappings '%s'\n", tok, copy);

			valid = false;

			break;

		}



		if (!spdk_cpuset_get_cpu(spdk_thread_get_cpumask(thread), core)) {

			SPDK_ERRLOG("core %d not in thread %d cpumask\n", core, thread_id);

			valid = false;

			break;

		}



		tok = strtok(NULL, ":");

	}

	free(mappings);

	if (!valid) {

		free(copy);

		return -EINVAL;

	}



	tok = strtok(copy, ":");

	while (tok) {

		struct spdk_lw_thread *lw_thread = NULL;

		struct spdk_thread *thread;

		int thread_id, core;



		thread_id = spdk_strtol(tok, 10);

		thread = spdk_thread_get_by_id(thread_id);

		lw_thread = spdk_thread_get_ctx(thread);

		tok = strtok(NULL, ",");

		core = spdk_strtol(tok, 10);

		/* initial_lcore saves the static scheduler's lcore mapping.

		 * This is used to restore the previous mapping if we

		 * change to another scheduler and then back. So we can just

		 * change the ->initial_lcore here and kick the scheduler to

		 * put the new mapping into effect.

		 */

		lw_thread->initial_lcore = core;

		tok = strtok(NULL, ":");

	}

	free(copy);



	/* We have updated some core placements, so kick the scheduler to

	 * apply those new placements.

	 */

	spdk_scheduler_set_period(1);

	return 0;

}



static struct spdk_scheduler scheduler = {

	.name = "static",

	.init = init_static,

	.deinit = deinit_static,

	.balance = balance_static,

	.set_opts = set_opts_static,

};

SPDK_SCHEDULER_REGISTER(scheduler);

python/spdk/rpc/app.py

+3 −1
Original line number Diff line number Diff line
@@ -38,7 +38,7 @@ def framework_get_reactors(client): 




def framework_set_scheduler(client, name, period=None, load_limit=None, core_limit=None,

                            core_busy=None):

                            core_busy=None, mappings=None):

    """Select threads scheduler that will be activated and its period.



    Args:
@@ -56,6 +56,8 @@ def framework_set_scheduler(client, name, period=None, load_limit=None, core_lim 
        params['core_limit'] = core_limit

    if core_busy is not None:

        params['core_busy'] = core_busy

    if mappings is not None:

        params['mappings'] = mappings

    return client.call('framework_set_scheduler', params)

scripts/rpc.py

+3 −1
Original line number Diff line number Diff line
@@ -171,7 +171,8 @@ if __name__ == "__main__": 
                                        period=args.period,

                                        load_limit=args.load_limit,

                                        core_limit=args.core_limit,

                                        core_busy=args.core_busy)

                                        core_busy=args.core_busy,

                                        mappings=args.mappings)



    p = subparsers.add_parser(

        'framework_set_scheduler', help='Select thread scheduler that will be activated and its period (experimental)')
@@ -180,6 +181,7 @@ if __name__ == "__main__": 
    p.add_argument('--load-limit', help="Scheduler load limit. Reserved for dynamic scheduler", type=int)

    p.add_argument('--core-limit', help="Scheduler core limit. Reserved for dynamic scheduler", type=int)

    p.add_argument('--core-busy', help="Scheduler core busy limit. Reserved for dynamic scheduler", type=int)

    p.add_argument('--mappings', help="Comma-separated list of thread:core mappings. Reserved for static scheduler")

    p.set_defaults(func=framework_set_scheduler)



    def framework_get_scheduler(args):

test/scheduler/rpc.sh

+7 −3
Original line number Diff line number Diff line
@@ -45,9 +45,6 @@ function scheduler_opts() { 
	trap 'killprocess $spdk_pid; exit 1' SIGINT SIGTERM EXIT

	waitforlisten $spdk_pid



	# It should not be possible to change settings that a scheduler does not support

	NOT $rpc framework_set_scheduler static --core-limit 42



	# It is possible to change settings generic scheduler opts for schedulers in event framework

	$rpc framework_set_scheduler dynamic -p 424242

	[[ "$($rpc framework_get_scheduler | jq -r '. | select(.scheduler_name == "dynamic") | .scheduler_period')" -eq 424242 ]]
@@ -105,6 +102,13 @@ function static_as_default() { 
	$rpc framework_get_reactors \

		| jq -e -r ".reactors[] | select(.lcore == $other_cpu).lw_threads[] | select(.id == $thread_id)"



	# Explicitly move that thread to the main core

	$rpc framework_set_scheduler static --mappings "$thread_id:$main_cpu"



	# Check that this thread is now on main core

	$rpc framework_get_reactors \

		| jq -e -r ".reactors[] | select(.lcore == $main_cpu).lw_threads[] | select(.id == $thread_id)"



	trap - SIGINT SIGTERM EXIT

	killprocess $spdk_pid

}