Commit b3a57634 authored by Marcin Spiewak's avatar Marcin Spiewak Committed by Jim Harris
Browse files

app: use --lcores to map ids greater than 128 



Fixes #2812

This patch adds	support	for '--lcores <map_list>'
parameter in spdk.
This parameter allow mapping of the lcores
to CPU IDs, if the system contains CPUs with IDs
greater	or equal to 128 (RTE_MAX_LCORE). Such CPUs
can not be directly included in core mask
specified in '-m <mask>' parameter, as the dpdk
rejects cores if IDs are greater than 127.
The only way to	use them in spdk is to map lcore
to CPU using --lcores parameters specified
in command line.
--lcores and -m parameters are mutually
exclusive. Please use only one of them.
Examples:
build/bin/nvmf_tgt --lcores 0@130
build/bin/nvmf_tgt --lcores 0@150,1@151
build/bin/nvmf_tgt --lcores "(5-7)@(10-12)"
build/bin/nvmf_tgt --lcores "(5-7)@(136,138,140)"

Change-Id: Ia92be4499c8daaa936b1a4357d52ae303d6f3eb1
Signed-off-by: default avatarMarcin Spiewak <marcin.spiewak@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/17403


Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Reviewed-by: default avatarAleksey Marchuk <alexeymar@nvidia.com>
Reviewed-by: default avatarShuhei Matsumoto <smatsumoto@nvidia.com>
parent cc541400

lib/env_dpdk/init.c

+28 −13
Original line number Diff line number Diff line
@@ -225,6 +225,22 @@ build_eal_cmdline(const struct spdk_env_opts *opts) 
		}

	}



	/* Either lcore_map or core_mask must be set. If both, or none specified, fail */

	if ((opts->core_mask == NULL) == (opts->lcore_map == NULL)) {

		if (opts->core_mask && opts->lcore_map) {

			fprintf(stderr,

				"Both, lcore map and core mask are provided, while only one can be set\n");

		} else {

			fprintf(stderr, "Core mask or lcore map must be specified\n");

		}

		free_args(args, argcount);

		return -1;

	}



	if (opts->lcore_map) {

		/* If lcore list is set, generate --lcores parameter */

		args = push_arg(args, &argcount, _sprintf_alloc("--lcores=%s", opts->lcore_map));

	} else if (opts->core_mask[0] == '-') {

		/*

		 * Set the coremask:

		 *
@@ -237,7 +253,6 @@ build_eal_cmdline(const struct spdk_env_opts *opts) 
		 * - otherwise, it's a CPU mask of the form "0xff.." as expected by the

		 *   -c option.

		 */

	if (opts->core_mask[0] == '-') {

		args = push_arg(args, &argcount, _sprintf_alloc("%s", opts->core_mask));

	} else if (opts->core_mask[0] == '[') {

		char *l_arg = _sprintf_alloc("-l %s", opts->core_mask + 1);

lib/event/app.c

+27 −1
Original line number Diff line number Diff line
@@ -129,6 +129,8 @@ static const struct option g_cmdline_options[] = { 
	{"vfio-vf-token",		required_argument,	NULL, ENV_VF_TOKEN_OPT_IDX},

#define MSG_MEMPOOL_SIZE_OPT_IDX 270

	{"msg-mempool-size",		required_argument,	NULL, MSG_MEMPOOL_SIZE_OPT_IDX},

#define LCORES_OPT_IDX	271

	{"lcores",			required_argument,	NULL, LCORES_OPT_IDX},

};



static void
@@ -203,7 +205,6 @@ spdk_app_opts_init(struct spdk_app_opts *opts, size_t opts_size) 
	SET_FIELD(mem_size, SPDK_APP_DPDK_DEFAULT_MEM_SIZE);

	SET_FIELD(main_core, SPDK_APP_DPDK_DEFAULT_MAIN_CORE);

	SET_FIELD(mem_channel, SPDK_APP_DPDK_DEFAULT_MEM_CHANNEL);

	SET_FIELD(reactor_mask, SPDK_APP_DPDK_DEFAULT_CORE_MASK);

	SET_FIELD(base_virtaddr, SPDK_APP_DPDK_DEFAULT_BASE_VIRTADDR);

	SET_FIELD(print_level, SPDK_APP_DEFAULT_LOG_PRINT_LEVEL);

	SET_FIELD(rpc_addr, SPDK_DEFAULT_RPC_ADDR);
@@ -326,6 +327,7 @@ app_setup_env(struct spdk_app_opts *opts) 


	env_opts.name = opts->name;

	env_opts.core_mask = opts->reactor_mask;

	env_opts.lcore_map = opts->lcore_map;

	env_opts.shm_id = opts->shm_id;

	env_opts.mem_channel = opts->mem_channel;

	env_opts.main_core = opts->main_core;
@@ -497,6 +499,7 @@ app_copy_opts(struct spdk_app_opts *opts, struct spdk_app_opts *opts_user, size_ 
	SET_FIELD(json_config_ignore_errors);

	SET_FIELD(rpc_addr);

	SET_FIELD(reactor_mask);

	SET_FIELD(lcore_map);

	SET_FIELD(tpoint_group_mask);

	SET_FIELD(shm_id);

	SET_FIELD(shutdown_cb);
@@ -669,6 +672,11 @@ spdk_app_start(struct spdk_app_opts *opts_user, spdk_msg_fn start_fn, 
		return 1;

	}



	if (!(opts->lcore_map || opts->reactor_mask)) {

		/* Set default CPU mask */

		opts->reactor_mask = SPDK_APP_DPDK_DEFAULT_CORE_MASK;

	}



	tty = ttyname(STDERR_FILENO);

	if (opts->print_level > SPDK_LOG_WARN &&

	    isatty(STDERR_FILENO) &&
@@ -860,6 +868,13 @@ usage(void (*app_usage)(void)) 
	printf(" -h, --help                show this usage\n");

	printf(" -i, --shm-id <id>         shared memory ID (optional)\n");

	printf(" -m, --cpumask <mask or list>    core mask (like 0xF) or core list of '[]' embraced (like [0,1,10]) for DPDK\n");

	printf("     --lcores <list>       lcore to CPU mapping list. The list is in the format:\n");

	printf("                           <lcores[@CPUs]>[<,lcores[@CPUs]>...]\n");

	printf("                           lcores and cpus list are grouped by '(' and ')', e.g '--lcores \"(5-7)@(10-12)\"'\n");

	printf("                           Within the group, '-' is used for range separator,\n");

	printf("                           ',' is used for single number separator.\n");

	printf("                           '( )' can be omitted for single element group,\n");

	printf("                           '@' can be omitted if cpus and lcores have the same value\n");

	printf(" -n, --mem-channels <num>  channel number of memory channels used for DPDK\n");

	printf(" -p, --main-core <id>      main (primary) core for DPDK\n");

	printf(" -r, --rpc-socket <path>   RPC listen address (default %s)\n", SPDK_DEFAULT_RPC_ADDR);
@@ -1001,8 +1016,19 @@ spdk_app_parse_args(int argc, char **argv, struct spdk_app_opts *opts, 
			}

			break;

		case CPUMASK_OPT_IDX:

			if (opts->lcore_map) {

				SPDK_ERRLOG("lcore map and core mask can't be set simultaneously\n");

				goto out;

			}

			opts->reactor_mask = optarg;

			break;

		case LCORES_OPT_IDX:

			if (opts->reactor_mask) {

				SPDK_ERRLOG("lcore map and core mask can't be set simultaneously\n");

				goto out;

			}

			opts->lcore_map = optarg;

			break;

		case DISABLE_CPUMASK_LOCKS_OPT_IDX:

			g_disable_cpumask_locks = true;

			break;