Commit 38a30da5 authored by Yuri's avatar Yuri Committed by Tomasz Zawadzki
Browse files

nvmf: accepts new parameter to set poll groups mask 



Parses and verifies hexadecimal cpu bit mask specified by the user.
Added verification to check for cpu cores range, making sure poll groups cores
assigned within the range of cpu cores allocated for the application.

RPC nvmf_set_config now takes an argument to configure ‘poll groups’,
a new parameter for NVMf subsystem. This parameter sets a CPU mask
to spawn threads which run an event loop for a ‘poll group’.

Change-Id: Ied9081c2213715ec94de00a8b37153730b8ac2ed
Signed-off-by: default avatarYuri <yuriy.kirichok@hpe.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/5611


Community-CI: Mellanox Build Bot
Reviewed-by: default avatarMatt Dumm <matt.dumm@hpe.com>
Reviewed-by: default avatarAleksey Marchuk <alexeymar@mellanox.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
parent ea0aaf5e

CHANGELOG.md

+4 −0
Original line number Diff line number Diff line
@@ -116,6 +116,10 @@ is associated with a fabrics controller. 
Added `min_cntlid` and `max_cntlid` to `nvmf_create_subsystem` to limit the controller ID range.

Added `spdk_nvmf_subsystem_get_min_cntlid` and `spdk_nvmf_subsystem_get_max_cntlid` to request those values.



A new parameter, `poll_groups_mask` was added to the nvmf_set_config RPC that allows specifying

a subset of cores for the nvmf poll groups. This helps to avoid imbalances when some cores are

busy with periodic timer tasks that run very frequently.



`spdk_nvmf_request_get_buffers_multi` API is removed.



Added the `nvmf_set_crdt` RPC for setting command retry delay times.

doc/jsonrpc.md

+1 −0
Original line number Diff line number Diff line
@@ -6783,6 +6783,7 @@ Name | Optional | Type | Description 
----------------------- | -------- | ----------- | -----------

acceptor_poll_rate      | Optional | number      | Polling interval of the acceptor for incoming connections (microseconds)

admin_cmd_passthru      | Optional | object      | Admin command passthru configuration

poll_groups_mask        | Optional | string      | Set cpumask for NVMf poll groups



### admin_cmd_passthru {#spdk_nvmf_admin_passthru_conf}

module/event/subsystems/nvmf/nvmf_rpc.c

+49 −1
Original line number Diff line number Diff line
@@ -35,6 +35,7 @@ 


#include "spdk/rpc.h"

#include "spdk/util.h"

#include "spdk/cpuset.h"



static const struct spdk_json_object_decoder nvmf_rpc_subsystem_tgt_opts_decoder[] = {

	{"max_subsystems", 0, spdk_json_decode_uint32, true}
@@ -98,10 +99,57 @@ static int decode_admin_passthru(const struct spdk_json_val *val, void *out) 
	return 0;

}



static int

nvmf_is_subset_of_env_core_mask(const struct spdk_cpuset *set)

{

	uint32_t i, tmp_counter = 0;



	SPDK_ENV_FOREACH_CORE(i) {

		if (spdk_cpuset_get_cpu(set, i)) {

			++tmp_counter;

		}

	}

	return spdk_cpuset_count(set) - tmp_counter;

}



static int

nvmf_decode_poll_groups_mask(const struct spdk_json_val *val, void *out)

{

	char *mask = spdk_json_strdup(val);

	int ret = -1;



	if (mask == NULL) {

		return -1;

	}



	if (!(g_poll_groups_mask = spdk_cpuset_alloc())) {

		SPDK_ERRLOG("Unable to allocate a poll groups mask object in nvmf_decode_poll_groups_mask.\n");

		free(mask);

		return -1;

	}



	ret = spdk_cpuset_parse(g_poll_groups_mask, mask);

	free(mask);

	if (ret == 0) {

		if (nvmf_is_subset_of_env_core_mask(g_poll_groups_mask) == 0) {

			return 0;

		} else {

			SPDK_ERRLOG("Poll groups cpumask 0x%s is out of range\n", spdk_cpuset_fmt(g_poll_groups_mask));

		}

	} else {

		SPDK_ERRLOG("Invalid cpumask\n");

	}



	spdk_cpuset_free(g_poll_groups_mask);

	g_poll_groups_mask = NULL;

	return -1;

}



static const struct spdk_json_object_decoder nvmf_rpc_subsystem_tgt_conf_decoder[] = {

	{"acceptor_poll_rate", offsetof(struct spdk_nvmf_tgt_conf, acceptor_poll_rate), spdk_json_decode_uint32, true},

	{"conn_sched", offsetof(struct spdk_nvmf_tgt_conf, conn_sched), decode_conn_sched, true},

	{"admin_cmd_passthru", offsetof(struct spdk_nvmf_tgt_conf, admin_passthru), decode_admin_passthru, true}

	{"admin_cmd_passthru", offsetof(struct spdk_nvmf_tgt_conf, admin_passthru), decode_admin_passthru, true},

	{"poll_groups_mask", 0, nvmf_decode_poll_groups_mask, true}

};



static void

module/event/subsystems/nvmf/nvmf_tgt.c

+3 −0
Original line number Diff line number Diff line
@@ -488,6 +488,9 @@ nvmf_subsystem_write_config_json(struct spdk_json_write_ctx *w) 
	spdk_json_write_named_bool(w, "identify_ctrlr",

				   g_spdk_nvmf_tgt_conf.admin_passthru.identify_ctrlr);

	spdk_json_write_object_end(w);

	if (g_poll_groups_mask) {

		spdk_json_write_named_string(w, "poll_groups_mask", spdk_cpuset_fmt(g_poll_groups_mask));

	}

	spdk_json_write_object_end(w);

	spdk_json_write_object_end(w);

scripts/rpc.py

+3 −1
Original line number Diff line number Diff line
@@ -1895,7 +1895,8 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse 
        rpc.nvmf.nvmf_set_config(args.client,

                                 acceptor_poll_rate=args.acceptor_poll_rate,

                                 conn_sched=args.conn_sched,

                                 passthru_identify_ctrlr=args.passthru_identify_ctrlr)

                                 passthru_identify_ctrlr=args.passthru_identify_ctrlr,

                                 poll_groups_mask=args.poll_groups_mask)



    p = subparsers.add_parser('nvmf_set_config', aliases=['set_nvmf_target_config'],

                              help='Set NVMf target config')
@@ -1903,6 +1904,7 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse 
    p.add_argument('-s', '--conn-sched', help='(Deprecated). Ignored.')

    p.add_argument('-i', '--passthru-identify-ctrlr', help="""Passthrough fields like serial number and model number

    when the controller has a single namespace that is an NVMe bdev""", action='store_true')

    p.add_argument('-m', '--poll-groups-mask', help='Set cpumask for NVMf poll groups (optional)', type=str)

    p.set_defaults(func=nvmf_set_config)



    def nvmf_create_transport(args):