Commit 42d1bd28 authored by Jim Harris's avatar Jim Harris
Browse files

thread: add enable_numa parameter to iobuf_set_options RPC 



This parameter does not yet actually enable per-NUMA node iobuf
buffer pools. It only checks that the application was built with
support for the number of NUMA nodes reported by the system.

Signed-off-by: default avatarJim Harris <jim.harris@samsung.com>
Change-Id: I1b9d11ccb8f6914280874a40754c51625d21645d
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/24539


Reviewed-by: default avatarBen Walker <ben@nvidia.com>
Reviewed-by: default avatarAleksey Marchuk <alexeymar@nvidia.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Community CI Samsung <spdk.community.ci.samsung@gmail.com>
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Reviewed-by: default avatarKonrad Sztyber <konrad.sztyber@intel.com>
Community-CI: Mellanox Build Bot
parent 081f8add

doc/jsonrpc.md

+1 −0
Original line number Diff line number Diff line
@@ -13269,6 +13269,7 @@ small_pool_count | Optional | number | Number of small buffers in th 
large_pool_count        | Optional | number      | Number of large buffers in the global pool

small_bufsize           | Optional | number      | Size of a small buffer

large_bufsize           | Optional | number      | Size of a small buffer

enable_numa             | Optional | boolean     | Enable per-NUMA node buffer pools. Each node will allocate a full pool based on small_pool_count and large_pool_count.



#### Example

include/spdk/thread.h

+2 −0
Original line number Diff line number Diff line
@@ -1050,6 +1050,8 @@ struct spdk_iobuf_opts { 
	 */

	size_t opts_size;



	/** Enable per-NUMA node buffer pools */

	uint8_t	enable_numa;

};



struct spdk_iobuf_pool_stats {

lib/thread/iobuf.c

+13 −1
Original line number Diff line number Diff line
@@ -282,6 +282,16 @@ spdk_iobuf_set_opts(const struct spdk_iobuf_opts *opts) 
		return -EINVAL;

	}



	if (opts->enable_numa &&

	    spdk_env_get_last_numa_id() >= SPDK_CONFIG_MAX_NUMA_NODES) {

		SPDK_ERRLOG("max NUMA ID %" PRIu32 " cannot be supported with "

			    "SPDK_CONFIG_MAX_NUMA_NODES %" PRIu32 "\n",

			    spdk_env_get_last_numa_id(), SPDK_CONFIG_MAX_NUMA_NODES);

		SPDK_ERRLOG("Re-configure with --max-numa-nodes=%" PRIu32 "\n",

			    spdk_env_get_last_numa_id() + 1);

		return -EINVAL;

	}



#define SET_FIELD(field) \

        if (offsetof(struct spdk_iobuf_opts, field) + sizeof(opts->field) <= opts->opts_size) { \

                g_iobuf.opts.field = opts->field; \
@@ -291,6 +301,7 @@ spdk_iobuf_set_opts(const struct spdk_iobuf_opts *opts) 
	SET_FIELD(large_pool_count);

	SET_FIELD(small_bufsize);

	SET_FIELD(large_bufsize);

	SET_FIELD(enable_numa);



	g_iobuf.opts.opts_size = opts->opts_size;
@@ -323,12 +334,13 @@ spdk_iobuf_get_opts(struct spdk_iobuf_opts *opts, size_t opts_size) 
	SET_FIELD(large_pool_count);

	SET_FIELD(small_bufsize);

	SET_FIELD(large_bufsize);

	SET_FIELD(enable_numa);



#undef SET_FIELD



	/* Do not remove this statement, you should always update this statement when you adding a new field,

	 * and do not forget to add the SET_FIELD statement for your added field. */

	SPDK_STATIC_ASSERT(sizeof(struct spdk_iobuf_opts) == 32, "Incorrect size");

	SPDK_STATIC_ASSERT(sizeof(struct spdk_iobuf_opts) == 40, "Incorrect size");

}

module/event/subsystems/iobuf/iobuf.c

+1 −0
Original line number Diff line number Diff line
@@ -51,6 +51,7 @@ iobuf_write_config_json(struct spdk_json_write_ctx *w) 
	spdk_json_write_named_uint64(w, "large_pool_count", opts.large_pool_count);

	spdk_json_write_named_uint32(w, "small_bufsize", opts.small_bufsize);

	spdk_json_write_named_uint32(w, "large_bufsize", opts.large_bufsize);

	spdk_json_write_named_bool(w, "enable_numa", opts.enable_numa);

	spdk_json_write_object_end(w);

	spdk_json_write_object_end(w);

module/event/subsystems/iobuf/iobuf_rpc.c

+1 −0
Original line number Diff line number Diff line
@@ -14,6 +14,7 @@ static const struct spdk_json_object_decoder rpc_iobuf_set_options_decoders[] = 
	{"large_pool_count", offsetof(struct spdk_iobuf_opts, large_pool_count), spdk_json_decode_uint64, true},

	{"small_bufsize", offsetof(struct spdk_iobuf_opts, small_bufsize), spdk_json_decode_uint32, true},

	{"large_bufsize", offsetof(struct spdk_iobuf_opts, large_bufsize), spdk_json_decode_uint32, true},

	{"enable_numa", offsetof(struct spdk_iobuf_opts, enable_numa), spdk_json_decode_bool, true},

};



static void