Commit 081f8add authored by Jim Harris's avatar Jim Harris
Browse files

config: add SPDK_CONFIG_MAX_NUMA_NODES 



By default, to minimize impact to existing use cases, iobuf will
be configured to only support a single set of buffer pools, as it is
today. Users who want to enable support for per-NUMA node buffer
pools will need to specify --max-numa-nodes parameter to the
configure script.

Signed-off-by: default avatarJim Harris <jim.harris@samsung.com>
Change-Id: I8c7cd1a921e538bdb8e3af84ba27a23b5294d776
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/24538


Community-CI: Community CI Samsung <spdk.community.ci.samsung@gmail.com>
Reviewed-by: default avatarShuhei Matsumoto <smatsumoto@nvidia.com>
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Reviewed-by: default avatarBen Walker <ben@nvidia.com>
Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarKonrad Sztyber <konrad.sztyber@intel.com>
parent f1900e4d

CONFIG

+3 −0
Original line number Diff line number Diff line
@@ -236,6 +236,9 @@ CONFIG_AVAHI=n 
# Setup DPDK's RTE_MAX_LCORES

CONFIG_MAX_LCORES=128



# Maximum number of iobuf NUMA nodes supported

CONFIG_MAX_NUMA_NODES=1



# Build all Go components

CONFIG_GOLANG=n

configure

+18 −0
Original line number Diff line number Diff line
@@ -32,6 +32,8 @@ function usage() { 
	echo "                           by EAL, or enables autodetection if set to 'detect'. When 'detect'"

	echo "                           is specified, DPDK will detect number of cores in the system during"

	echo "                           compilation, and will set maximum number of lcores to this value"

	echo " --max-numa-nodes=VAL      VAL defines maximum number of NUMA nodes supported for iobuf"

	echo "                           buffer pools."

	echo " --enable-debug            Configure for debug builds"

	echo " --enable-werror           Treat compiler warnings as errors"

	echo " --enable-asan             Enable address sanitizer"
@@ -693,6 +695,15 @@ for i in "$@"; do 
			CONFIG[MAX_LCORES]="${i#*=}"

			CONFIG["MAX_LCORES"]=${CONFIG["MAX_LCORES"],,}

			;;

		--max-numa-nodes='')

			echo "Must specify max number of nodes for --max-numa-nodes"

			usage

			exit 1

			;;

		--max-numa-nodes=*)

			CONFIG[MAX_NUMA_NODES]="${i#*=}"

			CONFIG["MAX_NUMA_NODES"]=${CONFIG["MAX_NUMA_NODES"],,}

			;;

		--with-aio-fsdev)

			CONFIG[AIO_FSDEV]=y

			AIO_FSDEV_REQUIRED=y
@@ -1356,6 +1367,13 @@ if [[ -n ${CONFIG[MAX_LCORES]} ]]; then 
	fi

fi



if [[ -n ${CONFIG[MAX_NUMA_NODES]} ]]; then

	if [[ ! ${CONFIG[MAX_NUMA_NODES]} =~ ^([1-9][0-9]*)$ ]] || ((CONFIG[MAX_NUMA_NODES] > 16)); then

		echo "ERROR: Max number of lcores must be a decimal number in range [1..16] (given: ${CONFIG[MAX_LCORES]})"

		exit 1

	fi

fi



if [[ "${CONFIG[FSDEV]}" = "n" ]]; then

	if [[ "$AIO_FSDEV_REQUIRED" = "y" ]]; then

		echo "ERROR: --with-aio-fsdev cannot be specified as fsdev is disabled."

include/spdk/thread.h

+2 −3
Original line number Diff line number Diff line
@@ -15,6 +15,7 @@ 
#include <sys/epoll.h>

#endif



#include "spdk/config.h"

#include "spdk/fd_group.h"

#include "spdk/stdinc.h"

#include "spdk/assert.h"
@@ -1108,8 +1109,6 @@ struct spdk_iobuf_node_cache { 
	struct spdk_iobuf_pool_cache	large;

};



#define SPDK_IOBUF_MAX_NUMA_NODES	1



/** iobuf channel */

struct spdk_iobuf_channel {

	/** Module pointer */
@@ -1117,7 +1116,7 @@ struct spdk_iobuf_channel { 
	/** Parent IO channel */

	struct spdk_io_channel		*parent;

	/* Buffer cache */

	struct spdk_iobuf_node_cache	cache[SPDK_IOBUF_MAX_NUMA_NODES];

	struct spdk_iobuf_node_cache	cache[SPDK_CONFIG_MAX_NUMA_NODES];

};



/**

lib/thread/iobuf.c

+2 −2
Original line number Diff line number Diff line
@@ -35,7 +35,7 @@ struct iobuf_channel_node { 
};



struct iobuf_channel {

	struct iobuf_channel_node	node[SPDK_IOBUF_MAX_NUMA_NODES];

	struct iobuf_channel_node	node[SPDK_CONFIG_MAX_NUMA_NODES];

	struct spdk_iobuf_channel	*channels[IOBUF_MAX_CHANNELS];

};
@@ -56,7 +56,7 @@ struct iobuf { 
	TAILQ_HEAD(, iobuf_module)	modules;

	spdk_iobuf_finish_cb		finish_cb;

	void				*finish_arg;

	struct iobuf_node		node[SPDK_IOBUF_MAX_NUMA_NODES];

	struct iobuf_node		node[SPDK_CONFIG_MAX_NUMA_NODES];

};



static struct iobuf g_iobuf = {